From 1779b5927128262f3d9de68d194bfa37394bafb7 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 12:23:39 +0000 Subject: [PATCH 01/11] disable parallel builds; enable batch mode to get rid of transfer progress --- integration-tests/script/copy_resources_template.sh | 2 +- it.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/integration-tests/script/copy_resources_template.sh b/integration-tests/script/copy_resources_template.sh index fade47c4e315..3f584cf4129e 100755 --- a/integration-tests/script/copy_resources_template.sh +++ b/integration-tests/script/copy_resources_template.sh @@ -30,7 +30,7 @@ cp -R docker $SHARED_DIR/docker pushd ../ rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin -mvn -Pskip-static-checks,skip-tests -T1C -Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true install -Pintegration-test +mvn -B -Pskip-static-checks,skip-tests -Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true install -Pintegration-test mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin $SHARED_DIR/docker/bin mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions diff --git a/it.sh b/it.sh index ce4e8e8483e1..5f9ff491cb9b 100755 --- a/it.sh +++ b/it.sh @@ -232,17 +232,17 @@ case $CMD in mvn -q clean install dependency:go-offline -P dist $MAVEN_IGNORE ;; "build" ) - mvn clean install -P dist $MAVEN_IGNORE -T1.0C $* + mvn -B clean install -P dist $MAVEN_IGNORE $* ;; "dist" ) - mvn install -P dist $MAVEN_IGNORE -pl :distribution + mvn -B install -P dist $MAVEN_IGNORE -pl :distribution ;; "tools" ) - mvn install -pl :druid-it-tools + mvn -B install -pl :druid-it-tools ;; "image" ) cd $DRUID_DEV/integration-tests-ex/image - mvn install -P test-image $MAVEN_IGNORE + mvn -B install -P test-image $MAVEN_IGNORE ;; "gen") # Generate the docker-compose.yaml files. Mostly for debugging @@ -264,7 +264,7 @@ case $CMD in "run" ) require_category reuse_override - mvn $TEST_OPTIONS -P IT-$CATEGORY -pl $MAVEN_PROJECT + mvn -B $TEST_OPTIONS -P IT-$CATEGORY -pl $MAVEN_PROJECT ;; "test" ) require_category @@ -275,7 +275,7 @@ case $CMD in # Run the test. On failure, still shut down the cluster. # Return Maven's return code as the script's return code. set +e - mvn $TEST_OPTIONS -P IT-$CATEGORY -pl $MAVEN_PROJECT + mvn -B $TEST_OPTIONS -P IT-$CATEGORY -pl $MAVEN_PROJECT RESULT=$? set -e $IT_CASES_DIR/cluster.sh down $CATEGORY From e8b50b0156b8906577e12ced00eae36a524f2bf0 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 12:35:53 +0000 Subject: [PATCH 02/11] restore .m2 from setup-java if not found --- .github/workflows/standard-its.yml | 1 + .github/workflows/unit-and-integration-tests-unified.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index f44dee04bdf8..2d9f4ba719de 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -159,6 +159,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-8-${{ github.sha }} + restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml)}} - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index ff963dac7718..41fe75bd1e47 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -72,6 +72,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }} + restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml)}} - name: Cache targets id: target From cb01977a331e8126f8aa58abef6aad132712a8c9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 13:50:02 +0000 Subject: [PATCH 03/11] some change to sql --- .../org/apache/druid/sql/calcite/CalciteSelectQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index a7ad956baee3..24f02bcd87ff 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -258,8 +258,8 @@ public void testSelectNonNumericNumberLiterals() ); } - // Test that the integers are getting correctly casted after being passed through a function when not selecting from - // a table + // Test that the integers are getting correctly casted after being passed through a function + // when not selecting from a table @Test public void testDruidLogicalValuesRule() { From 471a59b498fc673a7211e97f6e67050056b343b1 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 14:00:10 +0000 Subject: [PATCH 04/11] add ws --- .github/workflows/standard-its.yml | 2 +- .github/workflows/unit-and-integration-tests-unified.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index 2d9f4ba719de..bb58c92fe7b0 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -159,7 +159,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-8-${{ github.sha }} - restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml)}} + restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml) }} - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 41fe75bd1e47..2a69eaa1860d 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -72,7 +72,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }} - restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml)}} + restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml) }} - name: Cache targets id: target From 83984a72e4371a3777fdbb538fff1716f9db6804 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 14:01:58 +0000 Subject: [PATCH 05/11] fix quote --- .github/workflows/standard-its.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/standard-its.yml b/.github/workflows/standard-its.yml index bb58c92fe7b0..2648dc0993b6 100644 --- a/.github/workflows/standard-its.yml +++ b/.github/workflows/standard-its.yml @@ -159,7 +159,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-8-${{ github.sha }} - restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml) }} + restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }} - name: Maven build if: steps.maven-restore.outputs.cache-hit != 'true' From 124b37fb9b84fd514f4fbf75b96b58038b7faf7c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 14:02:58 +0000 Subject: [PATCH 06/11] fix quote --- .github/workflows/unit-and-integration-tests-unified.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-and-integration-tests-unified.yml b/.github/workflows/unit-and-integration-tests-unified.yml index 2a69eaa1860d..6ff6c8bd6500 100644 --- a/.github/workflows/unit-and-integration-tests-unified.yml +++ b/.github/workflows/unit-and-integration-tests-unified.yml @@ -72,7 +72,7 @@ jobs: with: path: ~/.m2/repository key: maven-${{ runner.os }}-${{ matrix.jdk }}-${{ github.sha }} - restore-keys: setup-java-Linux-maven-${{ hashFiles(**/pom.xml) }} + restore-keys: setup-java-Linux-maven-${{ hashFiles('**/pom.xml') }} - name: Cache targets id: target From 18ad307d76c32f1f9b4a89fd749fe63f37447c60 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 14:42:11 +0000 Subject: [PATCH 07/11] undo querytest change --- .../org/apache/druid/sql/calcite/CalciteSelectQueryTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index 24f02bcd87ff..a7ad956baee3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -258,8 +258,8 @@ public void testSelectNonNumericNumberLiterals() ); } - // Test that the integers are getting correctly casted after being passed through a function - // when not selecting from a table + // Test that the integers are getting correctly casted after being passed through a function when not selecting from + // a table @Test public void testDruidLogicalValuesRule() { From 438f2c54c90849474924888c47cea5556d1b9f53 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 27 Sep 2023 15:06:38 +0000 Subject: [PATCH 08/11] nullhandling in mvtest --- .../MaterializedViewQueryQueryToolChestTest.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java index b96ff17f670b..3b230d706bf2 100644 --- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryQueryToolChestTest.java @@ -23,6 +23,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableMap; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.data.input.MapBasedRow; import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.DateTimes; @@ -55,6 +56,10 @@ public class MaterializedViewQueryQueryToolChestTest { + static { + NullHandling.initializeForTests(); + } + private static final ObjectMapper JSON_MAPPER = new DefaultObjectMapper(); @Test @@ -244,7 +249,5 @@ public void testGetRealQuery() )); Assert.assertEquals(realQuery, materializedViewQueryQueryToolChest.getRealQuery(materializedViewQuery)); - } - } From a71f13680cd2d347c058b5c74043a3dc9a35b574 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 28 Sep 2023 06:14:01 +0000 Subject: [PATCH 09/11] init more --- .../query/materializedview/DatasourceOptimizerTest.java | 7 ++++++- .../query/materializedview/MaterializedViewQueryTest.java | 5 +++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java index 673c2531999c..80ade5c1c9c7 100644 --- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/DatasourceOptimizerTest.java @@ -34,6 +34,7 @@ import org.apache.druid.client.DruidServer; import org.apache.druid.client.selector.HighestPriorityTierSelectorStrategy; import org.apache.druid.client.selector.RandomServerSelectorStrategy; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.curator.CuratorTestBase; import org.apache.druid.indexing.materializedview.DerivativeDataSourceMetadata; import org.apache.druid.jackson.DefaultObjectMapper; @@ -72,6 +73,10 @@ public class DatasourceOptimizerTest extends CuratorTestBase { + static { + NullHandling.initializeForTests(); + } + @Rule public final TestDerbyConnector.DerbyConnectorRule derbyConnectorRule = new TestDerbyConnector.DerbyConnectorRule(); private DerivativeDataSourceManager derivativesManager; @@ -142,7 +147,7 @@ public void testOptimize() throws InterruptedException Set metrics = Sets.newHashSet("cost"); DerivativeDataSourceMetadata metadata = new DerivativeDataSourceMetadata(baseDataSource, dims, metrics); metadataStorageCoordinator.insertDataSourceMetadata(dataSource, metadata); - // insert base datasource segments + // insert base datasource segments List baseResult = Lists.transform( ImmutableList.of( "2011-04-01/2011-04-02", diff --git a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java index ad9913eca555..cb52e1b2a3bb 100644 --- a/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java +++ b/extensions-contrib/materialized-view-selection/src/test/java/org/apache/druid/query/materializedview/MaterializedViewQueryTest.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Iterables; import com.google.common.collect.Lists; +import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.HumanReadableBytes; import org.apache.druid.math.expr.ExprMacroTable; import org.apache.druid.query.Query; @@ -45,6 +46,10 @@ public class MaterializedViewQueryTest { + static { + NullHandling.initializeForTests(); + } + private static final ObjectMapper JSON_MAPPER = TestHelper.makeJsonMapper(); private DataSourceOptimizer optimizer; From 1ea72c1879be1a55be2804fd855db3b31155af89 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 28 Sep 2023 10:46:33 +0200 Subject: [PATCH 10/11] skip commitid plugin --- pom.xml | 1 + 1 file changed, 1 insertion(+) diff --git a/pom.xml b/pom.xml index 3f2dddfb2bb9..e34de8a287f9 100644 --- a/pom.xml +++ b/pom.xml @@ -2106,6 +2106,7 @@ true true true + true From 52873b0353539aaab31c38d56ab366974eb7919f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 28 Sep 2023 12:43:28 +0200 Subject: [PATCH 11/11] add-back 1.0C to build ; remove redundant skip-s from copy-resources; add comment --- integration-tests/script/copy_resources_template.sh | 3 ++- it.sh | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integration-tests/script/copy_resources_template.sh b/integration-tests/script/copy_resources_template.sh index 3f584cf4129e..7f1c41802473 100755 --- a/integration-tests/script/copy_resources_template.sh +++ b/integration-tests/script/copy_resources_template.sh @@ -30,7 +30,8 @@ cp -R docker $SHARED_DIR/docker pushd ../ rm -rf distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin -mvn -B -Pskip-static-checks,skip-tests -Danimal.sniffer.skip=true -Dcheckstyle.skip=true -Dweb.console.skip=true -Dcyclonedx.skip=true -Denforcer.skip=true -Dforbiddenapis.skip=true -Dmaven.javadoc.skip=true -Dpmd.skip=true -Dspotbugs.skip=true install -Pintegration-test +# using parallel build here may not yield significant speedups +mvn -B -Pskip-static-checks,skip-tests -Dweb.console.skip=true install -Pintegration-test mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/bin $SHARED_DIR/docker/bin mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/lib $SHARED_DIR/docker/lib mv distribution/target/apache-druid-$DRUID_VERSION-integration-test-bin/extensions $SHARED_DIR/docker/extensions diff --git a/it.sh b/it.sh index 5f9ff491cb9b..12519c24b54d 100755 --- a/it.sh +++ b/it.sh @@ -232,7 +232,7 @@ case $CMD in mvn -q clean install dependency:go-offline -P dist $MAVEN_IGNORE ;; "build" ) - mvn -B clean install -P dist $MAVEN_IGNORE $* + mvn -B clean install -P dist $MAVEN_IGNORE -T1.0C $* ;; "dist" ) mvn -B install -P dist $MAVEN_IGNORE -pl :distribution