diff --git a/.github/scripts/web-checks.sh b/.github/scripts/web-checks.sh
index 8f39c16a47f8..a1588c695a21 100755
--- a/.github/scripts/web-checks.sh
+++ b/.github/scripts/web-checks.sh
@@ -20,7 +20,8 @@ set -x
# Install node.js via maven frontend plugin (same approach as jacoco script)
./mvnw -B com.github.eirslett:frontend-maven-plugin:install-node-and-npm@install-node-and-npm -pl web-console/
-PATH+=:web-console/target/node/
+# Absolute, as the web console end-to-end tests run npx from another directory
+PATH+=:$(pwd)/web-console/target/node/
# docs
(cd website && npm ci)
@@ -30,12 +31,9 @@ npm run link-lint
npm run spellcheck
cd ..
-# web console
-./mvnw -B test -pl 'web-console'
+# web console (installed with the console built, for the Router of the end-to-end tests to serve)
+./mvnw -B install -pl 'web-console'
-# web console end-to-end test
-./.github/scripts/setup_generate_license.sh
-web-console/script/druid build
-web-console/script/druid start
-(cd web-console && npm run test-e2e)
-web-console/script/druid stop
+# web console end-to-end tests, on embedded clusters (builds the modules embedded-tests needs, not web-console again)
+./mvnw -B verify -pl 'embedded-tests,!web-console' -am -Pweb-console-tests,skip-static-checks -DskipUTs \
+ -Dmaven.javadoc.skip=true -Danimal.sniffer.skip=true -Denforcer.skip=true -Dcyclonedx.skip=true
diff --git a/.github/workflows/worker.yml b/.github/workflows/worker.yml
index fa0474460a87..4cd829fbfc34 100644
--- a/.github/workflows/worker.yml
+++ b/.github/workflows/worker.yml
@@ -161,4 +161,5 @@ jobs:
**/TEST-*.xml
**/target/jacoco*.exec
**/target/ci-debug/**
- distribution/target/apache-druid-*/log/**
+ embedded-tests/target/failsafe-reports/**
+ web-console/test-results/**
diff --git a/embedded-tests/pom.xml b/embedded-tests/pom.xml
index 3928027576ae..974c64248a4a 100644
--- a/embedded-tests/pom.xml
+++ b/embedded-tests/pom.xml
@@ -890,12 +890,12 @@
true
-
+
org.apache.maven.pluginsmaven-surefire-plugin
- docker-test,perf
+ docker-test,web-console,perf
+ web-console-tests
+
+
+ true
+
+
+
+
+ org.apache.maven.plugins
+ maven-surefire-plugin
+
+ true
+
+
+
+ org.apache.maven.plugins
+ maven-failsafe-plugin
+
+ web-console
+
+ *WebConsoleTest*
+
+
+ ${project.basedir}/../web-console
+
+
+ -XX:+UseCompactObjectHeaders
+ ${jdk.strong.encapsulation.argLine}
+ -Xmx3g
+ -XX:MaxDirectMemorySize=2500m
+ -XX:+ExitOnOutOfMemoryError
+ -XX:+HeapDumpOnOutOfMemoryError
+ -Duser.language=en
+ -Dfile.encoding=UTF-8
+ -Duser.timezone=UTC
+ -Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager
+ -Daws.region=us-east-1
+
+
+
+
+
+ integration-test
+ verify
+
+
+
+
+
+
+
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionWebConsoleTest.java
new file mode 100644
index 000000000000..8db123a0fc70
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/compact/AutoCompactionWebConsoleTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.compact;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Sets up auto-compaction from the Datasources view of the web console, and forces compaction runs from there.
+ */
+public class AutoCompactionWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testCompactionConfigAndForcedRun() throws Exception
+ {
+ runSpec("auto-compaction.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleSpecsTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleSpecsTest.java
new file mode 100644
index 000000000000..6b147a939216
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleSpecsTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.console;
+
+import com.google.common.collect.Sets;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Checks that every Playwright spec of {@code web-console/e2e-tests} is run by a {@link WebConsoleTestBase} test (the
+ * specs are passed to {@link WebConsoleTestBase#runSpec} by name), and that those tests run only specs that exist.
+ * It needs no cluster, so it runs with the unit tests rather than in the {@code web-console-tests} profile.
+ */
+public class WebConsoleSpecsTest
+{
+ private static final Pattern RUN_SPEC = Pattern.compile("runSpec\\(\\s*\"([^\"]+)\"");
+
+ @Test
+ public void testEverySpecIsRun() throws IOException
+ {
+ final Set specs;
+ try (Stream files = Files.list(new File(WebConsoleTestBase.webConsoleDir(), "e2e-tests").toPath())) {
+ specs = files.map(file -> file.getFileName().toString())
+ .filter(name -> name.endsWith(".spec.ts"))
+ .collect(Collectors.toCollection(TreeSet::new));
+ }
+
+ final Set runSpecs = new TreeSet<>();
+ try (Stream files = Files.walk(Path.of("src", "test", "java"))) {
+ for (Path file : files.filter(f -> f.getFileName().toString().endsWith("WebConsoleTest.java")).toList()) {
+ final Matcher matcher = RUN_SPEC.matcher(Files.readString(file, StandardCharsets.UTF_8));
+ while (matcher.find()) {
+ runSpecs.add(matcher.group(1));
+ }
+ }
+ }
+
+ Assertions.assertFalse(specs.isEmpty(), "Found no specs in web-console/e2e-tests");
+ Assertions.assertEquals(
+ Set.of(),
+ Sets.difference(specs, runSpecs),
+ "Specs of web-console/e2e-tests that no *WebConsoleTest runs (with runSpec)"
+ );
+ Assertions.assertEquals(
+ Set.of(),
+ Sets.difference(runSpecs, specs),
+ "Specs that a *WebConsoleTest runs, which are not in web-console/e2e-tests"
+ );
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleTestBase.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleTestBase.java
new file mode 100644
index 000000000000..ee62e292ad17
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/console/WebConsoleTestBase.java
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.console;
+
+import org.apache.druid.guice.SleepModule;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.java.util.common.logger.Logger;
+import org.apache.druid.query.aggregation.datasketches.hll.HllSketchModule;
+import org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchModule;
+import org.apache.druid.query.aggregation.datasketches.theta.SketchModule;
+import org.apache.druid.testing.embedded.EmbeddedBroker;
+import org.apache.druid.testing.embedded.EmbeddedCoordinator;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.EmbeddedHistorical;
+import org.apache.druid.testing.embedded.EmbeddedIndexer;
+import org.apache.druid.testing.embedded.EmbeddedOverlord;
+import org.apache.druid.testing.embedded.EmbeddedRouter;
+import org.apache.druid.testing.embedded.junit5.EmbeddedClusterTestBase;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+
+/**
+ * Base class for the web console end-to-end tests: a cluster with a Router (which serves the web console), on which
+ * the tests run the Playwright specs of {@code web-console/e2e-tests}.
+ *
+ * Steps to write a test:
+ *
+ *
Write the Playwright spec in {@code web-console/e2e-tests}.
+ *
Write a {@code *WebConsoleTest} class that extends this class, in the package of the functionality the spec
+ * covers (like {@code s3} for loading data from S3). Add the resources the spec needs (S3, Kafka, etc.) in
+ * {@link #addResources}, and the extensions and properties in {@link #configureCluster}.
+ *
Write a {@code @Test} method that calls {@link #runSpec}, passing the settings of the resources to the spec
+ * through environment variables.
+ *
+ * The tests run only with the {@code web-console-tests} profile, all of them or one class:
+ *
+ * With {@code -Dweb.console.keepAlive=true}, {@link #runSpec} doesn't run the spec. It prints the command to run it
+ * and waits, keeping the cluster up, so that the spec can be run (and debugged) from {@code web-console} against it.
+ * With {@code -Dweb.console.port}, the specs run against the console on that port (such as the dev server of
+ * {@code npm start}, which proxies to the Router on port 8888) rather than the console that the Router serves.
+ */
+@Tag("web-console")
+public abstract class WebConsoleTestBase extends EmbeddedClusterTestBase
+{
+ private static final Logger log = new Logger(WebConsoleTestBase.class);
+
+ private static final long SPEC_TIMEOUT_MINUTES = 15;
+ private static final Pattern NOT_FILE_NAME_CHARS = Pattern.compile("[^A-Za-z0-9.-]");
+
+ protected final EmbeddedCoordinator coordinator = new EmbeddedCoordinator();
+ protected final EmbeddedOverlord overlord = new EmbeddedOverlord();
+ protected final EmbeddedIndexer indexer = new EmbeddedIndexer();
+ protected final EmbeddedHistorical historical = new EmbeddedHistorical();
+ protected final EmbeddedBroker broker = new EmbeddedBroker();
+ protected final EmbeddedRouter router = new EmbeddedRouter();
+
+ @Override
+ protected final EmbeddedDruidCluster createCluster()
+ {
+ final EmbeddedDruidCluster cluster = EmbeddedDruidCluster.withEmbeddedDerbyAndZookeeper();
+ // Resources start in the order they are added, and must start before the Druid servers that use them
+ addResources(cluster);
+
+ // SleepModule has the sleep() function, for queries that run long enough to be canceled
+ cluster.addExtensions(SketchModule.class, HllSketchModule.class, DoublesSketchModule.class, SleepModule.class)
+ .addCommonProperty("druid.msq.dart.enabled", "true")
+ // Keep the reports of finished Dart queries, for the console to show them (as script/druid build does)
+ .addCommonProperty("druid.msq.dart.controller.maxRetainedReportCount", "100")
+ .addCommonProperty("druid.msq.dart.controller.maxRetainedReportDuration", "PT3600S")
+ .addCommonProperty("druid.sql.planner.enableSysQueriesTable", "true")
+ // The console waits for the cluster state to change, so make it change quickly
+ .addCommonProperty("druid.manager.segments.pollDuration", "PT1S")
+ .addServer(coordinator.addProperty("druid.coordinator.period", "PT1S"))
+ .addServer(overlord)
+ // Enough task slots for an MSQ query (a controller and a worker) next to an ingestion task
+ .addServer(indexer.addProperty("druid.worker.capacity", "4").setServerMemory(1_000_000_000L))
+ .addServer(historical)
+ .addServer(broker)
+ .addServer(router);
+
+ configureCluster(cluster);
+ return cluster;
+ }
+
+ /**
+ * Adds the resources (such as an S3 container) that the specs of this test need.
+ */
+ protected void addResources(EmbeddedDruidCluster cluster)
+ {
+ }
+
+ /**
+ * Adds the extensions and properties that the specs of this test need.
+ */
+ protected void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ }
+
+ /**
+ * Runs the Playwright spec (a file name or a pattern of {@code web-console/e2e-tests}) against this cluster and
+ * fails if it fails.
+ */
+ protected void runSpec(String spec) throws Exception
+ {
+ runSpec(spec, Map.of());
+ }
+
+ /**
+ * Runs the Playwright spec (a file name or a pattern of {@code web-console/e2e-tests}) against this cluster and
+ * fails if it fails.
+ *
+ * @param env environment variables to pass to the spec, such as the settings of the resources
+ */
+ protected void runSpec(String spec, Map env) throws Exception
+ {
+ final File webConsoleDir = webConsoleDir();
+ final Map specEnv = new HashMap<>(env);
+ specEnv.put(
+ "DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT",
+ System.getProperty("web.console.port", String.valueOf(router.bindings().selfNode().getPlaintextPort()))
+ );
+ // Each spec has its own output directory, as Playwright empties it on every run
+ final String outputDir = "test-results/" + NOT_FILE_NAME_CHARS.matcher(spec).replaceAll("_");
+ final List command = List.of("npx", "playwright", "test", spec, "--output", outputDir);
+
+ if (Boolean.getBoolean("web.console.keepAlive")) {
+ keepAlive(webConsoleDir, specEnv, command);
+ return;
+ }
+
+ // The cluster is thrown away after this class, so the specs don't need to delete what they create. (A kept-alive
+ // cluster is reused for runs of the specs by hand, so they do.)
+ specEnv.put("DRUID_E2E_TEST_CLUSTER_IS_DISPOSABLE", "true");
+
+ log.info("Running spec[%s] in [%s] with env[%s].", spec, webConsoleDir, specEnv);
+ final ProcessBuilder processBuilder = new ProcessBuilder(command).directory(webConsoleDir)
+ .redirectErrorStream(true);
+ processBuilder.environment().putAll(specEnv);
+ final Process process = processBuilder.start();
+
+ // Log the output as it comes (so that a hung spec can be seen) and keep it for the failure message
+ final List output = new ArrayList<>();
+ try (BufferedReader reader = new BufferedReader(
+ new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8)
+ )) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ log.info("[playwright] %s", line);
+ output.add(line);
+ }
+ }
+
+ if (!process.waitFor(SPEC_TIMEOUT_MINUTES, TimeUnit.MINUTES)) {
+ process.destroyForcibly();
+ Assertions.fail(StringUtils.format("Spec[%s] did not finish in %d minutes", spec, SPEC_TIMEOUT_MINUTES));
+ }
+ Assertions.assertEquals(
+ 0,
+ process.exitValue(),
+ StringUtils.format(
+ "Spec[%s] failed (its screenshots and traces are in [%s]):\n%s",
+ spec,
+ new File(webConsoleDir, outputDir),
+ String.join("\n", output)
+ )
+ );
+ }
+
+ /**
+ * The {@code web-console} directory of this checkout.
+ */
+ protected static File webConsoleDir() throws IOException
+ {
+ return new File(System.getProperty("web.console.dir", "../web-console")).getCanonicalFile();
+ }
+
+ private void keepAlive(File webConsoleDir, Map specEnv, List command)
+ throws InterruptedException
+ {
+ final String envString = specEnv.entrySet()
+ .stream()
+ .map(entry -> entry.getKey() + "=" + entry.getValue())
+ .collect(Collectors.joining(" "));
+ // Printed to stderr as well as logged, to stand out from the logs of the cluster
+ System.err.println(
+ StringUtils.format(
+ "%n%n==== The cluster is up, with the console at [%s]. Run the spec with:%n"
+ + "cd %s && %s %s%n"
+ + "==== Stop this process to stop the cluster.%n%n",
+ getServerUrl(router),
+ webConsoleDir,
+ envString,
+ String.join(" ", command)
+ )
+ );
+ Thread.sleep(Long.MAX_VALUE);
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/InputFormatsWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/InputFormatsWebConsoleTest.java
new file mode 100644
index 000000000000..1fe4c5f4a9b8
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexer/InputFormatsWebConsoleTest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.indexer;
+
+import org.apache.druid.data.input.avro.AvroExtensionsModule;
+import org.apache.druid.data.input.orc.OrcExtensionsModule;
+import org.apache.druid.data.input.parquet.ParquetExtensionsModule;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ * Loads the data files of {@link ITLocalInputSourceAllInputFormatTest} (CSV, TSV, Parquet, ORC and Avro OCF) through
+ * the data loader of the web console, checking the input format it picks for each.
+ */
+public class InputFormatsWebConsoleTest extends WebConsoleTestBase
+{
+ @Override
+ protected void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addExtensions(AvroExtensionsModule.class, ParquetExtensionsModule.class, OrcExtensionsModule.class);
+ }
+
+ @Test
+ public void testInputFormats() throws Exception
+ {
+ runSpec(
+ "input-formats.spec.ts",
+ Map.of("DRUID_E2E_TEST_DATA_DIR", new File("src/test/resources/data").getCanonicalPath())
+ );
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/BatchIndexingWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/BatchIndexingWebConsoleTest.java
new file mode 100644
index 000000000000..9fe3169fbf5e
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/BatchIndexingWebConsoleTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.indexing;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Runs batch ingestion through the data loader of the web console: from local files and reindexing from Druid.
+ */
+public class BatchIndexingWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testLoadFromLocalDisk() throws Exception
+ {
+ runSpec("tutorial-batch.spec.ts");
+ }
+
+ @Test
+ public void testReindex() throws Exception
+ {
+ runSpec("reindexing.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/DatasourceActionsWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/DatasourceActionsWebConsoleTest.java
new file mode 100644
index 000000000000..4d3ad9abc823
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/DatasourceActionsWebConsoleTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.indexing;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Marks the segments of a datasource unused and used again, then deletes them with a kill task, from the Datasources
+ * view of the web console.
+ */
+public class DatasourceActionsWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testDatasourceActions() throws Exception
+ {
+ runSpec("datasource-actions.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaWebConsoleTest.java
new file mode 100644
index 000000000000..1351ecab7df9
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/indexing/KafkaWebConsoleTest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.indexing;
+
+import org.apache.druid.indexing.kafka.simulate.KafkaResource;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * Sets up a Kafka supervisor through the data loader of the web console, then suspends, resumes and terminates it
+ * from the Supervisors view.
+ */
+public class KafkaWebConsoleTest extends WebConsoleTestBase
+{
+ private static final String TOPIC = "wikipedia";
+
+ private final KafkaResource kafka = new KafkaResource();
+
+ @Override
+ protected void addResources(EmbeddedDruidCluster cluster)
+ {
+ cluster.addResource(kafka);
+ }
+
+ /**
+ * Publishes the edits of the tutorial file to the topic, one per message, in one partition (so in the order of the
+ * file), as in the Kafka tutorial.
+ */
+ @BeforeAll
+ public void publishData() throws Exception
+ {
+ final File dataFile = new File(webConsoleDir(), "../examples/quickstart/tutorial/wikiticker-2015-09-12-sampled.json.gz");
+ final List records = new ArrayList<>();
+ // The file is its own resource, to be closed even when reading the gzip header fails
+ try (InputStream fileIn = new FileInputStream(dataFile);
+ BufferedReader reader = new BufferedReader(
+ new InputStreamReader(new GZIPInputStream(fileIn), StandardCharsets.UTF_8)
+ )) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ records.add(StringUtils.toUtf8(line));
+ }
+ }
+
+ kafka.createTopicWithPartitions(TOPIC, 1);
+ kafka.publishRecordsToTopic(TOPIC, records);
+ }
+
+ @Test
+ public void testKafkaIngestion() throws Exception
+ {
+ runSpec(
+ "kafka-ingestion.spec.ts",
+ Map.of(
+ "DRUID_E2E_TEST_KAFKA_BOOTSTRAP_SERVERS", kafka.getBootstrapServerUrl(),
+ "DRUID_E2E_TEST_KAFKA_TOPIC", TOPIC
+ )
+ );
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/lookup/JdbcLookupWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/lookup/JdbcLookupWebConsoleTest.java
new file mode 100644
index 000000000000..17a3c467e463
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/lookup/JdbcLookupWebConsoleTest.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.lookup;
+
+import org.apache.druid.metadata.SQLMetadataConnector;
+import org.apache.druid.metadata.TestDerbyConnector;
+import org.apache.druid.server.lookup.namespace.NamespaceExtractionModule;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+/**
+ * Initializes the lookups and adds a JDBC lookup (on a table of the Derby metadata store, as {@link JdbcLookupTest}
+ * does) from the Lookups view of the web console, then queries it.
+ */
+public class JdbcLookupWebConsoleTest extends WebConsoleTestBase
+{
+ private static final String LOOKUP_TABLE = "web_console_lookups";
+
+ @Override
+ protected void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addExtension(NamespaceExtractionModule.class);
+ // Push the lookups to the services every second (rather than every 2 minutes), for the Broker to have them soon
+ coordinator.addProperty("druid.manager.lookups.period", "1000");
+ }
+
+ @BeforeAll
+ public void createLookupTable()
+ {
+ final SQLMetadataConnector connector = coordinator.bindings().sqlMetadataConnector();
+ connector.retryWithHandle(
+ handle -> handle.update(
+ "CREATE TABLE " + LOOKUP_TABLE + "("
+ + " created_date TIMESTAMP NOT NULL,\n"
+ + " country_code VARCHAR(10) NOT NULL,\n"
+ + " country_name VARCHAR(255) NOT NULL,\n"
+ + " PRIMARY KEY (country_code)"
+ + ")"
+ )
+ );
+ connector.retryWithHandle(
+ handle -> handle.insert(
+ "INSERT INTO " + LOOKUP_TABLE
+ + " (created_date, country_code, country_name) VALUES"
+ + " ('2025-06-01 00:00:00', 'AU', 'Australia'),"
+ + " ('2025-06-02 00:00:00', 'PR', 'Puerto Rico')"
+ )
+ );
+ }
+
+ @Test
+ public void testJdbcLookup() throws Exception
+ {
+ runSpec(
+ "jdbc-lookup.spec.ts",
+ Map.of(
+ "DRUID_E2E_TEST_LOOKUP_CONNECT_URI",
+ ((TestDerbyConnector) coordinator.bindings().sqlMetadataConnector()).getJdbcUri(),
+ "DRUID_E2E_TEST_LOOKUP_TABLE",
+ LOOKUP_TABLE
+ )
+ );
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/DartWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/DartWebConsoleTest.java
new file mode 100644
index 000000000000..49081fc525db
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/DartWebConsoleTest.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.msq;
+
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Runs and cancels Dart queries from the Query view of the web console, following them in its current Dart queries
+ * panel, as {@link EmbeddedDartReportApiTest} does through the API.
+ */
+public class DartWebConsoleTest extends WebConsoleTestBase
+{
+ @Override
+ protected void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ // Dart runs the controller of a query on the Broker and its workers on the Historicals, with a part of their heap
+ // (as in EmbeddedDartReportApiTest), which needs to be more than the 100 MB that embedded servers have by default
+ broker.setServerMemory(1_000_000_000L)
+ .addProperty("druid.msq.dart.controller.heapFraction", "0.5");
+ historical.setServerMemory(1_000_000_000L)
+ .addProperty("druid.msq.dart.worker.heapFraction", "0.5")
+ .addProperty("druid.msq.dart.worker.concurrentQueries", "1");
+ }
+
+ @Test
+ public void testDart() throws Exception
+ {
+ runSpec("dart.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MultiStageQueryWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MultiStageQueryWebConsoleTest.java
new file mode 100644
index 000000000000..dd23cc70b58e
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/MultiStageQueryWebConsoleTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.msq;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Runs an MSQ query on external data from the Query view of the web console.
+ */
+public class MultiStageQueryWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testExternQuery() throws Exception
+ {
+ runSpec("multi-stage-query.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/SqlIngestionWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/SqlIngestionWebConsoleTest.java
new file mode 100644
index 000000000000..3ffc999d8978
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/msq/SqlIngestionWebConsoleTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.msq;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Ingests with SQL (MSQ tasks) from the web console: through the SQL data loader, and with REPLACE in the Query view,
+ * from external data and from a datasource (reindexing).
+ */
+public class SqlIngestionWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testSqlIngestion() throws Exception
+ {
+ runSpec("sql-ingestion.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/SqlQueryCancelWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/SqlQueryCancelWebConsoleTest.java
new file mode 100644
index 000000000000..220815a6035a
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/query/SqlQueryCancelWebConsoleTest.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.query;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Cancels a running SQL query from the Query view of the web console.
+ */
+public class SqlQueryCancelWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testCancelQuery() throws Exception
+ {
+ runSpec("cancel-query.spec.ts");
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3WebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3WebConsoleTest.java
new file mode 100644
index 000000000000..443e795b6628
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/s3/S3WebConsoleTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.s3;
+
+import org.apache.druid.data.input.s3.S3InputSourceDruidModule;
+import org.apache.druid.java.util.common.StringUtils;
+import org.apache.druid.testing.embedded.EmbeddedDruidCluster;
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+
+import java.io.File;
+import java.util.Map;
+
+/**
+ * Runs the web console specs that load data from S3, on a cluster with an S3-compatible container (which is also its
+ * deep storage and where it keeps the task logs).
+ */
+public class S3WebConsoleTest extends WebConsoleTestBase
+{
+ private static final String DATA_FILE = "wikiticker-2015-09-12-sampled.json.gz";
+ private static final String DATA_KEY = "web-console-data/" + DATA_FILE;
+
+ private final S3StorageResource s3 = new S3StorageResource();
+
+ @Override
+ protected void addResources(EmbeddedDruidCluster cluster)
+ {
+ cluster.addResource(s3);
+ }
+
+ @Override
+ protected void configureCluster(EmbeddedDruidCluster cluster)
+ {
+ cluster.addExtension(S3InputSourceDruidModule.class);
+ }
+
+ @BeforeAll
+ public void uploadData() throws Exception
+ {
+ final File dataFile = new File(webConsoleDir(), "../examples/quickstart/tutorial/" + DATA_FILE);
+ s3.getS3Client().putObject(
+ PutObjectRequest.builder().bucket(s3.getBucket()).key(DATA_KEY).build(),
+ RequestBody.fromFile(dataFile)
+ );
+ }
+
+ @Test
+ public void testS3Ingestion() throws Exception
+ {
+ runSpec(
+ "s3-ingestion.spec.ts",
+ Map.of("DRUID_E2E_TEST_S3_URI", StringUtils.format("s3://%s/%s", s3.getBucket(), DATA_KEY))
+ );
+ }
+}
diff --git a/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/RetentionRulesWebConsoleTest.java b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/RetentionRulesWebConsoleTest.java
new file mode 100644
index 000000000000..f40dca52c307
--- /dev/null
+++ b/embedded-tests/src/test/java/org/apache/druid/testing/embedded/server/RetentionRulesWebConsoleTest.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.druid.testing.embedded.server;
+
+import org.apache.druid.testing.embedded.console.WebConsoleTestBase;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Edits the retention rules of a datasource from the Datasources view of the web console, checking that the
+ * Coordinator drops and loads its segments by them.
+ */
+public class RetentionRulesWebConsoleTest extends WebConsoleTestBase
+{
+ @Test
+ public void testRetentionRules() throws Exception
+ {
+ runSpec("retention-rules.spec.ts");
+ }
+}
diff --git a/web-console/.gitignore b/web-console/.gitignore
index 940182a3e402..d3a59a502544 100644
--- a/web-console/.gitignore
+++ b/web-console/.gitignore
@@ -19,8 +19,10 @@ tscommand-*.tmp.txt
licenses.json
.druid.pid
-# Screenshots the e2e tests write on failure
-/*.jpeg
+# What the e2e tests write (screenshots and traces of failed tests, the HTML report, the steps of E2E_SHOW_STEPS=true)
+/test-results/
+/playwright-report/
+/e2e-tests/steps/
# Generated by script/build-grammars.mjs
src/editor-languages/*.parser.ts
diff --git a/web-console/AGENTS.md b/web-console/AGENTS.md
index ef2cf1821f34..99e5b26bf810 100644
--- a/web-console/AGENTS.md
+++ b/web-console/AGENTS.md
@@ -40,8 +40,9 @@ The Apache Druid web console: a React + TypeScript single page app built with Bl
bundle first.
- `npm run autofix` fixes ESLint, stylelint and Prettier issues.
- **End-to-end tests** (`e2e-tests/`, Playwright) need a running Druid: `script/druid build && script/druid start`,
- then `npm run test-e2e`. See the README for running one test or with a visible browser, and below for running them
- against your own changes.
+ then `npm run test-e2e`. CI runs them on embedded clusters instead (`*WebConsoleTest` in `embedded-tests`, profile
+ `web-console-tests`). See `e2e-tests/README.md` for both, for running one test or with a visible browser, and below
+ for running them against your own changes.
## Developing against a running Druid
diff --git a/web-console/README.md b/web-console/README.md
index baa34e74bc61..2a8f64363e42 100644
--- a/web-console/README.md
+++ b/web-console/README.md
@@ -109,40 +109,24 @@ From the web-console directory:
4. Stop druid cluster: `script/druid stop`
If you already have a druid cluster running on the standard ports, the steps to build/start/stop a druid cluster can
-be skipped.
+be skipped. CI runs the tests on embedded clusters instead, see [e2e-tests/README.md](e2e-tests/README.md).
-#### Screenshots for debugging
+#### Running one test, watching or debugging
-`e2e-tests/util/debug.ts:saveScreenshotIfError()` is used to save a screenshot of the web console
-when the test fails. For example, if `e2e-tests/tutorial-batch.spec.ts` fails, it will create
-`load-data-from-local-disk-error-screenshot.png`.
+- One spec: `npm run test-e2e -- cancel-query`
+- With a visible browser: `npm run test-e2e -- --headed`, or step through with `--debug` or `--ui`
+- Against a console on another port (like the dev server from `npm start` on `18081`):
+ `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT=18081 npm run test-e2e`
-#### Disabling headless mode
-
-Disabling headless mode while running the tests can be helpful. This can be done via the `DRUID_E2E_TEST_HEADLESS`
-environment variable, which defaults to `true`.
-
-Like so: `DRUID_E2E_TEST_HEADLESS=false npm run test-e2e`
-
-#### Running against alternate web console
-
-The environment variable `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT` can be used to target a web console running on a
-non-default port (i.e., not port `8888`). For example, this environment variable can be used to target the
-development mode of the web console (started via `npm start`), which runs on port `18081`.
-
-Like so: `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT=18081 npm run test-e2e`
-
-#### Running and debugging a single e2e test using Jest and Playwright
-
-- Run - `jest --config jest.e2e.config.js e2e-tests/tutorial-batch.spec.ts`
-- Debug - `PWDEBUG=console jest --config jest.e2e.config.js e2e-tests/tutorial-batch.spec.ts`
+A failed test leaves a screenshot and a trace in `test-results/` (open the trace with `npx playwright show-trace`).
+See `e2e-tests/README.md` for what the tests cover and how they are structured.
## Description of the directory structure
As part of this directory:
- `assets/` - The images (and other assets) used within the console
-- `e2e-tests/` - End-to-end tests for the console
+- `e2e-tests/` - End-to-end tests for the console (see `e2e-tests/README.md`)
- `public/` - The compiled destination for the files powering this console
- `script/` - Some helper bash scripts for running this console
- `src/` - This directory constitutes all the source code for this console
diff --git a/web-console/e2e-tests/README.md b/web-console/e2e-tests/README.md
new file mode 100644
index 000000000000..e6fbf5c5e514
--- /dev/null
+++ b/web-console/e2e-tests/README.md
@@ -0,0 +1,192 @@
+
+
+# End-to-end tests
+
+These tests drive the web console in a real (headless) Chromium against a real Druid cluster. They catch what the unit
+tests can't: the console and Druid's APIs disagreeing, a flow across several views breaking, or a request that only
+fails against a live cluster.
+
+They use the Playwright test runner (`@playwright/test`, configured in `../playwright.config.ts`). The Chromium it
+drives is downloaded by the `playwright-chromium` package's install script, so keep the two packages on the same
+version. They run in CI on embedded clusters (see below), as part of `.github/scripts/web-checks.sh`.
+
+## Running them
+
+The tests need a Druid cluster with its Router on :8888 (every request goes through the console's service), running
+on the same machine (its tasks read the `examples/quickstart/tutorial` data files of this checkout):
+
+```bash
+script/druid build # once, builds a distribution with the extensions the tests need
+script/druid start
+npm run test-e2e # all the tests, one at a time (they share the cluster)
+script/druid stop
+```
+
+Before the tests start, a global setup (`util/global-setup.ts`) waits for the console's SQL endpoint to answer.
+
+- **One file**: `npm run test-e2e -- cancel-query` (any `playwright test` arguments go after `--`)
+- **See the browser**: `npm run test-e2e -- --headed`, step through a test with `--debug`, or pick and watch tests in
+ the UI mode with `--ui`
+- **Test a dev server instead of the bundled console**: `DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT=18081 npm run test-e2e`.
+ Without it the tests run against the console that the router on :8888 serves, which was bundled when Druid was
+ built, not your working tree.
+- **On failure** a test leaves a screenshot, a trace (`npx playwright show-trace `: every action with the DOM,
+ console and network at that point) and an `error-context.md` (the page as an accessibility tree) in `test-results/`
+ (git ignored). Every HTTP response with a status of 400 or more is also logged with its body
+ (`util/fixtures.ts`).
+- **See the steps a test takes**: `E2E_SHOW_STEPS=true npm run test-e2e`. Each test saves a screenshot at each of
+ its steps (each step of the data loader, each view it reads, a query's results, a dialog it fills in, and the
+ page as it ends) to `steps/-001.png`, `-002.png`, ... (git ignored, replaced by the next run of the spec).
+ The quickest way to see what a test does, especially one someone else wrote. It works on the embedded clusters
+ too: `E2E_SHOW_STEPS=true mvn -pl embedded-tests verify -Pweb-console-tests ...`.
+
+### On an embedded cluster (as in CI)
+
+Each spec is run by a `*WebConsoleTest` class in `embedded-tests`, in the package of the functionality it covers. The
+class starts an embedded cluster (in one JVM, with an in-memory metadata store, its Router on :8888) with the resources
+the spec needs, and runs the spec on it with `npx playwright test `:
+
+| Spec | Run by (in `org.apache.druid.testing.embedded`) |
+|-----------------------------------------------|-------------------------------------------------|
+| `tutorial-batch.spec.ts`, `reindexing.spec.ts` | `indexing.BatchIndexingWebConsoleTest` |
+| `auto-compaction.spec.ts` | `compact.AutoCompactionWebConsoleTest` |
+| `multi-stage-query.spec.ts` | `msq.MultiStageQueryWebConsoleTest` |
+| `cancel-query.spec.ts` | `query.SqlQueryCancelWebConsoleTest` |
+| `s3-ingestion.spec.ts` | `s3.S3WebConsoleTest` (with an S3 container) |
+| `input-formats.spec.ts` | `indexer.InputFormatsWebConsoleTest` |
+| `kafka-ingestion.spec.ts` | `indexing.KafkaWebConsoleTest` (with Kafka) |
+| `sql-ingestion.spec.ts` | `msq.SqlIngestionWebConsoleTest` |
+| `datasource-actions.spec.ts` | `indexing.DatasourceActionsWebConsoleTest` |
+| `retention-rules.spec.ts` | `server.RetentionRulesWebConsoleTest` |
+| `jdbc-lookup.spec.ts` | `lookup.JdbcLookupWebConsoleTest` |
+| `dart.spec.ts` | `msq.DartWebConsoleTest` |
+| `input-formats.spec.ts` | Loads the same 10 rows (over 2 days) of Wikipedia edits from local CSV, TSV, Parquet, ORC and Avro OCF files through the data loader, one test per format. Checks that the **Parse data** step picks the right input format by itself and parses the columns, and that the datasource ends up as 2 segments with 10 rows. Skipped without `DRUID_E2E_TEST_DATA_DIR` (the data files of embedded-tests): it runs on an embedded cluster with the Avro, Parquet and ORC extensions, from `InputFormatsWebConsoleTest`. |
+| `kafka-ingestion.spec.ts` | Follows the [Kafka tutorial](https://druid.apache.org/docs/latest/tutorials/tutorial-kafka): sets up a supervisor through the data loader's **Apache Kafka** connector on a topic with the 39,244 edits of the tutorial file (reading it from the earliest offset), and waits for the rows to be queryable. Then, from the **Supervisors** view, suspends the supervisor, resumes it and terminates it, checking its state each time (through SQL on `sys.supervisors` and in the view). Skipped without `DRUID_E2E_TEST_KAFKA_BOOTSTRAP_SERVERS` and `DRUID_E2E_TEST_KAFKA_TOPIC`: it runs on an embedded cluster with a Kafka container, from `KafkaWebConsoleTest`. |
+| `sql-ingestion.spec.ts` | SQL-based ingestion (MSQ tasks). Loads the tutorial file through the **SQL data loader** ("Load data" > "Batch - SQL": input source, parse, schema with a new datasource as the destination, ingestion progress) and waits for its 39,244 rows to be queryable. Then, in the **Query** view, ingests it with `REPLACE ... FROM TABLE(EXTERN(...))` and reindexes the 11,549 rows of `#en.wikipedia` into another datasource with `REPLACE ... FROM` the first, checking what the view says it inserted and the rows queryable each time. |
+| `datasource-actions.spec.ts` | Loads `wikipedia-index.json` through the API, then from the **Datasources** view: marks all its segments unused (the datasource then shows, as "Unused", only with "Show unused"), marks them used again (the data is back, fully available), marks them unused again and deletes them with a kill task (ticking the dialog's checks of what it does), after which the kill task succeeds and the datasource is gone even with "Show unused". |
+| `retention-rules.spec.ts` | Loads `wikipedia-index.json` through the API, then edits its retention rules from the **Datasources** view (deleting the rules there, adding new ones and picking their type, then giving the reason the dialog asks for): a `loadForever` rule (as "New rule" makes it, 2 replicas), which the Retention column shows and with which the data stays loaded, then a `dropForever` rule in its place, with which the Coordinator drops the data (marking it unused). Checks the rules through the Coordinator API each time. |
+| `jdbc-lookup.spec.ts` | From the **Lookups** view, initializes the lookups (if the cluster has none) and adds a JDBC lookup (cachedNamespace, jdbc) on a table of country codes and names, checks that the view lists it, and waits for `LOOKUP('PR', ...)` and `LOOKUP('AU', ...)` to return the names. Skipped without `DRUID_E2E_TEST_LOOKUP_CONNECT_URI` and `DRUID_E2E_TEST_LOOKUP_TABLE`: it runs on an embedded cluster, with the table in its Derby metadata store, from `JdbcLookupWebConsoleTest`. |
+| `dart.spec.ts` | Dart (the multi-stage query engine on the Broker and Historicals). Loads `wikipedia-index.json` through the API, picks **SQL (Dart)** in the **Query** view's engine menu and runs a top channels query, checking the results, that the query was sent with the `msq-dart` engine and that it is in `sys.queries` as SUCCESS. Then, in the **Current Dart queries** panel, finds it as succeeded and opens its details (the stages of its report). A second test starts a long Dart query (sleeping for each row) and cancels it from the panel, which then shows it as canceled, as `sys.queries` does. |
+
+They extend `console.WebConsoleTestBase`. There's no distribution to build and nothing left behind, as every test class
+gets a new cluster. They need this checkout's Druid modules installed (`mvn install -DskipTests`), with the
+`web-console` one built with the console (not with `-Dweb.console.skip=true`, which leaves the Router with no console
+to serve):
+
+```bash
+# from the root of the checkout
+mvn -pl web-console install -DskipTests # after changing the console (not the tests), for the Router to serve it
+mvn -pl embedded-tests verify -Pweb-console-tests # all of them
+mvn -pl embedded-tests verify -Pweb-console-tests -Dit.test=BatchIndexingWebConsoleTest # one class
+```
+
+The Druid logs and the Playwright output are in `embedded-tests/target/failsafe-reports/*-output.txt`, and the
+artifacts of a failed spec in `test-results//`. The tests are tagged `web-console`, so they run only with the
+`web-console-tests` profile.
+
+- **Keep the cluster up to work on a spec**: add `-Dweb.console.keepAlive=true -Dmaven.test.redirectTestOutputToFile=false`.
+ Rather than running the specs, it prints the command to run one against the cluster (with `--ui` or `--debug` added
+ as you like) and waits until it's stopped.
+- **Test a dev server instead of the bundled console**: add `-Dweb.console.port=18081`, with `npm start` running (it
+ proxies to :8888).
+- **A new spec** gets a `@Test` in the `*WebConsoleTest` class of its functionality (or a new class, next to the
+ embedded tests of that functionality). `WebConsoleSpecsTest`, a unit test, fails if a spec isn't run by any. One that needs more than Druid (S3, Kafka...) adds the resource in
+ `addResources` and passes its settings (endpoint, bucket...) to the spec as environment variables of `runSpec`. See
+ `S3WebConsoleTest` and `s3-ingestion.spec.ts`. The resources run in Docker containers (with Testcontainers), so
+ Docker has to be running.
+
+A test deletes the datasources it created when it passes: it stops their tasks, removes their compaction config and
+permanently deletes their segments (with a `kill` task). When it fails, they are kept to look into (the log says which)
+and have to be deleted by hand, from the Datasources view. On an embedded cluster nothing is deleted, as the cluster is
+thrown away after the test class (`DRUID_E2E_TEST_CLUSTER_IS_DISPOSABLE=true`).
+
+## The tests
+
+Each test has a 5 minute timeout. Polling for cluster state (a task finishing, segments loading) retries every second
+for up to 2 minutes.
+
+| Spec | What it does |
+|------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `tutorial-batch.spec.ts` | Follows the [batch loading tutorial](https://druid.apache.org/docs/latest/tutorials/tutorial-batch) through the classic **data loader**: connects to `wikiticker-2015-09-12-sampled.json.gz` on local disk, checks the first and last preview lines, sets the timestamp to `timestamp_parse("time") + 1` (so the `__time` check below proves the expression was used), turns rollup off, sets DAY granularity and submits. It then waits (through SQL on `sys.tasks` and `sys.segments`) for the task to succeed and for the datasource to be fully available as 1 segment with 39,244 rows, checks that the **Tasks** and **Datasources** views show the same, and checks the first row of `SELECT *` in the **Query** view. The datasource name contains a string of special and non-Latin characters to test quoting and escaping. |
+| `reindexing.spec.ts` | Loads `wikipedia-index.json` through the API (1 segment), then reindexes it through the data loader's **Reindex from Druid** connector into range partitions on `channel` with 10,000 target rows per segment. Checks the preview rows, that the task succeeds, and that the datasource ends up as 4 segments with the same 39,244 rows. |
+| `auto-compaction.spec.ts` | Follows the [compaction tutorial](https://druid.apache.org/docs/latest/tutorials/tutorial-compaction): loads 2 hours of `compaction-init-index.json` through the API (3 segments, 1,412 rows), sets a compaction config (`skipOffsetFromLatest: PT0S`, hashed partitions) from the **Datasources** view, reopens the dialog until it reads back the same config, then forces compaction runs (from the Alt-click "more" menu) until there are 2 segments. |
+| `multi-stage-query.spec.ts` | Runs an MSQ `SELECT` over `EXTERN(...)` on the tutorial file in the **Query** view, clicking "Run it anyway" if the cluster warns it lacks task slots, and checks the top 2 of the 10 channels by count. |
+| `cancel-query.spec.ts` | Runs `SELECT sleep(40)` in the **Query** view, waits for its `POST` to `druid/v2`, clicks "Cancel query" and checks that the `DELETE` it sends is answered with `202 Accepted`. |
+| `s3-ingestion.spec.ts` | Loads the tutorial file from S3 through the data loader's **Amazon S3** connector (its URI from `DRUID_E2E_TEST_S3_URI`) and waits for the task to succeed and for 1 segment with 39,244 rows. Skipped without `DRUID_E2E_TEST_S3_URI`: it runs on an embedded cluster with an S3 container, from `S3WebConsoleTest` (which also makes S3 the deep storage and task log storage). |
+
+## Layout
+
+```
+e2e-tests/
+ *.spec.ts the tests, one describe per file
+ component/ page objects: one per view, wrapping its selectors
+ datasources/ Datasources view (reads the table, with unused datasources or not, runs and confirms a
+ datasource's actions, edits retention rules, edits and triggers compaction)
+ lookups/ Lookups view (initializes the lookups, adds a JDBC lookup, reads the table)
+ ingestion/ Tasks view (reads the table)
+ load-data/ the classic data loader: loadData goes through each step with a DataLoaderConfig
+ (data-connector.ts: local disk, S3, Kafka, reindex from Druid; partitions-spec.ts: hashed,
+ range); sql-data-loader.ts: the SQL data loader (loadDataWithSql)
+ supervisors/ Supervisors view (reads the table, runs and confirms a supervisor's actions)
+ workbench/ Query view: run or start a query, or an ingestion query, with the engine picked (accepting the
+ task slot warning), cancel a query; dart-panel.ts: the current Dart queries panel
+ util/
+ fixtures.ts the `test` and `expect` to import in specs (`test` logs failed responses, shows the last
+ step, and has the newDatasourceName fixture)
+ global-setup.ts waits for the console's SQL endpoint before the tests start
+ druid.ts the tutorial data dir and ingestion specs, runTask (submits a task and waits for it),
+ deleteDatasource
+ sql.ts querySql, and the cluster state the tests wait for: task statuses, a datasource's segments
+ and queryable rows, a supervisor's state, a Dart query's state
+ playwright.ts openView, and helpers that find inputs and buttons by their label or text
+ table.ts extractTable / extractTableRecords: read a table's rows as text (by position / by header)
+ steps.ts showStep: a screenshot of a step of the test, with E2E_SHOW_STEPS=true
+ steps/ the screenshots of showStep (git ignored)
+```
+
+### Writing a test
+
+- Import `test` and `expect` from `util/fixtures.ts` (not from `@playwright/test`) and take the `page` fixture:
+ `test('...', async ({ page }) => { ... })`. Each test gets a fresh browser context, so no local storage carries
+ over.
+- Put selectors in a page object under `component/` rather than in the spec (and keep data, like what to set in a
+ form, as plain objects, with a union type when it comes in kinds), and use locators (`page.locator`,
+ `getByRole`), not `page.$` / `waitForSelector` / XPath. The helpers in `util/playwright.ts` find form fields by
+ their visible label (`setLabeledInput(page, 'Datasource name', ...)`; the labels are not linked to their inputs,
+ so `getByLabel` does not work) and buttons by their exact text, so renaming a label or a button in the console
+ breaks the tests. Scope a locator (to a dialog, `.next-bar`, ...) when the same text appears twice.
+- Open a view with `openView(page, 'datasources', { datasource: ... })`. Filtering to the test's own rows keeps the
+ test working on a cluster with more datasources or tasks than fit on one page.
+- The query editor is CodeMirror; type into it with `setQueryInput`, which fills its `.cm-content`.
+- Wait for the cluster state (a task finishing, segments loading) through SQL rather than by reading a view:
+ `await expect.poll(() => getDatasourceSegments(request, name), CLUSTER_STATE_POLL).toEqual({ ... })`, with the
+ `request` fixture. Check a view only when the view is what you are testing, and once the cluster is in the state
+ it should show.
+- Read the console's tables with `extractTableRecords`, which keys each row by its column headers, so a column
+ being added, hidden or moved doesn't break the test.
+- Retry UI checks that depend on data loading with `await expect(async () => { ... }).toPass()`. Make each attempt
+ fail fast (like `WorkbenchOverview.runQuery` throwing on a query error) rather than wait out a timeout.
+- Set up data that the test is not about through the API, not the UI: `readTutorialIngestionSpec`, change what you
+ need (the datasource name at least) and `runTask(request, ingestionSpec)`.
+- Call `showStep(page, 'what the page shows')` at the points worth seeing, once the page shows them (in a page
+ object, so every test using it gets the step). With `E2E_SHOW_STEPS=true` it saves a screenshot, otherwise it does
+ nothing.
+- Name a datasource you create with the `newDatasourceName` fixture: `newDatasourceName('my-test')` makes a unique
+ name (the prefix and the time) and deletes the datasource after the test passes.
diff --git a/web-console/e2e-tests/auto-compaction.spec.ts b/web-console/e2e-tests/auto-compaction.spec.ts
index 6659ab50c574..85b4c64f4c7b 100644
--- a/web-console/e2e-tests/auto-compaction.spec.ts
+++ b/web-console/e2e-tests/auto-compaction.spec.ts
@@ -16,144 +16,71 @@
* limitations under the License.
*/
-import path from 'path';
-import type * as playwright from 'playwright-chromium';
+import type { APIRequestContext, Page } from '@playwright/test';
-import { CompactionConfig } from './component/datasources/compaction';
-import type { Datasource } from './component/datasources/datasource';
+import type { CompactionConfig } from './component/datasources/overview';
import { DatasourcesOverview } from './component/datasources/overview';
-import { HashedPartitionsSpec } from './component/load-data/config/partition';
-import { saveScreenshotIfError } from './util/debug';
-import {
- DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
- runIndexTask,
- UNIFIED_CONSOLE_URL,
-} from './util/druid';
-import { createBrowser, createPage } from './util/playwright';
-import { retryIfJestAssertionError } from './util/retry';
-import { waitTillWebConsoleReady } from './util/setup';
-
-jest.setTimeout(5 * 60 * 1000);
+import { readTutorialIngestionSpec, runTask } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments } from './util/sql';
// The workflow in these tests is based on the compaction tutorial:
// https://druid.apache.org/docs/latest/tutorials/tutorial-compaction.html
-describe('Auto-compaction', () => {
- let browser: playwright.Browser;
- let page: playwright.Page;
-
- beforeAll(async () => {
- await waitTillWebConsoleReady();
- browser = await createBrowser();
- });
-
- beforeEach(async () => {
- page = await createPage(browser);
- });
-
- afterAll(async () => {
- await browser.close();
- });
-
- it('Compacts segments from dynamic to hash partitions', async () => {
- const testName = 'autocompaction-dynamic-to-hash';
- const datasourceName = testName + new Date().toISOString();
- loadInitialData(datasourceName);
-
- await saveScreenshotIfError(testName, page, async () => {
- const uncompactedNumSegment = 3;
- const numRow = 1412;
- await validateDatasourceStatus(page, datasourceName, uncompactedNumSegment, numRow);
-
- const compactionConfig = new CompactionConfig({
- skipOffsetFromLatest: 'PT0S',
- partitionsSpec: new HashedPartitionsSpec({
- numShards: null,
- }),
- });
- await configureCompaction(page, datasourceName, compactionConfig);
-
- // Depending on the number of configured tasks slots, autocompaction may
- // need several iterations if several time chunks need compaction
- let currNumSegment = uncompactedNumSegment;
- await retryIfJestAssertionError(async () => {
- await triggerCompaction(page);
- currNumSegment = await waitForCompaction(page, datasourceName, currNumSegment);
-
- const compactedNumSegment = 2;
- expect(currNumSegment).toBe(compactedNumSegment);
- });
- });
+test.describe('Auto-compaction', () => {
+ test('Compacts segments from dynamic to hash partitions', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('autocompaction-dynamic-to-hash');
+ await loadInitialData(request, datasourceName);
+
+ const numRows = 1412;
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 3, numAvailableSegments: 3, numRows });
+
+ const compactionConfig: CompactionConfig = {
+ skipOffsetFromLatest: 'PT0S',
+ partitionsSpec: { type: 'hashed', numShards: null },
+ };
+ await configureCompaction(page, datasourceName, compactionConfig);
+
+ // Depending on the number of configured tasks slots, autocompaction may
+ // need several iterations if several time chunks need compaction
+ const datasourcesOverview = new DatasourcesOverview(page);
+ await expect(async () => {
+ await datasourcesOverview.triggerCompaction();
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), {
+ ...CLUSTER_STATE_POLL,
+ timeout: 60 * 1000,
+ })
+ .toEqual({ numSegments: 2, numAvailableSegments: 2, numRows });
+ }).toPass({ timeout: 4 * 60 * 1000 });
});
});
-function loadInitialData(datasourceName: string) {
- const ingestionSpec = path.join(
- DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
- 'compaction-init-index.json',
- );
- const setDatasourceName = `s/compaction-tutorial/${datasourceName}/`;
- const setIntervals = 's|2015-09-12/2015-09-13|2015-09-12/2015-09-12T02:00|'; // shorten to reduce test duration
- const sedCommands = [setDatasourceName, setIntervals];
- runIndexTask(ingestionSpec, sedCommands);
-}
-
-async function validateDatasourceStatus(
- page: playwright.Page,
- datasourceName: string,
- expectedNumSegment: number,
- expectedNumRow: number,
-) {
- await retryIfJestAssertionError(async () => {
- const datasource = await getDatasource(page, datasourceName);
- expect(datasource.availability).toMatch(`Fully available (${expectedNumSegment} segments)`);
- expect(datasource.totalRows).toBe(expectedNumRow);
- });
-}
-
-async function getDatasource(page: playwright.Page, datasourceName: string): Promise {
- const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
- const datasources = await datasourcesOverview.getDatasources();
- const datasource = datasources.find(t => t.name === datasourceName);
- expect(datasource).toBeDefined();
- return datasource!;
+async function loadInitialData(request: APIRequestContext, datasourceName: string) {
+ const ingestionSpec = readTutorialIngestionSpec('compaction-init-index.json');
+ const { dataSchema } = ingestionSpec.spec;
+ dataSchema.dataSource = datasourceName;
+ dataSchema.granularitySpec!.intervals = ['2015-09-12/2015-09-12T02:00']; // 2 hours rather than the day, to be faster
+ await runTask(request, ingestionSpec);
}
async function configureCompaction(
- page: playwright.Page,
+ page: Page,
datasourceName: string,
compactionConfig: CompactionConfig,
) {
- const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+ const datasourcesOverview = new DatasourcesOverview(page);
await datasourcesOverview.setCompactionConfiguration(datasourceName, compactionConfig);
// Saving the compaction config is not instantaneous
- await retryIfJestAssertionError(async () => {
+ await expect(async () => {
const savedCompactionConfig =
await datasourcesOverview.getCompactionConfiguration(datasourceName);
expect(savedCompactionConfig).toEqual(compactionConfig);
- });
-}
-
-async function triggerCompaction(page: playwright.Page) {
- const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
- await datasourcesOverview.triggerCompaction();
-}
-
-async function waitForCompaction(
- page: playwright.Page,
- datasourceName: string,
- prevNumSegment: number,
-): Promise {
- await retryIfJestAssertionError(async () => {
- const currNumSegment = await getNumSegment(page, datasourceName);
- expect(currNumSegment).toBeLessThan(prevNumSegment);
- });
-
- return getNumSegment(page, datasourceName);
-}
-
-async function getNumSegment(page: playwright.Page, datasourceName: string): Promise {
- const datasource = await getDatasource(page, datasourceName);
- const currNumSegmentString = /(\d+)/.exec(datasource.availability)![0];
- return Number(currNumSegmentString);
+ }).toPass();
}
diff --git a/web-console/e2e-tests/cancel-query.spec.ts b/web-console/e2e-tests/cancel-query.spec.ts
index e17ab36fc79d..c7ecfa0ed55e 100644
--- a/web-console/e2e-tests/cancel-query.spec.ts
+++ b/web-console/e2e-tests/cancel-query.spec.ts
@@ -16,46 +16,13 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
-
-import { QueryOverview } from './component/query/overview';
-import { saveScreenshotIfError } from './util/debug';
-import { UNIFIED_CONSOLE_URL } from './util/druid';
-import { createBrowser, createPage } from './util/playwright';
-import { waitTillWebConsoleReady } from './util/setup';
-
-jest.setTimeout(5 * 60 * 1000);
-
-describe('Cancel query', () => {
- let browser: playwright.Browser;
- let page: playwright.Page;
-
- beforeAll(async () => {
- await waitTillWebConsoleReady();
- browser = await createBrowser();
- });
-
- beforeEach(async () => {
- page = await createPage(browser);
- });
-
- afterAll(async () => {
- await browser.close();
- });
-
- it('delete accepted', async () => {
- const testName = 'cancel-query';
- await saveScreenshotIfError(testName, page, async () => {
- await validateCancelQuery(page);
- });
+import { WorkbenchOverview } from './component/workbench/overview';
+import { expect, test } from './util/fixtures';
+
+test.describe('Cancel query', () => {
+ test('delete accepted', async ({ page }) => {
+ const workbench = new WorkbenchOverview(page);
+ const status = await workbench.cancelQuery('SELECT sleep(40)');
+ expect(status).toBe(202);
});
});
-
-async function validateCancelQuery(page: playwright.Page) {
- const queryOverview = new QueryOverview(page, UNIFIED_CONSOLE_URL);
- const query = 'SELECT sleep(40)';
- const results = await queryOverview.cancelQuery(query);
- expect(results).toBeDefined();
- expect(results).toBeGreaterThan(0);
- expect(results).toStrictEqual(202);
-}
diff --git a/web-console/e2e-tests/component/datasources/compaction.ts b/web-console/e2e-tests/component/datasources/compaction.ts
deleted file mode 100644
index 175050406f6f..000000000000
--- a/web-console/e2e-tests/component/datasources/compaction.ts
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { PartitionsSpec } from '../load-data/config/partition';
-
-/**
- * Datasource compaction configuration
- */
-export class CompactionConfig {
- constructor(props: CompactionConfigProps) {
- Object.assign(this, props);
- }
-}
-
-interface CompactionConfigProps {
- readonly skipOffsetFromLatest: string;
- readonly partitionsSpec: PartitionsSpec;
-}
-
-export interface CompactionConfig extends CompactionConfigProps {}
diff --git a/web-console/e2e-tests/component/datasources/datasource.ts b/web-console/e2e-tests/component/datasources/datasource.ts
deleted file mode 100644
index a6cdc586d023..000000000000
--- a/web-console/e2e-tests/component/datasources/datasource.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Represents datasource row in datasource overview table.
- */
-export class Datasource {
- constructor(props: DatasourceProps) {
- Object.assign(this, props);
- }
-}
-
-interface DatasourceProps {
- readonly name: string;
- readonly availability: string;
- readonly totalRows: number;
-}
-
-export interface Datasource extends DatasourceProps {}
diff --git a/web-console/e2e-tests/component/datasources/overview.ts b/web-console/e2e-tests/component/datasources/overview.ts
index 94bf3bab7c86..461bd49524d6 100644
--- a/web-console/e2e-tests/component/datasources/overview.ts
+++ b/web-console/e2e-tests/component/datasources/overview.ts
@@ -16,133 +16,207 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
+import type { Page } from '@playwright/test';
+
+import {
+ clickButton,
+ clickMenuItem,
+ getLabeledInput,
+ openView,
+ setLabeledInput,
+} from '../../util/playwright';
+import { showStep } from '../../util/steps';
+import { extractTableRecords } from '../../util/table';
+import type { PartitionsSpec } from '../load-data/partitions-spec';
+import { applyPartitionsSpec, readPartitionsSpec } from '../load-data/partitions-spec';
-import { clickButton, getLabeledInput, setLabeledInput } from '../../util/playwright';
-import { extractTable } from '../../util/table';
-import { readPartitionSpec } from '../load-data/config/partition';
-
-import { CompactionConfig } from './compaction';
-import { Datasource } from './datasource';
+/**
+ * A row of the Datasources view.
+ */
+export interface Datasource {
+ readonly name: string;
+ readonly availability: string;
+ readonly totalRows: number;
+ /** The retention rules (like "dropForever()"), or the cluster default ones ("Cluster default: ...") */
+ readonly retention: string;
+}
/**
- * Datasource overview table column identifiers.
+ * What the tests set in the compaction config dialog.
*/
-enum DatasourceColumn {
- NAME = 0,
- AVAILABILITY,
- SEGMENT_LOAD_DROP,
- TOTAL_DATA_SIZE,
- RUNNING_TASKS,
- SEGMENT_ROWS,
- // SEGMENT_SIZE, (Hidden by default)
- // SEGMENT_GRANULARITY, (Hidden by default)
- TOTAL_ROWS,
- AVG_ROW_SIZE,
- REPLICATED_SIZE,
- COMPACTION,
- PERCENT_COMPACTED,
- LEFT_TO_BE_COMPACTED,
- RETENTION,
- ACTIONS,
+export interface CompactionConfig {
+ readonly skipOffsetFromLatest: string;
+ readonly partitionsSpec: PartitionsSpec;
}
const SKIP_OFFSET_FROM_LATEST = 'Skip offset from latest';
+/**
+ * An action of a datasource's menu that asks for confirmation, with the text of its confirm button. The menu of a
+ * datasource with used segments has the "unused" actions, the menu of one shown only as unused the others.
+ */
+const CONFIRMED_ACTIONS = {
+ 'Mark as unused all segments': 'Mark as unused all segments',
+ 'Mark as used all segments': 'Mark as used all segments',
+ 'Delete segments (issue kill task)': 'Permanently delete unused segments',
+} as const;
+
+export type DatasourceAction = keyof typeof CONFIRMED_ACTIONS;
+
/**
* Represents datasource overview tab.
*/
export class DatasourcesOverview {
- private readonly page: playwright.Page;
- private readonly baseUrl: string;
+ private readonly page: Page;
- constructor(page: playwright.Page, unifiedConsoleUrl: string) {
+ constructor(page: Page) {
this.page = page;
- this.baseUrl = unifiedConsoleUrl + '#datasources';
}
- async getDatasources(): Promise {
- await this.page.goto(this.baseUrl);
- await this.page.reload({ waitUntil: 'networkidle' });
+ private readonly table = () => this.page.locator('.datasources-view .console-table');
- const data = await extractTable(this.page, 'div div.ct-tr-group', 'div.ct-td');
+ /**
+ * The datasource named `datasourceName` (none, if it doesn't exist yet), as the view shows it.
+ * @param showUnused whether to show the datasource also when all of its segments are unused (with "Show unused")
+ */
+ async getDatasources(
+ datasourceName: string,
+ { showUnused = false }: { showUnused?: boolean } = {},
+ ): Promise {
+ await this.open(datasourceName, showUnused);
- return data.map(
- row =>
- new Datasource({
- name: row[DatasourceColumn.NAME],
- availability: row[DatasourceColumn.AVAILABILITY],
- totalRows: DatasourcesOverview.parseNumber(row[DatasourceColumn.TOTAL_ROWS]),
- }),
+ const records = await extractTableRecords(this.table());
+ await showStep(
+ this.page,
+ `Datasources view, filtered on ${datasourceName}${showUnused ? ', with unused' : ''}`,
);
- }
- private static parseNumber(text: string): number {
- return Number(text.replace(/,/g, ''));
+ return records.map(record => ({
+ name: record['Datasource name'],
+ availability: record['Availability'],
+ totalRows: Number(record['Total rows'].replace(/,/g, '')),
+ retention: record['Retention'],
+ }));
}
async setCompactionConfiguration(
datasourceName: string,
compactionConfig: CompactionConfig,
): Promise {
- await this.openCompactionConfigurationDialog(datasourceName);
+ const dialog = await this.openCompactionConfigurationDialog(datasourceName);
- await setLabeledInput(
- this.page,
- SKIP_OFFSET_FROM_LATEST,
- compactionConfig.skipOffsetFromLatest,
- );
- await compactionConfig.partitionsSpec.apply(this.page);
+ await setLabeledInput(dialog, SKIP_OFFSET_FROM_LATEST, compactionConfig.skipOffsetFromLatest);
+ await applyPartitionsSpec(this.page, compactionConfig.partitionsSpec);
+ await showStep(this.page, 'Compaction config dialog, filled in');
- await clickButton(this.page, 'Submit');
+ await clickButton(dialog, 'Submit');
}
- private async openCompactionConfigurationDialog(datasourceName: string): Promise {
- await this.openEditActions(datasourceName);
- await this.clickMenuItem('Edit compaction configuration');
- await this.page.waitForSelector('div.compaction-config-dialog');
- }
+ async getCompactionConfiguration(datasourceName: string): Promise {
+ const dialog = await this.openCompactionConfigurationDialog(datasourceName);
- private async clickMenuItem(text: string): Promise {
- const menuItemSelector = `//a[*[contains(text(),"${text}")]]`;
- await this.page.click(menuItemSelector);
+ const skipOffsetFromLatest = await getLabeledInput(dialog, SKIP_OFFSET_FROM_LATEST);
+ const partitionsSpec = await readPartitionsSpec(this.page);
+ await showStep(this.page, 'Compaction config dialog, as saved');
+
+ await clickButton(dialog.locator('.bp6-dialog-footer'), 'Close');
+ return { skipOffsetFromLatest, partitionsSpec: partitionsSpec! };
}
- async getCompactionConfiguration(datasourceName: string): Promise {
- await this.openCompactionConfigurationDialog(datasourceName);
+ /**
+ * Runs an action from the datasource's menu, and confirms it (ticking the checks of what it does, if it asks).
+ * @param showUnused whether the datasource is only shown with "Show unused" (all of its segments are unused)
+ */
+ async runAction(
+ datasourceName: string,
+ action: DatasourceAction,
+ { showUnused = false }: { showUnused?: boolean } = {},
+ ): Promise {
+ await this.open(datasourceName, showUnused);
+ await this.openActionMenu();
+ await clickMenuItem(this.page, action);
+
+ const confirmation = this.page.locator('.async-action-dialog');
+ await confirmation.waitFor();
+ // The checks of what it does (like "I understand that this operation cannot be undone") are switches
+ for (const check of await confirmation.locator('.warning-checklist .bp6-switch').all()) {
+ await check.click();
+ }
+ await showStep(this.page, `Datasources view: ${action}, to confirm`);
+ await clickButton(confirmation, CONFIRMED_ACTIONS[action]);
+ await confirmation.waitFor({ state: 'detached' });
+ }
- const skipOffsetFromLatest = await getLabeledInput(this.page, SKIP_OFFSET_FROM_LATEST);
- const partitionsSpec = await readPartitionSpec(this.page);
+ /**
+ * Replaces the retention rules of the datasource with rules of the given types (with their defaults), in order. With
+ * no rules, the datasource uses the cluster default rules.
+ * @param comment why (the dialog asks, for the audit history)
+ */
+ async setRetentionRules(
+ datasourceName: string,
+ ruleTypes: string[],
+ comment: string,
+ ): Promise {
+ await this.open(datasourceName, false);
+ await this.openActionMenu();
+ await clickMenuItem(this.page, 'Edit retention rules');
+
+ const dialog = this.page.locator('.retention-dialog');
+ // The datasource's rules (the dialog also shows the cluster default rules, which can't be deleted here)
+ const ruleEditors = dialog
+ .locator('.rule-editor')
+ .filter({ has: this.page.locator('.title .bp6-icon-trash') });
+ await dialog.waitFor();
+ while ((await ruleEditors.count()) > 0) {
+ await ruleEditors.first().locator('.title .bp6-icon-trash').click();
+ }
+ for (const ruleType of ruleTypes) {
+ await clickButton(dialog, 'New rule');
+ await ruleEditors.last().locator('select').selectOption(ruleType);
+ }
+ await showStep(this.page, 'Retention rules dialog, filled in');
+ await clickButton(dialog, 'Next');
- await clickButton(this.page, 'Close');
- return new CompactionConfig({ skipOffsetFromLatest, partitionsSpec: partitionsSpec! });
+ await this.page.getByPlaceholder('Enter description here').fill(comment);
+ await showStep(this.page, 'Retention rules dialog, why');
+ await clickButton(dialog, 'Save');
+ await dialog.waitFor({ state: 'detached' });
}
- private async openEditActions(datasourceName: string): Promise {
- const datasources = await this.getDatasources();
- const index = datasources.findIndex(t => t.name === datasourceName);
- if (index < 0) {
- throw new Error(`Could not find datasource: ${datasourceName}`);
- }
+ private async open(datasourceName: string, showUnused: boolean): Promise {
+ await openView(this.page, 'datasources', { datasource: datasourceName });
+ if (!showUnused) return;
- const editActions = await this.page.$$('.action-cell span.bp6-icon-more');
- await editActions[index].click();
- await this.waitForPopupMenu();
+ // Showing the unused datasources fetches them; until then the table shows what it had
+ const unusedFetched = this.page.waitForResponse(response =>
+ response.url().includes('/druid/coordinator/v1/metadata/datasources?includeUnused'),
+ );
+ await this.page.locator('.datasources-view').getByText('Show unused', { exact: true }).click();
+ await unusedFetched;
+ await this.table().locator('.loader').waitFor({ state: 'hidden' });
}
- private async waitForPopupMenu(): Promise {
- await this.page.waitForSelector('ul.bp6-menu');
+ private async openActionMenu(): Promise {
+ await this.table().locator('.ct-tbody .action-cell .bp6-icon-more').click();
}
- async triggerCompaction(): Promise {
- await this.page.goto(this.baseUrl);
- await this.clickMoreButton({ modifiers: ['Alt'] });
- await this.clickMenuItem('Force compaction run');
- await clickButton(this.page, 'Force compaction run');
+ private async openCompactionConfigurationDialog(datasourceName: string) {
+ await this.open(datasourceName, false);
+ await this.openActionMenu();
+ await clickMenuItem(this.page, 'Edit compaction configuration');
+
+ const dialog = this.page.locator('.compaction-config-dialog');
+ await dialog.waitFor();
+ return dialog;
}
- private async clickMoreButton(options: any): Promise {
- await this.page.click('.more-button button', options);
- await this.waitForPopupMenu();
+ async triggerCompaction(): Promise {
+ await openView(this.page, 'datasources');
+ await this.page.locator('.more-button button').click({ modifiers: ['Alt'] });
+ await clickMenuItem(this.page, 'Force compaction run');
+ const confirmation = this.page.locator('.bp6-alert');
+ await confirmation.waitFor();
+ await showStep(this.page, 'Force compaction run, to confirm');
+ await clickButton(confirmation, 'Force compaction run');
}
}
diff --git a/web-console/e2e-tests/component/ingestion/overview.ts b/web-console/e2e-tests/component/ingestion/overview.ts
index efc2bdb17244..3cb60517c6c1 100644
--- a/web-console/e2e-tests/component/ingestion/overview.ts
+++ b/web-console/e2e-tests/component/ingestion/overview.ts
@@ -16,50 +16,42 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
+import type { Page } from '@playwright/test';
-import { extractTable } from '../../util/table';
-
-import { IngestionTask } from './task';
+import { openView } from '../../util/playwright';
+import { showStep } from '../../util/steps';
+import { extractTableRecords } from '../../util/table';
/**
- * Ingestion overview task table column identifiers.
+ * A row of the Tasks view.
*/
-enum TaskColumn {
- TASK_ID = 0,
- GROUP_ID,
- TYPE,
- DATASOURCE,
- STATUS,
- CREATED_TIME,
- DURATION,
- LOCATION,
+export interface IngestionTask {
+ readonly datasource: string;
+ readonly status: string;
}
/**
* Represents task tab.
*/
export class TasksOverview {
- private readonly page: playwright.Page;
- private readonly baseUrl: string;
+ private readonly page: Page;
- constructor(page: playwright.Page, unifiedConsoleUrl: string) {
+ constructor(page: Page) {
this.page = page;
- this.baseUrl = unifiedConsoleUrl + '#tasks';
}
- async getTasks(): Promise {
- await this.page.goto(this.baseUrl);
- await this.page.reload({ waitUntil: 'networkidle' });
+ /**
+ * The tasks of the datasource `datasourceName`, as the view shows them.
+ */
+ async getTasks(datasourceName: string): Promise {
+ await openView(this.page, 'tasks', { datasource: datasourceName });
- const data = await extractTable(this.page, 'div.ct-tr-group', 'div.ct-td');
+ const records = await extractTableRecords(this.page.locator('.tasks-view .console-table'));
+ await showStep(this.page, `Tasks view, filtered on ${datasourceName}`);
- return data.map(
- row =>
- new IngestionTask({
- datasource: row[TaskColumn.DATASOURCE],
- status: row[TaskColumn.STATUS],
- }),
- );
+ return records.map(record => ({
+ datasource: record['Datasource'],
+ status: record['Status'],
+ }));
}
}
diff --git a/web-console/e2e-tests/component/ingestion/task.ts b/web-console/e2e-tests/component/ingestion/task.ts
deleted file mode 100644
index 254db429d285..000000000000
--- a/web-console/e2e-tests/component/ingestion/task.ts
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Represents row in ingestion overview task table.
- */
-export class IngestionTask {
- constructor(props: IngestionTaskProps) {
- Object.assign(this, props);
- }
-}
-
-interface IngestionTaskProps {
- readonly datasource: string;
- readonly status: string;
-}
-
-export interface IngestionTask extends IngestionTaskProps {}
diff --git a/web-console/e2e-tests/component/load-data/config/configure-schema.ts b/web-console/e2e-tests/component/load-data/config/configure-schema.ts
deleted file mode 100644
index 887a0380ef38..000000000000
--- a/web-console/e2e-tests/component/load-data/config/configure-schema.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Data loader configure schema step configuration.
- */
-export class ConfigureSchemaConfig {
- constructor(props: ConfigureSchemaConfigProps) {
- Object.assign(this, props);
- }
-}
-
-interface ConfigureSchemaConfigProps {
- readonly rollup: boolean;
-}
-
-export interface ConfigureSchemaConfig extends ConfigureSchemaConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts b/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts
deleted file mode 100644
index 73e9bfdff30f..000000000000
--- a/web-console/e2e-tests/component/load-data/config/configure-timestamp.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Data loader configure timestamp step configuration.
- */
-export class ConfigureTimestampConfig {
- constructor(props: ConfigureTimestampConfigProps) {
- Object.assign(this, props);
- }
-}
-
-interface ConfigureTimestampConfigProps {
- readonly timestampExpression: string;
-}
-
-export interface ConfigureTimestampConfig extends ConfigureTimestampConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/config/partition.ts b/web-console/e2e-tests/component/load-data/config/partition.ts
deleted file mode 100644
index 8c45e923c8da..000000000000
--- a/web-console/e2e-tests/component/load-data/config/partition.ts
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* eslint-disable max-classes-per-file */
-
-import type * as playwright from 'playwright-chromium';
-
-import {
- getLabeledInput,
- getLabeledTextarea,
- selectSuggestibleInput,
- setLabeledInput,
- setLabeledTextarea,
-} from '../../../util/playwright';
-
-/**
- * Possible values for partition step segment granularity.
- */
-export enum SegmentGranularity {
- HOUR = 'hour',
- DAY = 'day',
- MONTH = 'month',
- YEAR = 'year',
-}
-
-const PARTITIONING_TYPE = 'Partitioning type';
-
-export interface PartitionsSpec {
- readonly type: string;
- apply(page: playwright.Page): Promise;
-}
-
-export async function readPartitionSpec(page: playwright.Page): Promise {
- const type = await getLabeledInput(page, PARTITIONING_TYPE);
- switch (type) {
- case HashedPartitionsSpec.TYPE:
- return HashedPartitionsSpec.read(page);
- case RangePartitionsSpec.TYPE:
- return RangePartitionsSpec.read(page);
- }
- return null;
-}
-
-export class HashedPartitionsSpec implements PartitionsSpec {
- public static TYPE = 'hashed';
- private static readonly NUM_SHARDS = 'Num shards';
-
- readonly type: string;
-
- static async read(page: playwright.Page): Promise {
- // The shards control may not be visible in that case this is not an error, it is simply not set (null)
- let numShards: number | null = null;
- try {
- numShards = await getLabeledInputAsNumber(page, HashedPartitionsSpec.NUM_SHARDS);
- } catch {}
- return new HashedPartitionsSpec({ numShards });
- }
-
- constructor(props: HashedPartitionsSpecProps) {
- Object.assign(this, props);
- this.type = HashedPartitionsSpec.TYPE;
- }
-
- async apply(page: playwright.Page): Promise {
- await setLabeledInput(page, PARTITIONING_TYPE, this.type);
- if (this.numShards != null) {
- await setLabeledInput(page, HashedPartitionsSpec.NUM_SHARDS, String(this.numShards));
- }
- }
-}
-
-async function getLabeledInputAsNumber(
- page: playwright.Page,
- label: string,
-): Promise {
- const valueString = await getLabeledInput(page, label);
- return valueString === '' ? null : Number(valueString);
-}
-
-async function getLabeledTextareaAsArray(page: playwright.Page, label: string): Promise {
- const valueString = await getLabeledTextarea(page, label);
- return valueString === '' ? [] : valueString.split(',').map(v => v.trim());
-}
-
-interface HashedPartitionsSpecProps {
- readonly numShards: number | null;
-}
-
-export interface HashedPartitionsSpec extends HashedPartitionsSpecProps {}
-
-export class RangePartitionsSpec implements PartitionsSpec {
- public static TYPE = 'range';
- private static readonly PARTITION_DIMENSIONS = 'Partition dimensions';
- private static readonly TARGET_ROWS_PER_SEGMENT = 'Target rows per segment';
- private static readonly MAX_ROWS_PER_SEGMENT = 'Max rows per segment';
-
- readonly type: string;
-
- static async read(page: playwright.Page): Promise {
- const partitionDimensions = await getLabeledTextareaAsArray(
- page,
- RangePartitionsSpec.PARTITION_DIMENSIONS,
- );
- const targetRowsPerSegment = await getLabeledInputAsNumber(
- page,
- RangePartitionsSpec.TARGET_ROWS_PER_SEGMENT,
- );
- const maxRowsPerSegment = await getLabeledInputAsNumber(
- page,
- RangePartitionsSpec.MAX_ROWS_PER_SEGMENT,
- );
- return new RangePartitionsSpec({
- partitionDimensions,
- targetRowsPerSegment,
- maxRowsPerSegment,
- });
- }
-
- constructor(props: RangePartitionsSpecProps) {
- Object.assign(this, props);
- this.type = RangePartitionsSpec.TYPE;
- }
-
- async apply(page: playwright.Page): Promise {
- await selectSuggestibleInput(page, PARTITIONING_TYPE, this.type);
- await setLabeledTextarea(
- page,
- RangePartitionsSpec.PARTITION_DIMENSIONS,
- this.partitionDimensions.join(', '),
- );
- if (this.targetRowsPerSegment) {
- await setLabeledInput(
- page,
- RangePartitionsSpec.TARGET_ROWS_PER_SEGMENT,
- String(this.targetRowsPerSegment),
- );
- }
- if (this.maxRowsPerSegment) {
- await setLabeledInput(
- page,
- RangePartitionsSpec.MAX_ROWS_PER_SEGMENT,
- String(this.maxRowsPerSegment),
- );
- }
- }
-}
-
-interface RangePartitionsSpecProps {
- readonly partitionDimensions: string[];
- readonly targetRowsPerSegment: number | null;
- readonly maxRowsPerSegment: number | null;
-}
-
-export interface RangePartitionsSpec extends RangePartitionsSpecProps {}
-
-/**
- * Data loader partition step configuration.
- */
-export class PartitionConfig {
- constructor(props: PartitionConfigProps) {
- Object.assign(this, props);
- }
-}
-
-interface PartitionConfigProps {
- readonly segmentGranularity: SegmentGranularity;
- readonly timeIntervals: string | null;
- readonly partitionsSpec: PartitionsSpec | null;
-}
-
-export interface PartitionConfig extends PartitionConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/config/publish.ts b/web-console/e2e-tests/component/load-data/config/publish.ts
deleted file mode 100644
index caaa05bf7d51..000000000000
--- a/web-console/e2e-tests/component/load-data/config/publish.ts
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Data loader publish step configuration.
- */
-export class PublishConfig {
- constructor(props: PublishConfigProps) {
- Object.assign(this, props);
- }
-}
-
-interface PublishConfigProps {
- readonly datasourceName: string;
-}
-
-export interface PublishConfig extends PublishConfigProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-connector.ts b/web-console/e2e-tests/component/load-data/data-connector.ts
new file mode 100644
index 000000000000..44913ff0814c
--- /dev/null
+++ b/web-console/e2e-tests/component/load-data/data-connector.ts
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+
+import {
+ clickButton,
+ setLabeledBoolean,
+ setLabeledInput,
+ setLabeledTextarea,
+} from '../../util/playwright';
+
+/**
+ * Where the data loader reads the data from.
+ */
+export type DataConnector =
+ | { readonly type: 'local'; readonly baseDirectory: string; readonly fileFilter: string }
+ | { readonly type: 's3'; readonly uris: string[] }
+ | { readonly type: 'kafka'; readonly bootstrapServers: string; readonly topic: string }
+ | { readonly type: 'reindex'; readonly datasourceName: string; readonly interval: string };
+
+/**
+ * The title of the connector's card on the data loader's start page.
+ */
+export function connectorCardTitle(connector: DataConnector): string {
+ switch (connector.type) {
+ case 'local':
+ return 'Local disk';
+ case 's3':
+ return 'Amazon S3';
+ case 'kafka':
+ return 'Apache Kafka';
+ case 'reindex':
+ return 'Reindex from Druid';
+ }
+}
+
+/**
+ * Whether the data needs parsing, so goes through the Parse data and Parse time steps (data from Druid doesn't).
+ */
+export function connectorNeedsParse(connector: DataConnector): boolean {
+ return connector.type !== 'reindex';
+}
+
+/**
+ * Whether the connector reads a stream, so the data loader sets up a supervisor (rather than run a task).
+ */
+export function connectorIsStreaming(connector: DataConnector): boolean {
+ return connector.type === 'kafka';
+}
+
+/**
+ * Fills in the fields of the connector (of the classic data loader's Connect step, or of the SQL data loader's input
+ * source step, which share them).
+ */
+export async function fillConnector(page: Page, connector: DataConnector): Promise {
+ switch (connector.type) {
+ case 'local':
+ await setLabeledInput(page, 'Base directory', connector.baseDirectory);
+ await setLabeledInput(page, 'File filter', connector.fileFilter);
+ break;
+
+ case 's3':
+ await setLabeledTextarea(page, 'S3 URIs', connector.uris.join(', '));
+ break;
+
+ case 'kafka':
+ // The data is sampled from the start of the stream (the default)
+ await setLabeledInput(page, 'Bootstrap servers', connector.bootstrapServers);
+ await setLabeledInput(page, 'Topic', connector.topic);
+ break;
+
+ case 'reindex':
+ await setLabeledInput(page, 'Datasource', connector.datasourceName);
+ await setLabeledInput(page, 'Interval', connector.interval);
+ break;
+ }
+}
+
+/**
+ * Fills in the Connect step and applies it.
+ */
+export async function connect(page: Page, connector: DataConnector): Promise {
+ await fillConnector(page, connector);
+ await clickButton(page, 'Apply');
+}
+
+/**
+ * Fills in what the Tune step needs for the connector.
+ */
+export async function tune(page: Page, connector: DataConnector): Promise {
+ if (connector.type === 'kafka') {
+ // Required for streaming, as there's no default: read the topic from its start (the data the test put in it)
+ await setLabeledBoolean(page, 'Use earliest offset', true);
+ }
+}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts b/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts
deleted file mode 100644
index c0da3f9ae957..000000000000
--- a/web-console/e2e-tests/component/load-data/data-connector/data-connector.ts
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type * as playwright from 'playwright-chromium';
-
-import { clickButton } from '../../../util/playwright';
-
-/**
- * Connector for data loader input data.
- */
-export interface DataConnector {
- readonly name: string;
- readonly needParse: boolean;
- connect(): Promise;
-}
-
-export async function clickApplyButton(page: playwright.Page): Promise {
- await clickButton(page, 'Apply');
-}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts b/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
deleted file mode 100644
index cd33b7766e15..000000000000
--- a/web-console/e2e-tests/component/load-data/data-connector/local-file.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type * as playwright from 'playwright-chromium';
-
-import { setLabeledInput } from '../../../util/playwright';
-
-import type { DataConnector } from './data-connector';
-import { clickApplyButton } from './data-connector';
-
-/**
- * Local file connector for data loader input data.
- */
-export class LocalFileDataConnector implements DataConnector {
- readonly name: string;
- readonly needParse: boolean;
- private readonly page: playwright.Page;
-
- constructor(page: playwright.Page, props: LocalFileDataConnectorProps) {
- Object.assign(this, props);
- this.name = 'Local disk';
- this.needParse = true;
- this.page = page;
- }
-
- async connect() {
- await setLabeledInput(this.page, 'Base directory', this.baseDirectory);
- await setLabeledInput(this.page, 'File filter', this.fileFilter);
- await clickApplyButton(this.page);
- }
-}
-
-interface LocalFileDataConnectorProps {
- readonly baseDirectory: string;
- readonly fileFilter: string;
-}
-
-export interface LocalFileDataConnector extends LocalFileDataConnectorProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-connector/reindex.ts b/web-console/e2e-tests/component/load-data/data-connector/reindex.ts
deleted file mode 100644
index 35f1dc3d9b92..000000000000
--- a/web-console/e2e-tests/component/load-data/data-connector/reindex.ts
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type * as playwright from 'playwright-chromium';
-
-import { setLabeledInput } from '../../../util/playwright';
-
-import type { DataConnector } from './data-connector';
-import { clickApplyButton } from './data-connector';
-
-/**
- * Reindexing connector for data loader input data.
- */
-export class ReindexDataConnector implements DataConnector {
- readonly name: string;
- readonly needParse: boolean;
- private readonly page: playwright.Page;
-
- constructor(page: playwright.Page, props: ReindexDataConnectorProps) {
- Object.assign(this, props);
- this.name = 'Reindex from Druid';
- this.needParse = false;
- this.page = page;
- }
-
- async connect() {
- await setLabeledInput(this.page, 'Datasource', this.datasourceName);
- await setLabeledInput(this.page, 'Interval', this.interval);
- await clickApplyButton(this.page);
- }
-}
-
-interface ReindexDataConnectorProps {
- readonly datasourceName: string;
- readonly interval: string;
-}
-
-export interface ReindexDataConnector extends ReindexDataConnectorProps {}
diff --git a/web-console/e2e-tests/component/load-data/data-loader.ts b/web-console/e2e-tests/component/load-data/data-loader.ts
index a7a48639eb22..5dc1ca452f03 100644
--- a/web-console/e2e-tests/component/load-data/data-loader.ts
+++ b/web-console/e2e-tests/component/load-data/data-loader.ts
@@ -16,174 +16,155 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
-
-import { clickButton, setLabeledInput, setLabeledTextarea } from '../../util/playwright';
-
-import type { ConfigureSchemaConfig } from './config/configure-schema';
-import type { ConfigureTimestampConfig } from './config/configure-timestamp';
-import type { PartitionConfig } from './config/partition';
-import type { PublishConfig } from './config/publish';
-import type { DataConnector } from './data-connector/data-connector';
+import type { Page } from '@playwright/test';
+
+import {
+ clickButton,
+ getLabeledInput,
+ openView,
+ setLabeledInput,
+ setLabeledTextarea,
+} from '../../util/playwright';
+import { showStep } from '../../util/steps';
+
+import type { DataConnector } from './data-connector';
+import {
+ connect,
+ connectorCardTitle,
+ connectorIsStreaming,
+ connectorNeedsParse,
+ tune,
+} from './data-connector';
+import type { PartitionsSpec } from './partitions-spec';
+import { applyPartitionsSpec } from './partitions-spec';
/**
- * Represents load data tab.
+ * What to set at each step of the (classic) data loader.
*/
-export class DataLoader {
- private readonly baseUrl: string;
-
- constructor(props: DataLoaderProps) {
- Object.assign(this, props);
- this.baseUrl = props.unifiedConsoleUrl + '#data-loader';
- }
-
- /**
- * Execute each step to load data.
- */
- async load() {
- await this.page.goto(this.baseUrl);
- await this.startNewSpecIfNeeded();
- await this.start();
- await this.connect(this.connector, this.connectValidator);
- if (this.connector.needParse) {
- await this.parseData();
- await this.parseTime(this.configureTimestampConfig);
- }
- await this.transform();
- await this.filter();
- await this.configureSchema(this.configureSchemaConfig);
- await this.partition(this.partitionConfig);
- await this.tune();
- await this.publish(this.publishConfig);
- await this.editSpec();
- }
-
- private async startNewSpecIfNeeded() {
- const startNewSpecLocator = this.page.locator(`//*[contains(text(),"Start a new")]`);
- if (await startNewSpecLocator.count()) {
- await startNewSpecLocator.click();
- }
- }
-
- private async start() {
- const cardSelector = `//*[contains(@class,"bp6-card")][p[contains(text(),"${this.connector.name}")]]`;
- await this.page.click(cardSelector);
- await clickButton(this.page, 'Connect data');
- }
-
- private async connect(connector: DataConnector, validator: (previewLines: string[]) => void) {
- await connector.connect();
- await this.validateConnect(validator);
- const next = this.connector.needParse ? 'Parse data' : 'Transform';
- await clickButton(this.page, `Next: ${next}`);
- }
-
- private async validateConnect(validator: (previewLines: string[]) => void) {
- await this.page.waitForSelector('.raw-lines');
- const previewLines = await this.page.$$eval('.raw-lines .raw-line', els =>
- els.map(el => el.textContent ?? ''),
- );
- validator(previewLines);
- }
-
- private async parseData() {
- await this.page.waitForSelector('.parse-data-table');
- await clickButton(this.page, 'Next: Parse time');
- }
-
- private async parseTime(configureTimestampConfig?: ConfigureTimestampConfig) {
- await this.page.waitForSelector('.parse-time-table');
- if (configureTimestampConfig) {
- await this.applyConfigureTimestampConfig(configureTimestampConfig);
- }
- await clickButton(this.page, 'Next: Transform');
- }
-
- private async transform() {
- await this.page.waitForSelector('.transform-table');
- await clickButton(this.page, 'Next: Filter');
- }
-
- private async filter() {
- await this.page.waitForSelector('.filter-table');
- await clickButton(this.page, 'Next: Configure schema');
- }
-
- private async configureSchema(configureSchemaConfig: ConfigureSchemaConfig) {
- await this.page.waitForSelector('.schema-table');
- await this.applyConfigureSchemaConfig(configureSchemaConfig);
- await clickButton(this.page, 'Next: Partition');
- }
-
- private async applyConfigureTimestampConfig(configureTimestampConfig: ConfigureTimestampConfig) {
- await clickButton(this.page, 'Expression');
- await setLabeledInput(this.page, 'Expression', configureTimestampConfig.timestampExpression);
- await clickButton(this.page, 'Apply');
- }
-
- private async applyConfigureSchemaConfig(configureSchemaConfig: ConfigureSchemaConfig) {
- const rollupSelector = '//*[text()="Rollup"]';
- const rollupInput = await this.page.$(`${rollupSelector}/input`);
- const rollupChecked = await rollupInput!.evaluate(el => (el as HTMLInputElement).checked);
- if (rollupChecked !== configureSchemaConfig.rollup) {
- await this.page.click(rollupSelector);
- const confirmationDialogSelector = '//*[contains(@class,"bp6-alert-body")]';
- await this.page.waitForSelector(confirmationDialogSelector);
- await clickButton(this.page, 'Yes');
- const statusMessageSelector = '.recipe-toaster';
- await this.page.waitForSelector(statusMessageSelector);
- await this.page.click(`${statusMessageSelector} button`);
- }
- }
-
- private async partition(partitionConfig: PartitionConfig) {
- await this.page.waitForSelector('div.load-data-view.partition');
- await this.applyPartitionConfig(partitionConfig);
- await clickButton(this.page, 'Next: Tune');
- }
+export interface DataLoaderConfig {
+ // Connect
+ readonly connector: DataConnector;
+ /** Checks the raw lines of the preview */
+ readonly validateConnect: (previewLines: string[]) => void;
+ // Parse data (when the connector's data needs parsing)
+ /** Checks the input format that the data loader picked and the columns it parsed */
+ readonly validateParseData?: (parsed: { inputFormat: string; columns: string[] }) => void;
+ // Parse time (when the connector's data needs parsing)
+ readonly timestampExpression?: string;
+ // Configure schema
+ readonly rollup: boolean;
+ // Partition
+ readonly segmentGranularity: 'hour' | 'day' | 'month' | 'year';
+ readonly timeIntervals?: string;
+ readonly partitionsSpec?: PartitionsSpec;
+ // Publish
+ readonly datasourceName: string;
+}
- private async applyPartitionConfig(partitionConfig: PartitionConfig) {
- await setLabeledInput(this.page, 'Segment granularity', partitionConfig.segmentGranularity);
- if (partitionConfig.timeIntervals) {
- await setLabeledTextarea(this.page, 'Time intervals', partitionConfig.timeIntervals);
+/**
+ * Goes through each step of the data loader and submits the task (or the supervisor, for a stream).
+ */
+export async function loadData(page: Page, config: DataLoaderConfig): Promise {
+ const nextBar = page.locator('.next-bar');
+ const clickNext = (step: string) => clickButton(nextBar, `Next: ${step}`);
+
+ await openView(page, 'data-loader');
+ await page
+ .locator('.bp6-card')
+ .filter({ has: page.locator('p', { hasText: connectorCardTitle(config.connector) }) })
+ .click();
+ await showStep(page, `Data loader: ${connectorCardTitle(config.connector)} picked`);
+ await clickButton(page, 'Connect data');
+
+ // Connect
+ await connect(page, config.connector);
+ const rawLines = page.locator('.raw-lines .raw-line');
+ await rawLines.first().waitFor();
+ config.validateConnect(await rawLines.allTextContents());
+ await showStep(page, 'Data loader: Connect, with a preview of the data');
+
+ if (connectorNeedsParse(config.connector)) {
+ await clickNext('Parse data');
+
+ // Parse data
+ const parseDataTable = page.locator('.parse-data-table');
+ await parseDataTable.waitFor();
+ if (config.validateParseData) {
+ await parseDataTable.locator('.column-name').first().waitFor();
+ config.validateParseData({
+ inputFormat: await getLabeledInput(page, 'Input format'),
+ columns: await parseDataTable.locator('.column-name').allTextContents(),
+ });
}
- if (partitionConfig.partitionsSpec != null) {
- await partitionConfig.partitionsSpec.apply(this.page);
+ await showStep(page, 'Data loader: Parse data');
+ await clickNext('Parse time');
+
+ // Parse time
+ await page.locator('.parse-time-table').waitFor();
+ if (config.timestampExpression) {
+ await clickButton(page, 'Expression');
+ await setLabeledInput(page, 'Expression', config.timestampExpression);
+ await clickButton(page, 'Apply');
}
- }
-
- private async tune() {
- await this.page.waitForSelector('div.load-data-view.tuning');
- await clickButton(this.page, 'Next: Publish');
- }
-
- private async publish(publishConfig: PublishConfig) {
- await this.page.waitForSelector('div.load-data-view.publish');
- await this.applyPublishConfig(publishConfig);
- await clickButton(this.page, 'Edit spec');
- }
-
- private async applyPublishConfig(publishConfig: PublishConfig) {
- if (publishConfig.datasourceName != null) {
- await setLabeledInput(this.page, 'Datasource name', publishConfig.datasourceName);
- }
- }
-
- private async editSpec() {
- await this.page.waitForSelector('div.load-data-view.spec');
- await clickButton(this.page, 'Submit');
- }
+ await showStep(page, 'Data loader: Parse time');
+ }
+ await clickNext('Transform');
+
+ // Transform
+ await page.locator('.transform-table').waitFor();
+ await showStep(page, 'Data loader: Transform');
+ await clickNext('Filter');
+
+ // Filter
+ await page.locator('.filter-table').waitFor();
+ await showStep(page, 'Data loader: Filter');
+ await clickNext('Configure schema');
+
+ // Configure schema
+ await page.locator('.schema-table').waitFor();
+ await setRollup(page, config.rollup);
+ await showStep(page, 'Data loader: Configure schema');
+ await clickNext('Partition');
+
+ // Partition
+ await page.locator('.load-data-view.partition').waitFor();
+ await setLabeledInput(page, 'Segment granularity', config.segmentGranularity);
+ if (config.timeIntervals) {
+ await setLabeledTextarea(page, 'Time intervals', config.timeIntervals);
+ }
+ if (config.partitionsSpec) {
+ await applyPartitionsSpec(page, config.partitionsSpec);
+ }
+ await showStep(page, 'Data loader: Partition');
+ await clickNext('Tune');
+
+ // Tune
+ await page.locator('.load-data-view.tuning').waitFor();
+ await tune(page, config.connector);
+ await showStep(page, 'Data loader: Tune');
+ await clickNext('Publish');
+
+ // Publish
+ await page.locator('.load-data-view.publish').waitFor();
+ await setLabeledInput(page, 'Datasource name', config.datasourceName);
+ await showStep(page, 'Data loader: Publish');
+ await clickNext('Edit spec');
+
+ // Edit spec
+ await page.locator('.load-data-view.spec').waitFor();
+ await showStep(page, 'Data loader: Edit spec');
+ await clickButton(
+ nextBar,
+ connectorIsStreaming(config.connector) ? 'Submit supervisor (running)' : 'Submit task',
+ );
}
-interface DataLoaderProps {
- readonly page: playwright.Page;
- readonly unifiedConsoleUrl: string;
- readonly connector: DataConnector;
- readonly connectValidator: (previewLines: string[]) => void;
- readonly configureTimestampConfig?: ConfigureTimestampConfig;
- readonly configureSchemaConfig: ConfigureSchemaConfig;
- readonly partitionConfig: PartitionConfig;
- readonly publishConfig: PublishConfig;
-}
+async function setRollup(page: Page, rollup: boolean): Promise {
+ const rollupSwitch = page.getByLabel('Rollup', { exact: true });
+ if ((await rollupSwitch.isChecked()) === rollup) return;
-export interface DataLoader extends DataLoaderProps {}
+ // The switch's input is visually hidden, so click its label (which asks for confirmation)
+ await page.locator('label', { has: rollupSwitch }).click();
+ await clickButton(page.locator('.bp6-alert'), `Yes - ${rollup ? 'enable' : 'disable'} rollup`);
+ await page.locator('.recipe-toaster').getByRole('button').click();
+}
diff --git a/web-console/e2e-tests/component/load-data/partitions-spec.ts b/web-console/e2e-tests/component/load-data/partitions-spec.ts
new file mode 100644
index 000000000000..dd91fae2c7e7
--- /dev/null
+++ b/web-console/e2e-tests/component/load-data/partitions-spec.ts
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+
+import {
+ formGroup,
+ getLabeledInput,
+ getLabeledTextarea,
+ selectSuggestibleInput,
+ setLabeledInput,
+ setLabeledTextarea,
+} from '../../util/playwright';
+
+/**
+ * A partitions spec, as set in the form of the data loader's Partition step or of the compaction config dialog.
+ */
+export type PartitionsSpec =
+ | { readonly type: 'hashed'; readonly numShards: number | null }
+ | {
+ readonly type: 'range';
+ readonly partitionDimensions: string[];
+ readonly targetRowsPerSegment: number | null;
+ readonly maxRowsPerSegment: number | null;
+ };
+
+const PARTITIONING_TYPE = 'Partitioning type';
+const NUM_SHARDS = 'Num shards';
+const PARTITION_DIMENSIONS = 'Partition dimensions';
+const TARGET_ROWS_PER_SEGMENT = 'Target rows per segment';
+const MAX_ROWS_PER_SEGMENT = 'Max rows per segment';
+
+export async function applyPartitionsSpec(
+ page: Page,
+ partitionsSpec: PartitionsSpec,
+): Promise {
+ switch (partitionsSpec.type) {
+ case 'hashed':
+ await setLabeledInput(page, PARTITIONING_TYPE, partitionsSpec.type);
+ if (partitionsSpec.numShards != null) {
+ await setLabeledInput(page, NUM_SHARDS, String(partitionsSpec.numShards));
+ }
+ break;
+
+ case 'range':
+ await selectSuggestibleInput(page, PARTITIONING_TYPE, partitionsSpec.type);
+ await setLabeledTextarea(
+ page,
+ PARTITION_DIMENSIONS,
+ partitionsSpec.partitionDimensions.join(', '),
+ );
+ if (partitionsSpec.targetRowsPerSegment != null) {
+ await setLabeledInput(
+ page,
+ TARGET_ROWS_PER_SEGMENT,
+ String(partitionsSpec.targetRowsPerSegment),
+ );
+ }
+ if (partitionsSpec.maxRowsPerSegment != null) {
+ await setLabeledInput(page, MAX_ROWS_PER_SEGMENT, String(partitionsSpec.maxRowsPerSegment));
+ }
+ break;
+ }
+}
+
+export async function readPartitionsSpec(page: Page): Promise {
+ switch (await getLabeledInput(page, PARTITIONING_TYPE)) {
+ case 'hashed':
+ return {
+ type: 'hashed',
+ // The shards field is not always shown, then it is not set
+ numShards: (await formGroup(page, NUM_SHARDS).count())
+ ? await getLabeledNumber(page, NUM_SHARDS)
+ : null,
+ };
+
+ case 'range': {
+ const partitionDimensions = await getLabeledTextarea(page, PARTITION_DIMENSIONS);
+ return {
+ type: 'range',
+ partitionDimensions: partitionDimensions
+ ? partitionDimensions.split(',').map(d => d.trim())
+ : [],
+ targetRowsPerSegment: await getLabeledNumber(page, TARGET_ROWS_PER_SEGMENT),
+ maxRowsPerSegment: await getLabeledNumber(page, MAX_ROWS_PER_SEGMENT),
+ };
+ }
+
+ default:
+ return;
+ }
+}
+
+async function getLabeledNumber(page: Page, label: string): Promise {
+ const value = await getLabeledInput(page, label);
+ return value === '' ? null : Number(value);
+}
diff --git a/web-console/e2e-tests/component/load-data/sql-data-loader.ts b/web-console/e2e-tests/component/load-data/sql-data-loader.ts
new file mode 100644
index 000000000000..3a25cb83a165
--- /dev/null
+++ b/web-console/e2e-tests/component/load-data/sql-data-loader.ts
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+
+import { clickButton, getLabeledInput, openView, setLabeledInput } from '../../util/playwright';
+import { showStep } from '../../util/steps';
+import { waitForTaskSlots } from '../workbench/overview';
+
+import type { DataConnector } from './data-connector';
+import { connectorCardTitle, fillConnector } from './data-connector';
+
+/**
+ * What to set at each step of the SQL data loader ("Load data" > "Batch - SQL"), which ingests with an MSQ task.
+ */
+export interface SqlDataLoaderConfig {
+ readonly connector: DataConnector;
+ /** Checks the input format that the data loader picked and the columns it parsed */
+ readonly validateParseData?: (parsed: { inputFormat: string; columns: string[] }) => void;
+ /** The new datasource to create */
+ readonly datasourceName: string;
+}
+
+/**
+ * Goes through each step of the SQL data loader, starts the ingestion and waits for it to finish. Throws if it fails.
+ */
+export async function loadDataWithSql(page: Page, config: SqlDataLoaderConfig): Promise {
+ await openView(page, 'sql-data-loader');
+
+ // Select input type
+ await page
+ .locator('.input-source-step .bp6-card')
+ .filter({ has: page.locator('p', { hasText: connectorCardTitle(config.connector) }) })
+ .click();
+ await fillConnector(page, config.connector);
+ await showStep(page, `SQL data loader: ${connectorCardTitle(config.connector)}`);
+ await clickButton(page, 'Connect data');
+
+ // Parse
+ const parseDataTable = page.locator('.input-format-step .parse-data-table');
+ await parseDataTable.locator('.column-name').first().waitFor();
+ if (config.validateParseData) {
+ config.validateParseData({
+ inputFormat: await getLabeledInput(page, 'Input format'),
+ columns: await parseDataTable.locator('.column-name').allTextContents(),
+ });
+ }
+ await showStep(page, 'SQL data loader: Parse');
+ await clickButton(page.locator('.input-format-step .prev-next-bar'), 'Next');
+
+ // Schema, with the destination (a new datasource)
+ const schemaStep = page.locator('.schema-step');
+ await schemaStep.locator('.destination-button').click();
+ const destinationDialog = page.locator('.destination-dialog');
+ await clickButton(destinationDialog, 'New table');
+ await setLabeledInput(destinationDialog, 'New table name', config.datasourceName);
+ await showStep(page, 'SQL data loader: Destination');
+ await clickButton(destinationDialog, 'Save');
+ await destinationDialog.waitFor({ state: 'detached' });
+ await showStep(page, 'SQL data loader: Schema');
+ await clickButton(schemaStep.locator('.prev-next-bar'), 'Start loading data');
+
+ // Ingestion progress
+ const progressDialog = page.locator('.ingestion-progress-dialog');
+ const done = progressDialog.getByText('Done loading data into');
+ const error = progressDialog.getByText('Error ingesting data');
+ await waitForTaskSlots(page, done.or(error));
+ await showStep(page, 'SQL data loader: Ingestion progress, done');
+ if (await error.isVisible()) {
+ throw new Error(`SQL data loader failed: ${await error.innerText()}`);
+ }
+}
diff --git a/web-console/e2e-tests/component/lookups/overview.ts b/web-console/e2e-tests/component/lookups/overview.ts
new file mode 100644
index 000000000000..0983be89bc29
--- /dev/null
+++ b/web-console/e2e-tests/component/lookups/overview.ts
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+
+import { button, clickButton, openView, setLabeledInput } from '../../util/playwright';
+import { showStep } from '../../util/steps';
+import { extractTableRecords } from '../../util/table';
+
+/**
+ * A row of the Lookups view.
+ */
+export interface Lookup {
+ readonly name: string;
+ readonly tier: string;
+ readonly type: string;
+}
+
+/**
+ * A lookup on a table of a database (a cachedNamespace lookup with a jdbc extraction namespace).
+ */
+export interface JdbcLookup {
+ readonly name: string;
+ readonly connectUri: string;
+ readonly table: string;
+ readonly keyColumn: string;
+ readonly valueColumn: string;
+ readonly timestampColumn?: string;
+}
+
+/**
+ * Represents the Lookups view.
+ */
+export class LookupsOverview {
+ private readonly page: Page;
+
+ constructor(page: Page) {
+ this.page = page;
+ }
+
+ private readonly table = () => this.page.locator('.lookups-view .console-table');
+
+ /**
+ * Opens the view, initializing the lookups (their config, with the default tier) if the cluster has none yet, as
+ * the view asks.
+ */
+ async open(): Promise {
+ await this.load();
+
+ const initialize = button(this.page, 'Initialize lookups');
+ if (await initialize.isVisible()) {
+ await showStep(this.page, 'Lookups view, not initialized');
+ await initialize.click();
+ await this.table().waitFor();
+ }
+ }
+
+ /**
+ * Opens the view and waits for it to fetch the lookups, as it shows "No lookups" (and the "Add lookup" button) also
+ * while it does. It reloads, to be sure the fetch it waits for is one that starts after this.
+ */
+ private async load(filter?: Record): Promise {
+ await openView(this.page, 'lookups', filter);
+ const lookupsFetched = this.page.waitForResponse(response =>
+ response.url().includes('/druid/coordinator/v1/lookups/config/all'),
+ );
+ await this.page.reload();
+ await lookupsFetched;
+ await this.table().or(button(this.page, 'Initialize lookups')).waitFor();
+ await this.table().locator('.loader').waitFor({ state: 'hidden' });
+ }
+
+ async addJdbcLookup(lookup: JdbcLookup): Promise {
+ await this.open();
+ await clickButton(this.page, 'Add lookup');
+
+ const dialog = this.page.locator('.lookup-edit-dialog');
+ await setLabeledInput(dialog, 'Name', lookup.name);
+ await setLabeledInput(dialog, 'Type', 'cachedNamespace');
+ await setLabeledInput(dialog, 'Extraction type', 'jdbc');
+ await setLabeledInput(dialog, 'Connect URI', lookup.connectUri);
+ await setLabeledInput(dialog, 'Table', lookup.table);
+ await setLabeledInput(dialog, 'Key column', lookup.keyColumn);
+ await setLabeledInput(dialog, 'Value column', lookup.valueColumn);
+ if (lookup.timestampColumn) {
+ await setLabeledInput(dialog, 'Timestamp column', lookup.timestampColumn);
+ }
+ await showStep(this.page, 'Lookup dialog, filled in');
+ await clickButton(dialog, 'Submit');
+ await dialog.waitFor({ state: 'detached' });
+ }
+
+ /**
+ * The lookup named `lookupName` (none, if it doesn't exist), as the view shows it.
+ */
+ async getLookups(lookupName: string): Promise {
+ await this.load({ lookup_name: lookupName });
+
+ const records = await extractTableRecords(this.table());
+ await showStep(this.page, `Lookups view, filtered on ${lookupName}`);
+
+ return records.map(record => ({
+ name: record['Lookup name'],
+ tier: record['Lookup tier'],
+ type: record['Type'],
+ }));
+ }
+}
diff --git a/web-console/e2e-tests/component/query/overview.ts b/web-console/e2e-tests/component/query/overview.ts
deleted file mode 100644
index 541941d18bd4..000000000000
--- a/web-console/e2e-tests/component/query/overview.ts
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type * as playwright from 'playwright-chromium';
-
-import { clickButton, clickText, setQueryInput } from '../../util/playwright';
-import { extractTable } from '../../util/table';
-
-/**
- * Represents query overview tab.
- */
-export class QueryOverview {
- private readonly page: playwright.Page;
- private readonly baseUrl: string;
-
- constructor(page: playwright.Page, unifiedConsoleUrl: string) {
- this.page = page;
- this.baseUrl = unifiedConsoleUrl + '#workbench';
- }
-
- async runQuery(query: string): Promise {
- await this.page.goto(this.baseUrl);
- await this.page.reload({ waitUntil: 'networkidle' });
-
- await setQueryInput(this.page, query);
-
- await clickButton(this.page, 'Run');
- await this.page.waitForSelector('div.result-table-pane');
-
- return await extractTable(this.page, 'div.result-table-pane div.ct-tr-group', 'div.ct-td');
- }
-
- async cancelQuery(query: string): Promise {
- await this.page.goto(this.baseUrl);
- await this.page.reload({ waitUntil: 'networkidle' });
-
- await setQueryInput(this.page, query);
-
- await Promise.all([
- this.page.waitForRequest(
- request => request.url().includes('druid/v2') && request.method() === 'POST',
- ),
- clickButton(this.page, 'Run'),
- ]);
-
- await this.page.waitForSelector('.cancel-label');
-
- const [resp] = await Promise.all([
- this.page.waitForResponse(
- response => response.url().includes('druid/v2') && response.request().method() === 'DELETE',
- ),
-
- clickText(this.page, 'Cancel query'),
- ]);
-
- return resp.status();
- }
-}
diff --git a/web-console/e2e-tests/component/supervisors/overview.ts b/web-console/e2e-tests/component/supervisors/overview.ts
new file mode 100644
index 000000000000..58deb7cedda4
--- /dev/null
+++ b/web-console/e2e-tests/component/supervisors/overview.ts
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+
+import { clickButton, clickMenuItem, openView } from '../../util/playwright';
+import { showStep } from '../../util/steps';
+import { extractTableRecords } from '../../util/table';
+
+/**
+ * A row of the Supervisors view.
+ */
+export interface Supervisor {
+ readonly id: string;
+ readonly datasource: string;
+ readonly status: string;
+}
+
+/**
+ * An action of a supervisor's menu that asks for confirmation, with the text of its confirm button.
+ */
+const CONFIRMED_ACTIONS = {
+ 'Suspend': 'Suspend supervisor',
+ 'Resume': 'Resume supervisor',
+ 'Hard reset': 'Hard reset supervisor',
+ 'Terminate': 'Terminate supervisor',
+} as const;
+
+export type SupervisorAction = keyof typeof CONFIRMED_ACTIONS;
+
+/**
+ * Represents the Supervisors view.
+ */
+export class SupervisorsOverview {
+ private readonly page: Page;
+
+ constructor(page: Page) {
+ this.page = page;
+ }
+
+ private readonly table = () => this.page.locator('.supervisors-view .console-table');
+
+ /**
+ * The supervisor `supervisorId` (none, if it doesn't exist), as the view shows it.
+ */
+ async getSupervisors(supervisorId: string): Promise {
+ await openView(this.page, 'supervisors', { supervisor_id: supervisorId });
+
+ const records = await extractTableRecords(this.table());
+ await showStep(this.page, `Supervisors view, filtered on ${supervisorId}`);
+
+ return records.map(record => ({
+ id: record['Supervisor ID'],
+ datasource: record['Datasource'],
+ status: record['Status'],
+ }));
+ }
+
+ /**
+ * Runs an action from the supervisor's menu, and confirms it.
+ */
+ async runAction(supervisorId: string, action: SupervisorAction): Promise {
+ await openView(this.page, 'supervisors', { supervisor_id: supervisorId });
+ await this.table().locator('.ct-tbody .action-cell .bp6-icon-more').click();
+ await clickMenuItem(this.page, action);
+
+ const confirmation = this.page.locator('.bp6-alert');
+ await confirmation.waitFor();
+ await showStep(this.page, `Supervisors view: ${action}, to confirm`);
+ await clickButton(confirmation, CONFIRMED_ACTIONS[action]);
+ await confirmation.waitFor({ state: 'detached' });
+ }
+}
diff --git a/web-console/e2e-tests/component/workbench/dart-panel.ts b/web-console/e2e-tests/component/workbench/dart-panel.ts
new file mode 100644
index 000000000000..ab5d55197186
--- /dev/null
+++ b/web-console/e2e-tests/component/workbench/dart-panel.ts
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Locator, Page } from '@playwright/test';
+
+import { clickButton, clickMenuItem } from '../../util/playwright';
+import { showStep } from '../../util/steps';
+
+const STATES = ['ACCEPTED', 'RUNNING', 'SUCCESS', 'FAILED', 'CANCELED'];
+
+/**
+ * Represents the "Current Dart queries" panel of the Query view, which lists the Dart queries (running and recent).
+ */
+export class CurrentDartPanel {
+ private readonly page: Page;
+
+ constructor(page: Page) {
+ this.page = page;
+ }
+
+ private readonly panel = () => this.page.locator('.current-dart-panel');
+
+ /**
+ * Shows the panel (in the Query view, which must be open), if it isn't shown. The view remembers it.
+ */
+ async show(): Promise {
+ if (await this.panel().isVisible()) return;
+ await this.page.locator('[data-tooltip="Open helper panels"]').click();
+ await this.page.getByText('Current Dart query panel', { exact: true }).click();
+ await this.panel().waitFor();
+ }
+
+ /**
+ * The state of the query (like RUNNING or SUCCESS) as the panel shows it, or null when it doesn't list it (yet).
+ */
+ async getQueryState(sqlQueryId: string): Promise {
+ const entry = this.entry(sqlQueryId);
+ if (!(await entry.count())) return null;
+
+ const iconClasses = (await entry.locator('.status-icon').getAttribute('class')) ?? '';
+ return STATES.find(state => iconClasses.split(' ').includes(state.toLowerCase())) ?? null;
+ }
+
+ /**
+ * Opens the execution details of the query (its report) and returns the dialog.
+ */
+ async showDetails(sqlQueryId: string): Promise {
+ await this.entry(sqlQueryId).click();
+ await clickMenuItem(this.page, 'Show details');
+
+ const dialog = this.page.locator('.execution-details-dialog');
+ await dialog.waitFor();
+ return dialog;
+ }
+
+ /**
+ * Cancels the query, confirming it.
+ */
+ async cancelQuery(sqlQueryId: string): Promise {
+ await this.entry(sqlQueryId).click();
+ await clickMenuItem(this.page, 'Cancel query');
+
+ const confirmation = this.page.locator('.bp6-alert');
+ await confirmation.waitFor();
+ await showStep(this.page, 'Current Dart queries panel: cancel the query, to confirm');
+ await clickButton(confirmation, 'Cancel query');
+ await confirmation.waitFor({ state: 'detached' });
+ }
+
+ private entry(sqlQueryId: string): Locator {
+ return this.panel()
+ .locator('.work-entry')
+ .filter({ has: this.page.locator(`[data-tooltip$="SQL ID: ${sqlQueryId}"]`) });
+ }
+}
diff --git a/web-console/e2e-tests/component/workbench/overview.ts b/web-console/e2e-tests/component/workbench/overview.ts
index fc19a40812a3..74b34054ed14 100644
--- a/web-console/e2e-tests/component/workbench/overview.ts
+++ b/web-console/e2e-tests/component/workbench/overview.ts
@@ -16,43 +16,181 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
+import type { Locator, Page, Request } from '@playwright/test';
-import { clickButton, setQueryInput } from '../../util/playwright';
+import { expect } from '../../util/fixtures';
+import { clickButton, clickMenuItem, openView, setQueryInput } from '../../util/playwright';
+import { showStep } from '../../util/steps';
import { extractTable } from '../../util/table';
/**
- * Represents the workbench tab.
+ * How to run a query.
+ */
+export interface QueryOptions {
+ /** The engine to pick in the view's engine menu, by its name there (like "SQL (Dart)"), rather than "Auto" */
+ readonly engine?: string;
+}
+
+/**
+ * What the view sent to run a query.
+ */
+export interface SubmittedQuery {
+ /** The engine of the query's context, which the view sets for the engine picked (like "msq-dart") */
+ readonly engine?: string;
+ /** The ID that the view gives the query (SQL queries, natively or with Dart), to cancel it */
+ readonly sqlQueryId?: string;
+}
+
+/**
+ * Represents the Query view (the workbench).
*/
export class WorkbenchOverview {
- private readonly page: playwright.Page;
- private readonly baseUrl: string;
+ private readonly page: Page;
+
+ /** What the view sent to run the last query run (or started) */
+ lastSubmittedQuery: SubmittedQuery | undefined;
- constructor(page: playwright.Page, unifiedConsoleUrl: string) {
+ constructor(page: Page) {
this.page = page;
- this.baseUrl = unifiedConsoleUrl + '#workbench';
}
- async runQuery(query: string): Promise {
- await this.page.goto(this.baseUrl);
- await this.page.reload({ waitUntil: 'networkidle' });
+ /**
+ * Runs a query and returns its results (as text, one array of cells per row). Throws if the query fails.
+ */
+ async runQuery(query: string, options: QueryOptions = {}): Promise {
+ const results = this.page.locator('.result-table-pane');
+ await this.run(query, options, results);
+ await showStep(this.page, 'Query view: the query results');
+ return await extractTable(results);
+ }
+
+ /**
+ * Runs an ingestion query (INSERT or REPLACE, run as an MSQ task) and returns what the view says when it is done
+ * (like "39,244 rows inserted into ..."). Throws if the query fails.
+ */
+ async runIngestQuery(query: string, options: QueryOptions = {}): Promise {
+ const ingestSuccess = this.page.locator('.ingest-success-pane');
+ await this.run(query, options, ingestSuccess);
+ await showStep(this.page, 'Query view: the data ingested');
+ return await ingestSuccess.innerText();
+ }
+
+ /**
+ * Starts a query, without waiting for it to finish, and returns what the view sent to run it.
+ */
+ async startQuery(query: string, options: QueryOptions = {}): Promise {
+ await this.submit(query, options);
+ await showStep(this.page, 'Query view: the query running');
+ return this.lastSubmittedQuery!;
+ }
+
+ private async run(query: string, options: QueryOptions, done: Locator): Promise {
+ await this.submit(query, options);
+
+ const error = this.page.locator('.execution-error-pane');
+ await done.or(error).waitFor({ timeout: QUERY_TIMEOUT });
+ if (await error.isVisible()) {
+ await showStep(this.page, 'Query view: the query failed');
+ throw new Error(`Query failed: ${await error.innerText()}`);
+ }
+ }
+
+ private async submit(query: string, options: QueryOptions): Promise {
+ await openView(this.page, 'workbench');
await setQueryInput(this.page, query);
+ if (options.engine) await this.pickEngine(options.engine);
+ // The view shows the result of the last query until it sends the new one (even after a reload)
+ const queryRequest = this.page.waitForRequest(request => isQueryRequest(request, query), {
+ timeout: QUERY_TIMEOUT,
+ });
await clickButton(this.page, 'Run');
- const results = this.page.locator('div.result-table-pane');
- const capacityAlert = this.page.locator('.alert-dialog').filter({
- hasText: 'The cluster does not currently have enough available task slots',
- });
- const first = await Promise.race([
- results.waitFor({ timeout: 4 * 60 * 1000 }).then(() => 'results'),
- capacityAlert.waitFor({ timeout: 4 * 60 * 1000 }).then(() => 'capacityAlert'),
+ const capacityAlert = getCapacityAlert(this.page);
+ await Promise.race([
+ queryRequest,
+ capacityAlert.waitFor({ timeout: QUERY_TIMEOUT }).catch(() => {}),
]);
- if (first === 'capacityAlert') {
- await capacityAlert.getByRole('button', { name: 'Run it anyway' }).click();
- await results.waitFor({ timeout: 4 * 60 * 1000 });
+ if (await capacityAlert.isVisible()) {
+ await runAnyway(this.page, capacityAlert);
}
- return await extractTable(this.page, 'div.result-table-pane div.ct-tr-group', 'div.ct-td');
+ const context = (await queryRequest).postDataJSON()?.context ?? {};
+ this.lastSubmittedQuery = { engine: context.engine, sqlQueryId: context.sqlQueryId };
+ }
+
+ private async pickEngine(engine: string): Promise {
+ const engineButton = this.page.getByRole('button', { name: /^Engine: / });
+ await engineButton.click();
+ await clickMenuItem(this.page, engine);
+ // The menu stays open (to pick other settings in it)
+ await this.page.keyboard.press('Escape');
+ await expect(engineButton).toHaveText(`Engine: ${engine}`);
+ await showStep(this.page, `Query view: engine ${engine} picked`);
+ }
+
+ /**
+ * Runs a query, cancels it once it is running and returns the status of the cancel request.
+ */
+ async cancelQuery(query: string): Promise {
+ await openView(this.page, 'workbench');
+
+ await setQueryInput(this.page, query);
+
+ const queryRequest = this.page.waitForRequest(request => isQueryRequest(request, query));
+ await clickButton(this.page, 'Run');
+ await queryRequest;
+ await showStep(this.page, 'Query view: the query running');
+
+ const cancelResponse = this.page.waitForResponse(
+ response => response.url().includes('druid/v2') && response.request().method() === 'DELETE',
+ );
+ await this.page.locator('.cancel-label', { hasText: 'Cancel query' }).click();
+
+ const status = (await cancelResponse).status();
+ await showStep(this.page, 'Query view: the query canceled');
+ return status;
+ }
+}
+
+const QUERY_TIMEOUT = 4 * 60 * 1000;
+
+/**
+ * Whether the request runs the query. Matched on the query (its first line), as the view sends other SQL (like the one
+ * of its column tree) as it loads.
+ */
+function isQueryRequest(request: Request, query: string): boolean {
+ return (
+ request.url().includes('druid/v2') &&
+ request.method() === 'POST' &&
+ (request.postData() ?? '').includes(JSON.stringify(query.split('\n')[0]).slice(1, -1))
+ );
+}
+
+/**
+ * The alert that the console shows before running a query with the MSQ task engine when the cluster lacks the task
+ * slots for it.
+ */
+function getCapacityAlert(page: Page): Locator {
+ return page.locator('.alert-dialog').filter({
+ hasText: 'The cluster does not currently have enough available task slots',
+ });
+}
+
+async function runAnyway(page: Page, capacityAlert: Locator): Promise {
+ await showStep(page, 'Not enough task slots, to confirm');
+ await clickButton(capacityAlert, 'Run it anyway');
+}
+
+/**
+ * Waits for `done`, after starting an MSQ task (like the SQL data loader does), running it anyway if the console
+ * asks because the cluster lacks the task slots for it.
+ */
+export async function waitForTaskSlots(page: Page, done: Locator): Promise {
+ const capacityAlert = getCapacityAlert(page);
+ await done.or(capacityAlert).waitFor({ timeout: QUERY_TIMEOUT });
+ if (await capacityAlert.isVisible()) {
+ await runAnyway(page, capacityAlert);
+ await done.waitFor({ timeout: QUERY_TIMEOUT });
}
}
diff --git a/web-console/e2e-tests/dart.spec.ts b/web-console/e2e-tests/dart.spec.ts
new file mode 100644
index 000000000000..4048d8b2b844
--- /dev/null
+++ b/web-console/e2e-tests/dart.spec.ts
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext } from '@playwright/test';
+import { T } from 'druid-query-toolkit';
+
+import { CurrentDartPanel } from './component/workbench/dart-panel';
+import { WorkbenchOverview } from './component/workbench/overview';
+import { readTutorialIngestionSpec, runTask } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDartQueryState, getDatasourceSegments } from './util/sql';
+import { showStep } from './util/steps';
+
+const DART = { engine: 'SQL (Dart)' };
+
+// Dart runs queries with the multi-stage query engine on the Broker and the Historicals (rather than in tasks), so it
+// needs to be enabled (druid.msq.dart.enabled=true), as it is on the embedded clusters
+test.describe('Dart', () => {
+ test('Runs a query with Dart, which the current Dart queries panel shows with its details', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('dart');
+ await loadData(request, datasourceName);
+
+ const workbench = new WorkbenchOverview(page);
+ const results = await workbench.runQuery(
+ `SELECT channel, CAST(COUNT(*) AS VARCHAR) AS "count"
+FROM ${T(datasourceName)}
+GROUP BY 1
+ORDER BY COUNT(*) DESC
+LIMIT 2`,
+ DART,
+ );
+ expect(results).toEqual([
+ ['#en.wikipedia', '11549'],
+ ['#vi.wikipedia', '9747'],
+ ]);
+ const { engine, sqlQueryId } = workbench.lastSubmittedQuery!;
+ expect(engine).toBe('msq-dart');
+ expect(await getDartQueryState(request, sqlQueryId!)).toBe('SUCCESS');
+
+ const dartPanel = new CurrentDartPanel(page);
+ await dartPanel.show();
+ await expect
+ .poll(() => dartPanel.getQueryState(sqlQueryId!), CLUSTER_STATE_POLL)
+ .toBe('SUCCESS');
+ await showStep(page, 'Current Dart queries panel: the query SUCCESS');
+
+ // The details are the report of the query, with its stages
+ const details = await dartPanel.showDetails(sqlQueryId!);
+ await expect(details.locator('.execution-stages-pane .ct-tbody .ct-tr').first()).toBeVisible();
+ });
+
+ test('Cancels a running Dart query from the current Dart queries panel', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('dart-cancel');
+ await loadData(request, datasourceName);
+
+ // Sleeps for each row (10 ms, so minutes for all of them). The sleep depends on a column, as a constant one would
+ // be done once, when the query is planned.
+ const workbench = new WorkbenchOverview(page);
+ const { sqlQueryId } = await workbench.startQuery(
+ `SELECT COUNT(*)
+FROM ${T(datasourceName)}
+WHERE sleep(CASE WHEN "added" > 0 THEN 0.01 ELSE 0.02 END) IS NULL`,
+ DART,
+ );
+
+ const dartPanel = new CurrentDartPanel(page);
+ await dartPanel.show();
+ await expect
+ .poll(() => dartPanel.getQueryState(sqlQueryId!), CLUSTER_STATE_POLL)
+ .toBe('RUNNING');
+ await showStep(page, 'Current Dart queries panel: the query RUNNING');
+
+ await dartPanel.cancelQuery(sqlQueryId!);
+ await expect
+ .poll(() => dartPanel.getQueryState(sqlQueryId!), CLUSTER_STATE_POLL)
+ .toBe('CANCELED');
+ await showStep(page, 'Current Dart queries panel: the query CANCELED');
+ expect(await getDartQueryState(request, sqlQueryId!)).toBe('CANCELED');
+ });
+});
+
+async function loadData(request: APIRequestContext, datasourceName: string) {
+ const ingestionSpec = readTutorialIngestionSpec('wikipedia-index.json');
+ ingestionSpec.spec.dataSchema.dataSource = datasourceName;
+ await runTask(request, ingestionSpec);
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: 39244 });
+}
diff --git a/web-console/e2e-tests/datasource-actions.spec.ts b/web-console/e2e-tests/datasource-actions.spec.ts
new file mode 100644
index 000000000000..a6c175007c32
--- /dev/null
+++ b/web-console/e2e-tests/datasource-actions.spec.ts
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext } from '@playwright/test';
+
+import type { Datasource } from './component/datasources/overview';
+import { DatasourcesOverview } from './component/datasources/overview';
+import { readTutorialIngestionSpec, runTask } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments, getTaskStatuses } from './util/sql';
+
+const NUM_ROWS = 39244;
+
+test.describe('Datasource actions', () => {
+ test('Marks the segments unused, used again, then deletes them with a kill task', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('datasource-actions');
+ await loadData(request, datasourceName);
+ const datasourcesOverview = new DatasourcesOverview(page);
+
+ // Mark unused: the datasource is only shown with "Show unused", as unused
+ await datasourcesOverview.runAction(datasourceName, 'Mark as unused all segments');
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 0, numAvailableSegments: 0, numRows: null });
+ await expectInView(datasourcesOverview, datasourceName, false, []);
+ await expectInView(datasourcesOverview, datasourceName, true, [
+ expect.objectContaining({
+ name: datasourceName,
+ availability: expect.stringContaining('Unused'),
+ }),
+ ]);
+
+ // Mark used: the data is back
+ await datasourcesOverview.runAction(datasourceName, 'Mark as used all segments', {
+ showUnused: true,
+ });
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: NUM_ROWS });
+ await expectInView(datasourcesOverview, datasourceName, false, [
+ expect.objectContaining({
+ name: datasourceName,
+ availability: expect.stringContaining('Fully available (1 segment)'),
+ totalRows: NUM_ROWS,
+ }),
+ ]);
+
+ // Mark unused again, and delete the unused segments with a kill task: the datasource is gone
+ await datasourcesOverview.runAction(datasourceName, 'Mark as unused all segments');
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 0, numAvailableSegments: 0, numRows: null });
+ await expectInView(datasourcesOverview, datasourceName, true, [
+ expect.objectContaining({ name: datasourceName }),
+ ]);
+ await datasourcesOverview.runAction(datasourceName, 'Delete segments (issue kill task)', {
+ showUnused: true,
+ });
+ await expect
+ .poll(() => getTaskStatuses(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual(['SUCCESS', 'SUCCESS']); // the index task, and the kill task
+ await expectInView(datasourcesOverview, datasourceName, true, []);
+ });
+});
+
+async function loadData(request: APIRequestContext, datasourceName: string) {
+ const ingestionSpec = readTutorialIngestionSpec('wikipedia-index.json');
+ ingestionSpec.spec.dataSchema.dataSource = datasourceName;
+ await runTask(request, ingestionSpec);
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: NUM_ROWS });
+}
+
+async function expectInView(
+ datasourcesOverview: DatasourcesOverview,
+ datasourceName: string,
+ showUnused: boolean,
+ expected: (Datasource | ReturnType)[],
+) {
+ // Retried in case the view is a step behind the cluster state polled before
+ await expect(async () => {
+ expect(await datasourcesOverview.getDatasources(datasourceName, { showUnused })).toEqual(
+ expected,
+ );
+ }).toPass({ timeout: 30 * 1000 });
+}
diff --git a/web-console/e2e-tests/input-formats.spec.ts b/web-console/e2e-tests/input-formats.spec.ts
new file mode 100644
index 000000000000..7ff635b9c55d
--- /dev/null
+++ b/web-console/e2e-tests/input-formats.spec.ts
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as path from 'path';
+
+import { loadData } from './component/load-data/data-loader';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments, getTaskStatuses } from './util/sql';
+
+// The data/ dir of embedded-tests' test resources, which has the same 10 rows of Wikipedia edits (over 2 days) in each
+// format, in 3 files. InputFormatsWebConsoleTest (in embedded-tests) sets it, on a cluster with the extensions that
+// read Avro, Parquet and ORC.
+const DATA_DIR = process.env['DRUID_E2E_TEST_DATA_DIR'];
+
+const FORMATS = [
+ { inputFormat: 'csv', dir: 'csv', file: 'wikipedia_index_data*.csv', text: true },
+ { inputFormat: 'tsv', dir: 'tsv', file: 'wikipedia_index_data*.tsv', text: true },
+ { inputFormat: 'parquet', dir: 'parquet', file: 'wikipedia_index_data*.parquet', text: false },
+ { inputFormat: 'orc', dir: 'orc', file: 'wikipedia_index_data*.orc', text: false },
+ { inputFormat: 'avro_ocf', dir: 'avro', file: 'wikipedia_index_data*.avro', text: false },
+];
+
+test.describe('Input formats', () => {
+ test.skip(
+ !DATA_DIR,
+ 'needs DRUID_E2E_TEST_DATA_DIR (run by InputFormatsWebConsoleTest in embedded-tests)',
+ );
+
+ for (const format of FORMATS) {
+ test(`Loads ${format.inputFormat} data`, async ({ page, request, newDatasourceName }) => {
+ const datasourceName = newDatasourceName(`input-format-${format.inputFormat}`);
+ await loadData(page, {
+ connector: {
+ type: 'local',
+ baseDirectory: path.join(DATA_DIR!, format.dir),
+ fileFilter: format.file,
+ },
+ validateConnect: lines => {
+ // The text formats show their lines (a header and the rows of each file), the others their bytes
+ if (format.text) {
+ expect(lines).toHaveLength(13);
+ expect(lines[0]).toMatch(/^timestamp.page.language.user/);
+ } else {
+ expect(lines.length).toBeGreaterThan(0);
+ }
+ },
+ validateParseData: ({ inputFormat, columns }) => {
+ // Picked by the data loader, from the sampled data
+ expect(inputFormat).toBe(format.inputFormat);
+ expect(columns).toEqual(
+ expect.arrayContaining(['timestamp', 'page', 'user', 'added', 'deleted', 'delta']),
+ );
+ },
+ rollup: false,
+ segmentGranularity: 'day',
+ datasourceName,
+ });
+
+ await expect
+ .poll(() => getTaskStatuses(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual(['SUCCESS']);
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 2, numAvailableSegments: 2, numRows: 10 });
+ });
+ }
+});
diff --git a/web-console/e2e-tests/jdbc-lookup.spec.ts b/web-console/e2e-tests/jdbc-lookup.spec.ts
new file mode 100644
index 000000000000..db24845ca605
--- /dev/null
+++ b/web-console/e2e-tests/jdbc-lookup.spec.ts
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext } from '@playwright/test';
+import { L } from 'druid-query-toolkit';
+
+import { LookupsOverview } from './component/lookups/overview';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, querySql } from './util/sql';
+
+// The JDBC URI of a database with a table of country codes and names, which the cluster can read. JdbcLookupWebConsoleTest
+// (in embedded-tests) sets it up, in the cluster's (Derby) metadata store.
+const CONNECT_URI = process.env['DRUID_E2E_TEST_LOOKUP_CONNECT_URI'];
+const TABLE = process.env['DRUID_E2E_TEST_LOOKUP_TABLE'];
+
+test.describe('JDBC lookup', () => {
+ test.skip(
+ !CONNECT_URI || !TABLE,
+ 'needs DRUID_E2E_TEST_LOOKUP_CONNECT_URI and DRUID_E2E_TEST_LOOKUP_TABLE (run by JdbcLookupWebConsoleTest in embedded-tests)',
+ );
+
+ test('Adds a JDBC lookup and queries it', async ({ page, request }) => {
+ // Lookups are not deleted with the datasources, so the name is unique
+ const lookupName = `country_names_${Date.now()}`;
+ const lookupsOverview = new LookupsOverview(page);
+ await lookupsOverview.addJdbcLookup({
+ name: lookupName,
+ connectUri: CONNECT_URI!,
+ table: TABLE!,
+ keyColumn: 'country_code',
+ valueColumn: 'country_name',
+ timestampColumn: 'created_date',
+ });
+
+ expect(await lookupsOverview.getLookups(lookupName)).toEqual([
+ { name: lookupName, tier: '__default', type: 'cachedNamespace' },
+ ]);
+
+ // Once the Broker loads the lookup (from the table), queries can use it
+ await expect
+ .poll(() => lookUp(request, lookupName, ['PR', 'AU']), CLUSTER_STATE_POLL)
+ .toEqual(['Puerto Rico', 'Australia']);
+ });
+});
+
+/**
+ * The values of the keys in the lookup, or null while the lookup can not be queried (yet).
+ */
+async function lookUp(
+ request: APIRequestContext,
+ lookupName: string,
+ keys: string[],
+): Promise {
+ try {
+ const [row] = await querySql(
+ request,
+ `SELECT ${keys.map((key, i) => `LOOKUP(${L(key)}, ${L(lookupName)}) AS "v${i}"`).join(', ')}`,
+ );
+ return keys.map((_, i) => row[`v${i}`]);
+ } catch {
+ return null;
+ }
+}
diff --git a/web-console/e2e-tests/kafka-ingestion.spec.ts b/web-console/e2e-tests/kafka-ingestion.spec.ts
new file mode 100644
index 000000000000..d1ef502d3e77
--- /dev/null
+++ b/web-console/e2e-tests/kafka-ingestion.spec.ts
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext, Page } from '@playwright/test';
+
+import { loadData } from './component/load-data/data-loader';
+import type { SupervisorAction } from './component/supervisors/overview';
+import { SupervisorsOverview } from './component/supervisors/overview';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getQueryableRowCount, getSupervisorState } from './util/sql';
+
+// A Kafka topic with the 39,244 Wikipedia edits of wikiticker-2015-09-12-sampled.json.gz (of
+// examples/quickstart/tutorial), one per message, in one partition. KafkaWebConsoleTest (in embedded-tests) sets it
+// up, with a Kafka container.
+const BOOTSTRAP_SERVERS = process.env['DRUID_E2E_TEST_KAFKA_BOOTSTRAP_SERVERS'];
+const TOPIC = process.env['DRUID_E2E_TEST_KAFKA_TOPIC'];
+
+const NUM_ROWS = 39244;
+
+// The workflow in this test is based on the Kafka tutorial:
+// https://druid.apache.org/docs/latest/tutorials/tutorial-kafka
+test.describe('Kafka ingestion', () => {
+ test.skip(
+ !BOOTSTRAP_SERVERS || !TOPIC,
+ 'needs DRUID_E2E_TEST_KAFKA_BOOTSTRAP_SERVERS and DRUID_E2E_TEST_KAFKA_TOPIC (run by KafkaWebConsoleTest in embedded-tests)',
+ );
+
+ test('Loads data from Kafka, then suspends, resumes and terminates the supervisor', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ // The supervisor is named after the datasource
+ const datasourceName = newDatasourceName('load-data-from-kafka');
+ await loadData(page, {
+ connector: { type: 'kafka', bootstrapServers: BOOTSTRAP_SERVERS!, topic: TOPIC! },
+ validateConnect: lines => {
+ expect(lines.length).toBeGreaterThan(0);
+ // Each message is shown after its Kafka metadata ([ Kafka timestamp: ... ])
+ expect(lines[0]).toContain('{"time":"2015-09-12T00:46:58.771Z","channel":"#en.wikipedia",');
+ },
+ rollup: false,
+ segmentGranularity: 'day',
+ datasourceName,
+ });
+
+ await expectSupervisorState(request, datasourceName, 'RUNNING');
+ await expect
+ .poll(() => getQueryableRowCount(request, datasourceName), CLUSTER_STATE_POLL)
+ .toBe(NUM_ROWS);
+
+ const supervisorsOverview = new SupervisorsOverview(page);
+ await expectSupervisorInView(supervisorsOverview, datasourceName, 'RUNNING');
+
+ await runSupervisorAction(page, request, datasourceName, 'Suspend', 'SUSPENDED');
+ await runSupervisorAction(page, request, datasourceName, 'Resume', 'RUNNING');
+
+ await supervisorsOverview.runAction(datasourceName, 'Terminate');
+ await expectSupervisorState(request, datasourceName, null);
+ expect(await supervisorsOverview.getSupervisors(datasourceName)).toEqual([]);
+
+ // Terminating the supervisor publishes what its tasks ingested
+ await expect
+ .poll(() => getQueryableRowCount(request, datasourceName), CLUSTER_STATE_POLL)
+ .toBe(NUM_ROWS);
+ });
+});
+
+async function runSupervisorAction(
+ page: Page,
+ request: APIRequestContext,
+ supervisorId: string,
+ action: SupervisorAction,
+ expectedState: string,
+) {
+ const supervisorsOverview = new SupervisorsOverview(page);
+ await supervisorsOverview.runAction(supervisorId, action);
+ await expectSupervisorState(request, supervisorId, expectedState);
+ await expectSupervisorInView(supervisorsOverview, supervisorId, expectedState);
+}
+
+async function expectSupervisorState(
+ request: APIRequestContext,
+ supervisorId: string,
+ state: string | null,
+) {
+ await expect
+ .poll(() => getSupervisorState(request, supervisorId), CLUSTER_STATE_POLL)
+ .toBe(state);
+}
+
+async function expectSupervisorInView(
+ supervisorsOverview: SupervisorsOverview,
+ supervisorId: string,
+ status: string,
+) {
+ // Retried in case the view is a step behind the cluster state polled above
+ await expect(async () => {
+ const supervisors = await supervisorsOverview.getSupervisors(supervisorId);
+ expect(supervisors).toHaveLength(1);
+ expect(supervisors[0].status).toContain(status);
+ }).toPass({ timeout: 30 * 1000 });
+}
diff --git a/web-console/e2e-tests/multi-stage-query.spec.ts b/web-console/e2e-tests/multi-stage-query.spec.ts
index 4647764266ac..c0ec3cadb6db 100644
--- a/web-console/e2e-tests/multi-stage-query.spec.ts
+++ b/web-console/e2e-tests/multi-stage-query.spec.ts
@@ -16,38 +16,15 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
-
import { WorkbenchOverview } from './component/workbench/overview';
-import { saveScreenshotIfError } from './util/debug';
-import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, UNIFIED_CONSOLE_URL } from './util/druid';
-import { createBrowser, createPage } from './util/playwright';
-import { waitTillWebConsoleReady } from './util/setup';
-
-jest.setTimeout(5 * 60 * 1000);
-
-describe('Multi-stage query', () => {
- let browser: playwright.Browser;
- let page: playwright.Page;
-
- beforeAll(async () => {
- await waitTillWebConsoleReady();
- browser = await createBrowser();
- });
-
- beforeEach(async () => {
- page = await createPage(browser);
- });
-
- afterAll(async () => {
- await browser.close();
- });
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
+import { expect, test } from './util/fixtures';
- it('runs a query that reads external data', async () => {
- const workbench = new WorkbenchOverview(page, UNIFIED_CONSOLE_URL);
+test.describe('Multi-stage query', () => {
+ test('runs a query that reads external data', async ({ page }) => {
+ const workbench = new WorkbenchOverview(page);
- await saveScreenshotIfError('multi-stage-query', page, async () => {
- const results = await workbench.runQuery(`WITH ext AS (SELECT *
+ const results = await workbench.runQuery(`WITH ext AS (SELECT *
FROM TABLE(
EXTERN(
'{"type":"local","filter":"wikiticker-2015-09-12-sampled.json.gz","baseDir":${JSON.stringify(
@@ -63,10 +40,8 @@ FROM ext
GROUP BY 1
ORDER BY COUNT(*) DESC
LIMIT 10`);
- expect(results).toBeDefined();
- expect(results.length).toBe(10);
- expect(results[0]).toStrictEqual(['#en.wikipedia', '11549']);
- expect(results[1]).toStrictEqual(['#vi.wikipedia', '9747']);
- });
+ expect(results.length).toBe(10);
+ expect(results[0]).toStrictEqual(['#en.wikipedia', '11549']);
+ expect(results[1]).toStrictEqual(['#vi.wikipedia', '9747']);
});
});
diff --git a/web-console/e2e-tests/reindexing.spec.ts b/web-console/e2e-tests/reindexing.spec.ts
index caa419e98459..842366b2a1a0 100644
--- a/web-console/e2e-tests/reindexing.spec.ts
+++ b/web-console/e2e-tests/reindexing.spec.ts
@@ -16,99 +16,55 @@
* limitations under the License.
*/
-import path from 'path';
-import type * as playwright from 'playwright-chromium';
-
-import { DatasourcesOverview } from './component/datasources/overview';
-import { TasksOverview } from './component/ingestion/overview';
-import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema';
-import {
- PartitionConfig,
- RangePartitionsSpec,
- SegmentGranularity,
-} from './component/load-data/config/partition';
-import { PublishConfig } from './component/load-data/config/publish';
-import { ReindexDataConnector } from './component/load-data/data-connector/reindex';
-import { DataLoader } from './component/load-data/data-loader';
-import { saveScreenshotIfError } from './util/debug';
-import {
- DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
- runIndexTask,
- UNIFIED_CONSOLE_URL,
-} from './util/druid';
-import { createBrowser, createPage } from './util/playwright';
-import { retryIfJestAssertionError } from './util/retry';
-import { waitTillWebConsoleReady } from './util/setup';
-
-jest.setTimeout(5 * 60 * 1000);
-
-describe('Reindexing from Druid', () => {
- let browser: playwright.Browser;
- let page: playwright.Page;
-
- beforeAll(async () => {
- await waitTillWebConsoleReady();
- browser = await createBrowser();
- });
-
- beforeEach(async () => {
- page = await createPage(browser);
- });
-
- afterAll(async () => {
- await browser.close();
- });
-
- it('Reindex datasource from dynamic to range partitions', async () => {
- const testName = 'reindex-dynamic-to-range';
- const datasourceName = testName + new Date().toISOString();
- const interval = '2015-09-12/2015-09-13';
- const dataConnector = new ReindexDataConnector(page, {
- datasourceName,
- interval,
- });
- const configureSchemaConfig = new ConfigureSchemaConfig({ rollup: false });
- const partitionConfig = new PartitionConfig({
- segmentGranularity: SegmentGranularity.DAY,
- timeIntervals: null,
- partitionsSpec: new RangePartitionsSpec({
+import type { APIRequestContext } from '@playwright/test';
+
+import { loadData } from './component/load-data/data-loader';
+import { readTutorialIngestionSpec, runTask } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments, getTaskStatuses } from './util/sql';
+
+test.describe('Reindexing from Druid', () => {
+ test('Reindex datasource from dynamic to range partitions', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('reindex-dynamic-to-range');
+ await loadInitialData(request, datasourceName);
+
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: 39244 });
+
+ await loadData(page, {
+ connector: { type: 'reindex', datasourceName, interval: '2015-09-12/2015-09-13' },
+ validateConnect: validateConnectLocalData,
+ rollup: false,
+ segmentGranularity: 'day',
+ partitionsSpec: {
+ type: 'range',
partitionDimensions: ['channel'],
targetRowsPerSegment: 10_000,
maxRowsPerSegment: null,
- }),
- });
- const publishConfig = new PublishConfig({ datasourceName: datasourceName });
-
- const dataLoader = new DataLoader({
- page: page,
- unifiedConsoleUrl: UNIFIED_CONSOLE_URL,
- connector: dataConnector,
- connectValidator: validateConnectLocalData,
- configureSchemaConfig: configureSchemaConfig,
- partitionConfig: partitionConfig,
- publishConfig: publishConfig,
- });
-
- loadInitialData(datasourceName);
-
- await saveScreenshotIfError(testName, page, async () => {
- const numInitialSegment = 1;
- await validateDatasourceStatus(page, datasourceName, numInitialSegment);
-
- await dataLoader.load();
- await validateTaskStatus(page, datasourceName);
-
- const numReindexedSegment = 4; // 39k rows into segments of ~10k rows
- await validateDatasourceStatus(page, datasourceName, numReindexedSegment);
+ },
+ datasourceName,
});
+ // The initial load and the reindexing
+ await expect
+ .poll(() => getTaskStatuses(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual(['SUCCESS', 'SUCCESS']);
+
+ // 39k rows into segments of ~10k rows
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 4, numAvailableSegments: 4, numRows: 39244 });
});
});
-function loadInitialData(datasourceName: string) {
- const ingestionSpec = path.join(DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, 'wikipedia-index.json');
- const setDatasourceName = `s/wikipedia/${datasourceName}/`;
- const sedCommands = [setDatasourceName];
- runIndexTask(ingestionSpec, sedCommands);
+async function loadInitialData(request: APIRequestContext, datasourceName: string) {
+ const ingestionSpec = readTutorialIngestionSpec('wikipedia-index.json');
+ ingestionSpec.spec.dataSchema.dataSource = datasourceName;
+ await runTask(request, ingestionSpec);
}
function validateConnectLocalData(lines: string[]) {
@@ -152,31 +108,3 @@ function validateConnectLocalData(lines: string[]) {
'}]',
);
}
-
-async function validateTaskStatus(page: playwright.Page, datasourceName: string) {
- const tasksOverview = new TasksOverview(page, UNIFIED_CONSOLE_URL);
-
- await retryIfJestAssertionError(async () => {
- const tasks = await tasksOverview.getTasks();
- const task = tasks.find(t => t.datasource === datasourceName);
- expect(task).toBeDefined();
- expect(task!.status).toMatch('SUCCESS');
- });
-}
-
-async function validateDatasourceStatus(
- page: playwright.Page,
- datasourceName: string,
- expectedNumSegment: number,
-) {
- const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
- const numSegmentString = `${expectedNumSegment} segment` + (expectedNumSegment !== 1 ? 's' : '');
-
- await retryIfJestAssertionError(async () => {
- const datasources = await datasourcesOverview.getDatasources();
- const datasource = datasources.find(t => t.name === datasourceName);
- expect(datasource).toBeDefined();
- expect(datasource!.availability).toMatch(`Fully available (${numSegmentString})`);
- expect(datasource!.totalRows).toBe(39244);
- });
-}
diff --git a/web-console/e2e-tests/retention-rules.spec.ts b/web-console/e2e-tests/retention-rules.spec.ts
new file mode 100644
index 000000000000..e77d928e016f
--- /dev/null
+++ b/web-console/e2e-tests/retention-rules.spec.ts
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext } from '@playwright/test';
+
+import { DatasourcesOverview } from './component/datasources/overview';
+import { getRetentionRules, readTutorialIngestionSpec, runTask } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments } from './util/sql';
+
+test.describe('Retention rules', () => {
+ test('Sets a loadForever rule, then replaces it with a dropForever rule, which drops the data', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const datasourceName = newDatasourceName('retention-rules');
+ await loadData(request, datasourceName);
+ const datasourcesOverview = new DatasourcesOverview(page);
+
+ // loadForever (as "New rule" makes it, with 2 replicas): the data stays loaded
+ await datasourcesOverview.setRetentionRules(datasourceName, ['loadForever'], 'Load it all');
+ expect(await getRetentionRules(request, datasourceName)).toEqual([
+ expect.objectContaining({ type: 'loadForever', tieredReplicants: { _default_tier: 2 } }),
+ ]);
+ await expectRetentionInView(datasourcesOverview, datasourceName, 'loadForever(2x)');
+ expect(await getUsedAndAvailableSegments(request, datasourceName)).toEqual([1, 1]);
+
+ // dropForever (replacing loadForever): the Coordinator drops the segment, marking it unused, so the datasource is
+ // only shown with "Show unused"
+ await datasourcesOverview.setRetentionRules(datasourceName, ['dropForever'], 'Drop it all');
+ expect(await getRetentionRules(request, datasourceName)).toEqual([{ type: 'dropForever' }]);
+ await expect
+ .poll(() => getUsedAndAvailableSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual([0, 0]);
+ await expect(async () => {
+ expect(await datasourcesOverview.getDatasources(datasourceName)).toEqual([]);
+ expect(
+ await datasourcesOverview.getDatasources(datasourceName, { showUnused: true }),
+ ).toEqual([expect.objectContaining({ availability: expect.stringContaining('Unused') })]);
+ }).toPass({ timeout: 30 * 1000 });
+ });
+});
+
+async function loadData(request: APIRequestContext, datasourceName: string) {
+ const ingestionSpec = readTutorialIngestionSpec('wikipedia-index.json');
+ ingestionSpec.spec.dataSchema.dataSource = datasourceName;
+ await runTask(request, ingestionSpec);
+ await expect
+ .poll(() => getUsedAndAvailableSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual([1, 1]);
+}
+
+async function getUsedAndAvailableSegments(request: APIRequestContext, datasourceName: string) {
+ const { numSegments, numAvailableSegments } = await getDatasourceSegments(
+ request,
+ datasourceName,
+ );
+ return [numSegments, numAvailableSegments];
+}
+
+async function expectRetentionInView(
+ datasourcesOverview: DatasourcesOverview,
+ datasourceName: string,
+ retention: string,
+) {
+ // Retried in case the view is a step behind the cluster state polled before
+ await expect(async () => {
+ const [datasource] = await datasourcesOverview.getDatasources(datasourceName);
+ expect(datasource.retention).toContain(retention);
+ }).toPass({ timeout: 30 * 1000 });
+}
diff --git a/web-console/e2e-tests/s3-ingestion.spec.ts b/web-console/e2e-tests/s3-ingestion.spec.ts
new file mode 100644
index 000000000000..2da583527fed
--- /dev/null
+++ b/web-console/e2e-tests/s3-ingestion.spec.ts
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { loadData } from './component/load-data/data-loader';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments, getTaskStatuses } from './util/sql';
+
+// The S3 URI of wikiticker-2015-09-12-sampled.json.gz (of examples/quickstart/tutorial), on an S3 that the cluster is
+// configured for. S3WebConsoleTest (in embedded-tests) sets it up, with an S3 container.
+const S3_URI = process.env['DRUID_E2E_TEST_S3_URI'];
+
+test.describe('S3 ingestion', () => {
+ test.skip(!S3_URI, 'needs DRUID_E2E_TEST_S3_URI (run by S3WebConsoleTest in embedded-tests)');
+
+ test('Loads data from S3', async ({ page, request, newDatasourceName }) => {
+ const datasourceName = newDatasourceName('load-data-from-s3');
+ await loadData(page, {
+ connector: { type: 's3', uris: [S3_URI!] },
+ validateConnect: lines => {
+ expect(lines.length).toBe(500);
+ expect(lines[0]).toMatch(/^\{"time":"2015-09-12T00:46:58.771Z","channel":"#en.wikipedia",/);
+ },
+ rollup: false,
+ segmentGranularity: 'day',
+ datasourceName,
+ });
+
+ await expect
+ .poll(() => getTaskStatuses(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual(['SUCCESS']);
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: 39244 });
+ });
+});
diff --git a/web-console/e2e-tests/sql-ingestion.spec.ts b/web-console/e2e-tests/sql-ingestion.spec.ts
new file mode 100644
index 000000000000..6a76a906dc94
--- /dev/null
+++ b/web-console/e2e-tests/sql-ingestion.spec.ts
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { T } from 'druid-query-toolkit';
+
+import { loadDataWithSql } from './component/load-data/sql-data-loader';
+import { WorkbenchOverview } from './component/workbench/overview';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getQueryableRowCount } from './util/sql';
+
+const DATA_FILE = 'wikiticker-2015-09-12-sampled.json.gz';
+const NUM_ROWS = 39244;
+const NUM_EN_WIKIPEDIA_ROWS = 11549;
+
+test.describe('SQL-based ingestion', () => {
+ test('Loads data through the SQL data loader', async ({ page, request, newDatasourceName }) => {
+ const datasourceName = newDatasourceName('sql-data-loader');
+ await loadDataWithSql(page, {
+ connector: {
+ type: 'local',
+ baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
+ fileFilter: DATA_FILE,
+ },
+ validateParseData: ({ inputFormat, columns }) => {
+ expect(inputFormat).toBe('json');
+ expect(columns).toEqual(expect.arrayContaining(['time', 'channel', 'page', 'added']));
+ },
+ datasourceName,
+ });
+
+ await expect
+ .poll(() => getQueryableRowCount(request, datasourceName), CLUSTER_STATE_POLL)
+ .toBe(NUM_ROWS);
+ });
+
+ test('Ingests with REPLACE in the Query view, then reindexes with REPLACE from the datasource', async ({
+ page,
+ request,
+ newDatasourceName,
+ }) => {
+ const workbench = new WorkbenchOverview(page);
+
+ const datasourceName = newDatasourceName('sql-replace');
+ const ingested = await workbench.runIngestQuery(`REPLACE INTO ${T(datasourceName)} OVERWRITE ALL
+SELECT
+ TIME_PARSE("time") AS __time,
+ channel,
+ page,
+ added
+FROM TABLE(
+ EXTERN(
+ '{"type":"local","filter":"${DATA_FILE}","baseDir":${JSON.stringify(
+ DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
+ )}}',
+ '{"type":"json"}'
+ )
+) EXTEND ("time" VARCHAR, channel VARCHAR, page VARCHAR, added BIGINT)
+PARTITIONED BY DAY`);
+ expect(ingested).toContain(`39,244 rows inserted into ${T(datasourceName)}`);
+ await expect
+ .poll(() => getQueryableRowCount(request, datasourceName), CLUSTER_STATE_POLL)
+ .toBe(NUM_ROWS);
+
+ // Reindex the rows of one channel into another datasource
+ const reindexedName = newDatasourceName('sql-reindex');
+ const reindexed = await workbench.runIngestQuery(`REPLACE INTO ${T(reindexedName)} OVERWRITE ALL
+SELECT *
+FROM ${T(datasourceName)}
+WHERE channel = '#en.wikipedia'
+PARTITIONED BY DAY`);
+ expect(reindexed).toContain(`11,549 rows inserted into ${T(reindexedName)}`);
+ await expect
+ .poll(() => getQueryableRowCount(request, reindexedName), CLUSTER_STATE_POLL)
+ .toBe(NUM_EN_WIKIPEDIA_ROWS);
+ });
+});
diff --git a/web-console/e2e-tests/tutorial-batch.spec.ts b/web-console/e2e-tests/tutorial-batch.spec.ts
index 6da01d57cc99..24855f7ed9d1 100644
--- a/web-console/e2e-tests/tutorial-batch.spec.ts
+++ b/web-console/e2e-tests/tutorial-batch.spec.ts
@@ -16,74 +16,46 @@
* limitations under the License.
*/
+import type { Page } from '@playwright/test';
import { T } from 'druid-query-toolkit';
-import type * as playwright from 'playwright-chromium';
import { DatasourcesOverview } from './component/datasources/overview';
import { TasksOverview } from './component/ingestion/overview';
-import { ConfigureSchemaConfig } from './component/load-data/config/configure-schema';
-import { ConfigureTimestampConfig } from './component/load-data/config/configure-timestamp';
-import { PartitionConfig, SegmentGranularity } from './component/load-data/config/partition';
-import { PublishConfig } from './component/load-data/config/publish';
-import { LocalFileDataConnector } from './component/load-data/data-connector/local-file';
-import { DataLoader } from './component/load-data/data-loader';
-import { QueryOverview } from './component/query/overview';
-import { saveScreenshotIfError } from './util/debug';
-import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, UNIFIED_CONSOLE_URL } from './util/druid';
-import { createBrowser, createPage } from './util/playwright';
-import { retryIfJestAssertionError, retryOnAnyError } from './util/retry';
-import { waitTillWebConsoleReady } from './util/setup';
-
-jest.setTimeout(5 * 60 * 1000);
+import { loadData } from './component/load-data/data-loader';
+import { WorkbenchOverview } from './component/workbench/overview';
+import { DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR } from './util/druid';
+import { expect, test } from './util/fixtures';
+import { CLUSTER_STATE_POLL, getDatasourceSegments, getTaskStatuses } from './util/sql';
const ALL_SORTS_OF_CHARS = '<>|!@#$%^&`\'".,:;\\*()[]{}Україна 한국 中国!?~';
-describe('Tutorial: Loading a file', () => {
- let browser: playwright.Browser;
- let page: playwright.Page;
-
- beforeAll(async () => {
- await waitTillWebConsoleReady();
- browser = await createBrowser();
- });
-
- beforeEach(async () => {
- page = await createPage(browser);
- });
-
- afterAll(async () => {
- await browser.close();
- });
-
- it('Loads data from local disk', async () => {
- const testName = 'load-data-from-local-disk';
- const datasourceName = testName + ALL_SORTS_OF_CHARS + new Date().toISOString();
- const dataLoader = new DataLoader({
- page: page,
- unifiedConsoleUrl: UNIFIED_CONSOLE_URL,
- connector: new LocalFileDataConnector(page, {
+test.describe('Tutorial: Loading a file', () => {
+ test('Loads data from local disk', async ({ page, request, newDatasourceName }) => {
+ const datasourceName = newDatasourceName('load-data-from-local-disk' + ALL_SORTS_OF_CHARS);
+ await loadData(page, {
+ connector: {
+ type: 'local',
baseDirectory: DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR,
fileFilter: 'wikiticker-2015-09-12-sampled.json.gz',
- }),
- connectValidator: validateConnectLocalData,
- configureTimestampConfig: new ConfigureTimestampConfig({
- timestampExpression: 'timestamp_parse("time") + 1',
- }),
- configureSchemaConfig: new ConfigureSchemaConfig({ rollup: false }),
- partitionConfig: new PartitionConfig({
- segmentGranularity: SegmentGranularity.DAY,
- timeIntervals: null,
- partitionsSpec: null,
- }),
- publishConfig: new PublishConfig({ datasourceName: datasourceName }),
+ },
+ validateConnect: validateConnectLocalData,
+ timestampExpression: 'timestamp_parse("time") + 1',
+ rollup: false,
+ segmentGranularity: 'day',
+ datasourceName,
});
- await saveScreenshotIfError(testName, page, async () => {
- await dataLoader.load();
- await validateTaskStatus(page, datasourceName);
- await validateDatasourceStatus(page, datasourceName);
- await validateQuery(page, datasourceName);
- });
+ await expect
+ .poll(() => getTaskStatuses(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual(['SUCCESS']);
+ await expect
+ .poll(() => getDatasourceSegments(request, datasourceName), CLUSTER_STATE_POLL)
+ .toEqual({ numSegments: 1, numAvailableSegments: 1, numRows: 39244 });
+
+ // Now that the cluster has the data, the views should show it
+ await validateTasksView(page, datasourceName);
+ await validateDatasourcesView(page, datasourceName);
+ await validateQuery(page, datasourceName);
});
});
@@ -141,41 +113,40 @@ function validateConnectLocalData(lines: string[]) {
);
}
-async function validateTaskStatus(page: playwright.Page, datasourceName: string) {
- const tasksOverview = new TasksOverview(page, UNIFIED_CONSOLE_URL);
+async function validateTasksView(page: Page, datasourceName: string) {
+ const tasksOverview = new TasksOverview(page);
- await retryIfJestAssertionError(async () => {
- const tasks = await tasksOverview.getTasks();
+ // Retried in case the view is a step behind the cluster state polled above
+ await expect(async () => {
+ const tasks = await tasksOverview.getTasks(datasourceName);
const task = tasks.find(t => t.datasource === datasourceName);
expect(task).toBeDefined();
expect(task!.status).toMatch('SUCCESS');
- });
+ }).toPass({ timeout: 30 * 1000 });
}
-async function validateDatasourceStatus(page: playwright.Page, datasourceName: string) {
- const datasourcesOverview = new DatasourcesOverview(page, UNIFIED_CONSOLE_URL);
+async function validateDatasourcesView(page: Page, datasourceName: string) {
+ const datasourcesOverview = new DatasourcesOverview(page);
- await retryIfJestAssertionError(async () => {
- const datasources = await datasourcesOverview.getDatasources();
+ // Retried in case the view is a step behind the cluster state polled above
+ await expect(async () => {
+ const datasources = await datasourcesOverview.getDatasources(datasourceName);
const datasource = datasources.find(t => t.name === datasourceName);
expect(datasource).toBeDefined();
expect(datasource!.availability).toMatch('Fully available (1 segment)');
expect(datasource!.totalRows).toBe(39244);
- });
+ }).toPass({ timeout: 30 * 1000 });
}
-async function validateQuery(page: playwright.Page, datasourceName: string) {
- const queryOverview = new QueryOverview(page, UNIFIED_CONSOLE_URL);
+async function validateQuery(page: Page, datasourceName: string) {
+ const workbench = new WorkbenchOverview(page);
const query = `SELECT * FROM ${T(datasourceName)} ORDER BY __time`;
let results!: string[][];
- await retryOnAnyError(
- async () => {
- results = await queryOverview.runQuery(query);
- expect(results.length).toBeGreaterThan(0);
- },
- 1000,
- 3,
- );
+ // The datasource can be available before the Broker's SQL schema knows about it
+ await expect(async () => {
+ results = await workbench.runQuery(query);
+ expect(results.length).toBeGreaterThan(0);
+ }).toPass();
expect(results[0]).toStrictEqual([
/* __time */ '2015-09-12T00:46:58.772Z',
/* time */ '2015-09-12T00:46:58.771Z',
diff --git a/web-console/e2e-tests/util/debug.ts b/web-console/e2e-tests/util/debug.ts
deleted file mode 100644
index bfd2e2db7006..000000000000
--- a/web-console/e2e-tests/util/debug.ts
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { resolve } from 'path';
-import type * as playwright from 'playwright-chromium';
-
-export async function saveScreenshotIfError(
- filenamePrefix: string,
- page: playwright.Page,
- test: () => Promise,
-) {
- try {
- await test();
- } catch (e) {
- console.log(`Grabbing error screenshot for: ${filenamePrefix}`);
- const resolvedPath = resolve(filenamePrefix + '-error-screenshot.jpeg');
- try {
- const imageBuffer = await page.screenshot({ path: resolvedPath, type: 'jpeg', quality: 80 });
- console.log(`Image: data:image/jpeg;base64,${imageBuffer.toString('base64')}`);
- console.log(`Written error screenshot to: ${resolvedPath}`);
- } catch (screenshotError) {
- console.log(`Failed to capture screenshot due to: ${screenshotError.message}`);
- }
- throw e;
- }
-}
diff --git a/web-console/e2e-tests/util/druid.ts b/web-console/e2e-tests/util/druid.ts
index 7345a66b9092..1ca2defab1d9 100644
--- a/web-console/e2e-tests/util/druid.ts
+++ b/web-console/e2e-tests/util/druid.ts
@@ -16,12 +16,12 @@
* limitations under the License.
*/
-import { execSync } from 'child_process';
+import type { APIRequestContext, APIResponse } from '@playwright/test';
+import { expect } from '@playwright/test';
+import { readFileSync } from 'fs';
import path from 'path';
-const UNIFIED_CONSOLE_PORT = process.env['DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT'] || '8888';
-export const UNIFIED_CONSOLE_URL = `http://localhost:${UNIFIED_CONSOLE_PORT}/unified-console.html`;
-export const COORDINATOR_URL = 'http://localhost:8081';
+import type { IngestionSpec } from '../../src/druid-models';
const UTIL_DIR = __dirname;
const E2E_TEST_DIR = path.dirname(UTIL_DIR);
@@ -34,16 +34,102 @@ export const DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR = path.join(
'tutorial',
);
-export function runIndexTask(ingestionSpecPath: string, sedCommands: string[]) {
- const postIndexTask = path.join(DRUID_DIR, 'examples', 'bin', 'post-index-task');
- const sedCommandsString = sedCommands.map(sedCommand => `-e '${sedCommand}'`).join(' ');
- execSync(
- `${postIndexTask} \
- --file <(sed ${sedCommandsString} ${ingestionSpecPath}) \
- --url ${COORDINATOR_URL}`,
- {
- shell: 'bash',
- timeout: 3 * 60 * 1000,
- },
+/**
+ * Reads one of the ingestion specs of the tutorials (in examples/quickstart/tutorial), reading its input from this
+ * checkout (rather than from `quickstart/tutorial/` relative to where Druid runs).
+ */
+export function readTutorialIngestionSpec(fileName: string): IngestionSpec {
+ const ingestionSpec = JSON.parse(
+ readFileSync(path.join(DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR, fileName), 'utf-8'),
+ ) as IngestionSpec;
+ ingestionSpec.spec.ioConfig.inputSource!.baseDir = DRUID_EXAMPLES_QUICKSTART_TUTORIAL_DIR;
+ return ingestionSpec;
+}
+
+/**
+ * Encodes a datasource name (or other value) for a URL path, like Api.encodePath in src/singletons/api.ts (which
+ * can't be imported here).
+ */
+function encodePath(path: string): string {
+ return path.replace(/[#%&';?[\\\]^|]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase());
+}
+
+/**
+ * Submits a task (through the console's service, which `request`, the Playwright request fixture, uses) and waits
+ * for it to succeed. For an ingestion task, its segments are loaded some time after that.
+ */
+export async function runTask(
+ request: APIRequestContext,
+ task: IngestionSpec | Record,
+): Promise {
+ const submitResponse = await request.post('/druid/indexer/v1/task', { data: task });
+ expect(submitResponse.ok(), await submitResponse.text()).toBe(true);
+ const taskId = ((await submitResponse.json()) as { task: string }).task;
+
+ let statusCode: string | undefined;
+ let errorMsg: string | undefined;
+ await expect
+ .poll(
+ async () => {
+ const statusResponse = await request.get(
+ `/druid/indexer/v1/task/${encodePath(taskId)}/status`,
+ );
+ expect(statusResponse.ok(), await statusResponse.text()).toBe(true);
+ const { status } = (await statusResponse.json()) as {
+ status: { statusCode: string; errorMsg?: string };
+ };
+ ({ statusCode, errorMsg } = status);
+ return statusCode;
+ },
+ { timeout: 3 * 60 * 1000, intervals: [1000], message: `task ${taskId} to finish` },
+ )
+ .not.toBe('RUNNING');
+
+ // Fail here, with why the task failed, rather than time out waiting for its segments
+ expect(statusCode, errorMsg).toBe('SUCCESS');
+}
+
+/**
+ * Removes a datasource that a test created, and everything about it: stops its tasks, removes its compaction config
+ * and permanently deletes its segments.
+ */
+export async function deleteDatasource(
+ request: APIRequestContext,
+ datasource: string,
+): Promise {
+ const encodedDatasource = encodePath(datasource);
+ const expectOk = async (response: APIResponse, allowNotFound = false) => {
+ if (allowNotFound && response.status() === 404) return;
+ expect(response.ok(), `${response.url()}: ${await response.text()}`).toBe(true);
+ };
+
+ await expectOk(
+ await request.post(`/druid/indexer/v1/datasources/${encodedDatasource}/shutdownAllTasks`),
+ true, // It has no running tasks
+ );
+ await expectOk(
+ await request.delete(`/druid/indexer/v1/compaction/config/datasources/${encodedDatasource}`),
+ true, // It has no compaction config
);
+ await expectOk(
+ await request.delete(`/druid/indexer/v1/datasources/${encodedDatasource}`),
+ true, // It has no segments
+ );
+ await runTask(request, {
+ type: 'kill',
+ dataSource: datasource,
+ interval: '1000-01-01/3000-01-01',
+ });
+}
+
+/**
+ * The retention rules of a datasource (none when it uses the cluster default rules).
+ */
+export async function getRetentionRules(
+ request: APIRequestContext,
+ datasourceName: string,
+): Promise[]> {
+ const response = await request.get(`/druid/coordinator/v1/rules/${encodePath(datasourceName)}`);
+ expect(response.status(), await response.text()).toBe(200);
+ return (await response.json()) as Record[];
}
diff --git a/web-console/e2e-tests/util/fixtures.ts b/web-console/e2e-tests/util/fixtures.ts
new file mode 100644
index 000000000000..5e57e9c2db24
--- /dev/null
+++ b/web-console/e2e-tests/util/fixtures.ts
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test as base } from '@playwright/test';
+
+import { deleteDatasource } from './druid';
+import { showStep } from './steps';
+
+export { expect } from '@playwright/test';
+
+export const test = base.extend<{
+ logFailedResponses: void;
+ showLastStep: void;
+ newDatasourceName: (prefix: string) => string;
+}>({
+ // Logs every response with an error status (and its body) so that a failure can be understood from the log alone
+ logFailedResponses: [
+ async ({ page }, use) => {
+ page.on('response', async response => {
+ if (response.status() < 400) return;
+
+ let bodyText: string;
+ try {
+ bodyText = await response.text();
+ } catch (e) {
+ bodyText = `Could not get the body of the error message due to: ${e.message}`;
+ }
+
+ console.log(`==============================================`);
+ console.log(`Request failed on ${response.url()} (with status ${response.status()})`);
+ console.log(`Body: ${bodyText}`);
+ console.log(`==============================================`);
+ });
+
+ await use();
+ },
+ { auto: true },
+ ],
+
+ // Shows the page as the test ends (see showStep), as the last step
+ showLastStep: [
+ async ({ page }, use, testInfo) => {
+ await use();
+ await showStep(
+ page,
+ testInfo.status === testInfo.expectedStatus ? 'The end' : 'The end (failed)',
+ );
+ },
+ { auto: true },
+ ],
+
+ // Makes a unique datasource name (the prefix and the time), for the test to create. After the test passes, the
+ // datasource is deleted (with its tasks, compaction config and segments). After it fails, it is kept to look into.
+ // Nothing is deleted on a cluster that is thrown away after the tests (an embedded one, see WebConsoleTestBase).
+ newDatasourceName: [
+ async ({ request }, use, testInfo) => {
+ const datasourceNames: string[] = [];
+ await use(prefix => {
+ const datasourceName = prefix + new Date().toISOString();
+ datasourceNames.push(datasourceName);
+ return datasourceName;
+ });
+
+ if (process.env['DRUID_E2E_TEST_CLUSTER_IS_DISPOSABLE'] === 'true') return;
+ if (testInfo.status !== testInfo.expectedStatus) {
+ for (const datasourceName of datasourceNames) {
+ console.log(`Keeping datasource ${datasourceName} of the failed test`);
+ }
+ return;
+ }
+ for (const datasourceName of datasourceNames) {
+ await deleteDatasource(request, datasourceName);
+ }
+ },
+ { timeout: 2 * 60 * 1000 },
+ ],
+});
diff --git a/web-console/e2e-tests/util/setup.ts b/web-console/e2e-tests/util/global-setup.ts
similarity index 58%
rename from web-console/e2e-tests/util/setup.ts
rename to web-console/e2e-tests/util/global-setup.ts
index 27bc807cd81c..7e6f732d0063 100644
--- a/web-console/e2e-tests/util/setup.ts
+++ b/web-console/e2e-tests/util/global-setup.ts
@@ -16,19 +16,21 @@
* limitations under the License.
*/
-import { UNIFIED_CONSOLE_URL } from './druid';
-import { createBrowser, createPage } from './playwright';
-
-export async function waitTillWebConsoleReady() {
- const browser = await createBrowser();
+import type { FullConfig } from '@playwright/test';
+import { expect, request } from '@playwright/test';
+/**
+ * Waits until the console can run SQL, which is what it needs to function.
+ */
+export default async function globalSetup(config: FullConfig) {
+ const { baseURL } = config.projects[0].use;
+ const api = await request.newContext({ baseURL });
try {
- const page = await createPage(browser);
- await page.goto(UNIFIED_CONSOLE_URL);
- await page.waitForSelector('//*[contains(text(),"console will not function at the moment")]', {
- state: 'hidden',
- });
+ await expect(async () => {
+ const response = await api.post('/druid/v2/sql', { data: { query: 'SELECT 1' } });
+ expect(response.status()).toBe(200);
+ }).toPass({ timeout: 2 * 60 * 1000, intervals: [1000] });
} finally {
- await browser.close();
+ await api.dispose();
}
}
diff --git a/web-console/e2e-tests/util/playwright.ts b/web-console/e2e-tests/util/playwright.ts
index efd79764b5a5..06ab8beb6dab 100644
--- a/web-console/e2e-tests/util/playwright.ts
+++ b/web-console/e2e-tests/util/playwright.ts
@@ -16,129 +16,125 @@
* limitations under the License.
*/
-import * as playwright from 'playwright-chromium';
-
-const TRUE = 'true';
-const WIDTH = 1250;
-const HEIGHT = 760;
-const PADDING = 128;
-
-export async function createBrowser(): Promise {
- const headless = process.env['DRUID_E2E_TEST_HEADLESS'] || TRUE;
- const debug = headless !== TRUE;
- const launchOptions: any = {
- args: [`--window-size=${WIDTH},${HEIGHT + PADDING}`, `--disable-local-storage`],
- };
- if (debug) {
- launchOptions.headless = false;
- launchOptions.slowMo = 20;
- }
- return playwright.chromium.launch(launchOptions);
+import type { Locator, Page } from '@playwright/test';
+
+const CONSOLE_PATH = '/unified-console.html';
+
+/**
+ * Opens a view of the console (`view` is its hash route, like 'datasources'), optionally with its table filtered to
+ * the rows where each `filter` column equals the given value. If the console is already open it is reloaded, so that
+ * the view shows fresh data.
+ */
+export async function openView(
+ page: Page,
+ view: string,
+ filter?: Record,
+): Promise {
+ // Encoded like TableFilters.eq(filter).toString() in src/utils/table-filters (which can't be imported here as it
+ // brings in the whole of src/utils)
+ const filterParam = Object.entries(filter ?? {})
+ .map(
+ ([column, value]) =>
+ `${column}=${value.replace(/[\\|]/g, '\\$&').replace(/[#%&/?]/g, encodeURIComponent)}`,
+ )
+ .join('&');
+ const wasOpen = page.url().includes(CONSOLE_PATH);
+ await page.goto(`${CONSOLE_PATH}#${view}${filterParam ? `/${filterParam}` : ''}`);
+ if (wasOpen) await page.reload();
}
-export async function createPage(browser: playwright.Browser): Promise {
- const context = await browser.newContext();
- const page = await context.newPage();
- await page.setViewportSize({ width: WIDTH, height: HEIGHT });
-
- page.on('response', async response => {
- if (response.status() < 400) return;
-
- const request = response.request();
- let bodyText: string;
- try {
- bodyText = await response.text();
- } catch (e) {
- bodyText = `Could not get the body of the error message due to: ${e.message}`;
- }
-
- console.log(`==============================================`);
- console.log(`Request failed on ${request.url()} (with status ${response.status()})`);
- console.log(`Body: ${bodyText}`);
- console.log(`==============================================`);
- });
+function escapeRegExp(text: string): string {
+ return text.replace(/[$()*+.?[\\\]^{|}]/g, '\\$&');
+}
- return page;
+/**
+ * The form group (a Blueprint FormGroup, as rendered by AutoForm and FormGroupWithInfo) labeled `label`. The labels
+ * are not linked to their inputs, so `getByLabel` can not find them.
+ */
+export function formGroup(scope: Page | Locator, label: string): Locator {
+ // The `has` locator is matched relative to each form group, so it must not start from `scope`
+ const page = 'page' in scope ? scope.page() : scope;
+ return scope.locator('.bp6-form-group').filter({
+ has: page.locator(':scope > .bp6-label', {
+ hasText: new RegExp(`^\\s*${escapeRegExp(label)}\\s*$`),
+ }),
+ });
}
-export async function getLabeledInput(page: playwright.Page, label: string): Promise {
- return await page.$eval(
- `//*[text()="${label}"]/following-sibling::div//input`,
- el => (el as HTMLInputElement).value,
- );
+export function labeledInput(scope: Page | Locator, label: string): Locator {
+ return formGroup(scope, label).locator('input');
}
-export async function getLabeledTextarea(page: playwright.Page, label: string): Promise {
- return await page.$eval(
- `//*[text()="${label}"]/following-sibling::div//textarea`,
- el => (el as HTMLInputElement).value,
- );
+export function labeledTextarea(scope: Page | Locator, label: string): Locator {
+ return formGroup(scope, label).locator('textarea');
}
-export async function setLabeledInput(
- page: playwright.Page,
+/**
+ * Sets a labeled boolean field of an AutoForm (its False / True buttons).
+ */
+export async function setLabeledBoolean(
+ scope: Page | Locator,
label: string,
- value: string,
+ value: boolean,
): Promise {
- return setLabeledElement(page, 'input', label, value);
+ await formGroup(scope, label)
+ .getByText(value ? 'True' : 'False', { exact: true })
+ .click();
}
-export async function setLabeledTextarea(
- page: playwright.Page,
+export async function getLabeledInput(scope: Page | Locator, label: string): Promise {
+ return await labeledInput(scope, label).inputValue();
+}
+
+export async function getLabeledTextarea(scope: Page | Locator, label: string): Promise {
+ return await labeledTextarea(scope, label).inputValue();
+}
+
+export async function setLabeledInput(
+ scope: Page | Locator,
label: string,
value: string,
): Promise {
- return setLabeledElement(page, 'textarea', label, value);
+ await labeledInput(scope, label).fill(value);
}
-async function setLabeledElement(
- page: playwright.Page,
- type: string,
+export async function setLabeledTextarea(
+ scope: Page | Locator,
label: string,
value: string,
): Promise {
- const element = await page.$(`//*[text()="${label}"]/following-sibling::div//${type}`);
- await setInput(element!, value);
+ await labeledTextarea(scope, label).fill(value);
}
-export async function setInput(
- input: playwright.ElementHandle,
+/**
+ * Picks `value` from the suggestions of a labeled SuggestibleInput.
+ */
+export async function selectSuggestibleInput(
+ page: Page,
+ label: string,
value: string,
): Promise {
- await input.fill(value);
+ await formGroup(page, label).getByRole('button').click();
+ await page.getByRole('menuitem', { name: value, exact: true }).click();
}
-export async function setQueryInput(page: playwright.Page, value: string): Promise {
+export async function setQueryInput(page: Page, value: string): Promise {
// The query input is a CodeMirror editor, its editable surface is a contenteditable div
- const editor = await page.waitForSelector('div.flexible-query-input .cm-content');
- await editor.fill(value);
-}
-
-function buttonSelector(text: string) {
- return `//button/*[contains(text(),"${text}")]`;
-}
-
-export async function clickButton(page: playwright.Page, text: string): Promise {
- await page.click(buttonSelector(text));
+ const input = page.locator('.flexible-query-input .cm-content');
+ await input.fill(value);
+ // Closes the autocomplete that typing opens, which would cover the query (and its results)
+ await input.press('Escape');
}
-export async function clickLabeledButton(
- page: playwright.Page,
- label: string,
- text: string,
-): Promise {
- await page.click(`//*[text()="${label}"]/following-sibling::div${buttonSelector(text)}`);
+export function button(scope: Page | Locator, text: string): Locator {
+ return scope.getByRole('button', { name: text, exact: true });
}
-export async function clickText(page: playwright.Page, text: string): Promise {
- await page.click(`//*[text()="${text}"]`);
+export async function clickButton(scope: Page | Locator, text: string): Promise {
+ await button(scope, text).click();
}
-export async function selectSuggestibleInput(
- page: playwright.Page,
- label: string,
- value: string,
-): Promise {
- await page.click(`//*[text()="${label}"]/following-sibling::div//button`);
- await page.click(`"${value}"`);
+export async function clickMenuItem(page: Page, text: string): Promise {
+ // Not exact: a menu item's label (like "(debug)") is part of its name
+ await page.getByRole('menuitem', { name: text }).click();
}
diff --git a/web-console/e2e-tests/util/retry.ts b/web-console/e2e-tests/util/retry.ts
deleted file mode 100644
index b077ebda8c76..000000000000
--- a/web-console/e2e-tests/util/retry.ts
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-function sleep(msec: number) {
- return new Promise(resolve => setTimeout(resolve, msec));
-}
-
-export async function retryIfJestAssertionError(
- callback: () => Promise,
- msec = 1000,
- maxTries = 60,
-) {
- let i = 0;
-
- while (true) {
- try {
- await callback();
- return;
- } catch (e) {
- i++;
- if (e.constructor.name === 'JestAssertionError' && i < maxTries) {
- await sleep(msec);
- } else {
- throw e;
- }
- }
- }
-}
-
-export async function retryOnAnyError(callback: () => Promise, msec = 1000, maxTries = 60) {
- let i = 0;
-
- while (true) {
- try {
- await callback();
- return;
- } catch (e) {
- i++;
- if (i < maxTries) {
- await sleep(msec);
- } else {
- throw e;
- }
- }
- }
-}
diff --git a/web-console/e2e-tests/util/sql.ts b/web-console/e2e-tests/util/sql.ts
new file mode 100644
index 000000000000..33abd51e40fb
--- /dev/null
+++ b/web-console/e2e-tests/util/sql.ts
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { APIRequestContext } from '@playwright/test';
+import { T } from 'druid-query-toolkit';
+
+/**
+ * For expect.poll() on the cluster state: a task finishing, segments loading.
+ */
+export const CLUSTER_STATE_POLL = { timeout: 2 * 60 * 1000, intervals: [1000] };
+
+/**
+ * Runs a SQL query through the console's service (`request` is the Playwright request fixture, which uses the
+ * console's URL), with `parameters` for its `?` placeholders.
+ */
+export async function querySql(
+ request: APIRequestContext,
+ query: string,
+ parameters: string[] = [],
+): Promise[]> {
+ const response = await request.post('/druid/v2/sql', {
+ data: { query, parameters: parameters.map(value => ({ type: 'VARCHAR', value })) },
+ });
+ if (!response.ok()) {
+ throw new Error(`SQL query failed with status ${response.status()}: ${await response.text()}`);
+ }
+ return (await response.json()) as Record[];
+}
+
+/**
+ * The statuses of the tasks of a datasource, oldest first. Only the tasks that were submitted, not the subtasks they
+ * started (which are in the group of the task that started them).
+ */
+export async function getTaskStatuses(
+ request: APIRequestContext,
+ datasource: string,
+): Promise {
+ const rows = await querySql(
+ request,
+ `SELECT "status"
+FROM sys.tasks
+WHERE "datasource" = ? AND "task_id" = "group_id"
+ORDER BY "created_time"`,
+ [datasource],
+ );
+ return rows.map(row => String(row['status']));
+}
+
+export interface DatasourceSegments {
+ readonly numSegments: number;
+ readonly numAvailableSegments: number;
+ readonly numRows: number | null;
+}
+
+/**
+ * The used segments (published and not overshadowed) of a datasource, how many of them are available (loaded) and
+ * the rows in them.
+ */
+export async function getDatasourceSegments(
+ request: APIRequestContext,
+ datasource: string,
+): Promise {
+ const [row] = await querySql(
+ request,
+ `SELECT
+ COUNT(*) AS "numSegments",
+ COALESCE(SUM("is_available"), 0) AS "numAvailableSegments",
+ SUM("num_rows") AS "numRows"
+FROM sys.segments
+WHERE "datasource" = ? AND "is_published" = 1 AND "is_overshadowed" = 0`,
+ [datasource],
+ );
+ // Druid returns no row (rather than a row of zeros) when there are no segments
+ if (!row) return { numSegments: 0, numAvailableSegments: 0, numRows: null };
+ return {
+ numSegments: Number(row['numSegments']),
+ numAvailableSegments: Number(row['numAvailableSegments']),
+ numRows: row['numRows'] == null ? null : Number(row['numRows']),
+ };
+}
+
+/**
+ * The number of rows of a datasource that queries see, including the rows of realtime (streaming) tasks that are not
+ * in published segments yet.
+ */
+export async function getQueryableRowCount(
+ request: APIRequestContext,
+ datasource: string,
+): Promise {
+ // Through INFORMATION_SCHEMA first, as querying a datasource that doesn't exist yet is an error
+ const tables = await querySql(
+ request,
+ `SELECT 1 FROM INFORMATION_SCHEMA.TABLES WHERE "TABLE_SCHEMA" = 'druid' AND "TABLE_NAME" = ?`,
+ [datasource],
+ );
+ if (!tables.length) return 0;
+
+ const [row] = await querySql(request, `SELECT COUNT(*) AS "numRows" FROM ${T(datasource)}`);
+ return Number(row['numRows']);
+}
+
+/**
+ * The detailed state of a supervisor (like RUNNING or SUSPENDED), or null when there is no such supervisor (not yet,
+ * or not anymore as it was terminated).
+ */
+export async function getSupervisorState(
+ request: APIRequestContext,
+ supervisorId: string,
+): Promise {
+ const [row] = await querySql(
+ request,
+ `SELECT "detailed_state" FROM sys.supervisors WHERE "supervisor_id" = ?`,
+ [supervisorId],
+ );
+ return row ? String(row['detailed_state']) : null;
+}
+
+/**
+ * The state of a Dart query (like RUNNING or SUCCESS) in sys.queries, or null when it isn't there (not yet, or not
+ * anymore).
+ */
+export async function getDartQueryState(
+ request: APIRequestContext,
+ sqlQueryId: string,
+): Promise {
+ const [row] = await querySql(
+ request,
+ `SELECT "state" FROM sys.queries WHERE "engine" = 'msq-dart' AND "info" LIKE ?`,
+ [`%${sqlQueryId}%`],
+ );
+ return row ? String(row['state']) : null;
+}
diff --git a/web-console/e2e-tests/util/steps.ts b/web-console/e2e-tests/util/steps.ts
new file mode 100644
index 000000000000..b074e46f7356
--- /dev/null
+++ b/web-console/e2e-tests/util/steps.ts
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Page } from '@playwright/test';
+import { test } from '@playwright/test';
+import * as fs from 'fs';
+import * as path from 'path';
+
+const SHOW_STEPS = process.env['E2E_SHOW_STEPS'] === 'true';
+const STEPS_DIR = path.join(__dirname, '..', 'steps');
+
+// The number of the last step shown, by spec (the counting starts over in each run)
+const lastStepBySpec = new Map();
+
+/**
+ * Marks a step of the test worth seeing: with E2E_SHOW_STEPS=true, it saves a screenshot of the page to
+ * e2e-tests/steps/-.png (like s3-ingestion-001.png), so that the steps a test takes can be seen at a
+ * glance. Otherwise it does nothing.
+ * @param title what the page shows at this step, which is how the step is named in the trace and the report
+ */
+export async function showStep(page: Page, title: string): Promise {
+ if (!SHOW_STEPS) return;
+
+ const spec = path.basename(test.info().file).replace(/\.spec\.ts$/, '');
+ const step = (lastStepBySpec.get(spec) ?? 0) + 1;
+ if (step === 1) deleteSteps(spec); // of an earlier run, which could have had more steps
+ lastStepBySpec.set(spec, step);
+
+ await test.step(`Step ${step}: ${title}`, async () => {
+ await page.screenshot({
+ path: path.join(STEPS_DIR, `${spec}-${String(step).padStart(3, '0')}.png`),
+ animations: 'disabled', // finishes the transitions (like a dialog fading in) rather than catch them midway
+ });
+ });
+}
+
+function deleteSteps(spec: string): void {
+ if (!fs.existsSync(STEPS_DIR)) return;
+ for (const file of fs.readdirSync(STEPS_DIR)) {
+ if (file.startsWith(`${spec}-`) && /^-\d+\.png$/.test(file.slice(spec.length))) {
+ fs.unlinkSync(path.join(STEPS_DIR, file));
+ }
+ }
+}
diff --git a/web-console/e2e-tests/util/table.ts b/web-console/e2e-tests/util/table.ts
index e40148f42a85..e4a59bd5119f 100644
--- a/web-console/e2e-tests/util/table.ts
+++ b/web-console/e2e-tests/util/table.ts
@@ -16,41 +16,45 @@
* limitations under the License.
*/
-import type * as playwright from 'playwright-chromium';
+import type { Locator } from '@playwright/test';
/**
- * Extracts an HTML table into a text representation.
- * @param page Playwright page from which to extract HTML table
- * @param tableSelector Playwright selector for table
- * @param rowSelector Playwright selector for table row
+ * Reads the data rows (not the rows that pad the table) of a ConsoleTable as text, one array of cells per row.
+ * @param table locator of the table (its `.console-table` element, or one containing it)
*/
-export async function extractTable(
- page: playwright.Page,
- tableSelector: string,
- rowSelector: string,
-): Promise {
- await page.waitForSelector(tableSelector);
+export async function extractTable(table: Locator): Promise {
+ return (await readTable(table)).rows;
+}
+
+/**
+ * Reads the data rows of a view's ConsoleTable as text, each row keyed by its column headers, once the view has
+ * loaded its data. Headers on two lines are read with a space for the line break (like "Datasource name").
+ * @param table locator of the table (its `.console-table` element, or one containing it)
+ */
+export async function extractTableRecords(table: Locator): Promise[]> {
+ // A view loads its data after it opens, and shows its no data text (like "No tasks", or the error) only after that
+ await table
+ .locator('.ct-tbody .ct-tr:not(.-padRow)')
+ .or(table.locator('.ct-no-data').filter({ hasText: /\S/ }))
+ .first()
+ .waitFor();
+
+ const { headers, rows } = await readTable(table);
+ return rows.map(row => Object.fromEntries(headers.map((header, i) => [header, row[i]])));
+}
+
+async function readTable(table: Locator): Promise<{ headers: string[]; rows: string[][] }> {
+ return await table.evaluate(tableElement => {
+ const cellText = (cell: Element) =>
+ ((cell.querySelector('.real-text') ?? cell) as HTMLElement).innerText;
- return page.evaluate(
- ([tableSelector, rowSelector]) => {
- const BLANK_VALUE = '\xa0';
- const data = [];
- const rows = document.querySelectorAll(tableSelector);
- for (let i = 0; i < rows.length; i++) {
- const row = rows[i];
- const columns = row.querySelectorAll(rowSelector);
- const values = Array.from(columns).map(c => {
- const realTexts = Array.from(c.querySelectorAll('.real-text'));
- return realTexts.length
- ? (realTexts[0] as HTMLElement).innerText
- : (c as HTMLElement).innerText;
- });
- if (!values.every(value => value === BLANK_VALUE)) {
- data.push(values);
- }
- }
- return data;
- },
- [tableSelector, rowSelector],
- );
+ return {
+ headers: Array.from(tableElement.querySelectorAll('.ct-thead.-header .ct-th'), header =>
+ (header as HTMLElement).innerText.replace(/\s+/g, ' ').trim(),
+ ),
+ rows: Array.from(tableElement.querySelectorAll('.ct-tbody .ct-tr:not(.-padRow)'), row =>
+ Array.from(row.querySelectorAll('.ct-td'), cellText),
+ ),
+ };
+ });
}
diff --git a/web-console/eslint.config.mjs b/web-console/eslint.config.mjs
index b5739a2d82ae..88e60ba78d2b 100644
--- a/web-console/eslint.config.mjs
+++ b/web-console/eslint.config.mjs
@@ -84,10 +84,4 @@ export default [
'@typescript-eslint/no-require-imports': [0],
},
},
- {
- files: ['e2e-tests/**/*.ts'],
- rules: {
- '@typescript-eslint/no-unsafe-declaration-merging': [0],
- },
- },
];
diff --git a/web-console/package-lock.json b/web-console/package-lock.json
index b27cefd9a46e..0cc26eef6329 100644
--- a/web-console/package-lock.json
+++ b/web-console/package-lock.json
@@ -61,6 +61,7 @@
"@awesome-code-style/stylelint-config": "^6.0.2",
"@lezer/generator": "^1.8.1",
"@lizenz/checker": "^0.0.2",
+ "@playwright/test": "^1.63.0",
"@testing-library/dom": "^10.4.0",
"@testing-library/react": "^16.0.1",
"@types/d3-array": "^3.2.1",
@@ -7831,6 +7832,22 @@
"url": "https://opencollective.com/pkgr"
}
},
+ "node_modules/@playwright/test": {
+ "version": "1.63.0",
+ "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.63.0.tgz",
+ "integrity": "sha512-oxMK4vllB9RK5NQ2l1pq1IfOf2AvnEuj/vYGDj0H2nMtmtZpKtCwt/l00GEO6xjGfpBNAvjovvYdCm50dRQkpQ==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright": "1.63.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
"node_modules/@polka/url": {
"version": "1.0.0-next.29",
"resolved": "https://registry.npmjs.org/@polka/url/-/url-1.0.0-next.29.tgz",
@@ -17535,6 +17552,22 @@
"node": ">=16.0.0"
}
},
+ "node_modules/playwright": {
+ "version": "1.63.0",
+ "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.63.0.tgz",
+ "integrity": "sha512-+7ziBLidS4NaNCdt57SUDT+wYmmd5fmiQejUic/kb+YsYSCPyOOE9sebzMjNmQrsnNpDJqd4WHvV/8lfKfUDUg==",
+ "dev": true,
+ "license": "Apache-2.0",
+ "dependencies": {
+ "playwright-core": "1.63.0"
+ },
+ "bin": {
+ "playwright": "cli.js"
+ },
+ "engines": {
+ "node": ">=20"
+ }
+ },
"node_modules/playwright-chromium": {
"version": "1.63.0",
"resolved": "https://registry.npmjs.org/playwright-chromium/-/playwright-chromium-1.63.0.tgz",
diff --git a/web-console/package.json b/web-console/package.json
index e32e196c7cd5..881e18bb0eeb 100644
--- a/web-console/package.json
+++ b/web-console/package.json
@@ -16,7 +16,7 @@
"test-base": "npm run eslint && npm run sasslint && npm run prettify-check && npm run jest",
"test": "npm run test-base -- --silent 2>&1",
"test-ci": "npm run test-base -- --coverage",
- "test-e2e": "jest --runInBand --config jest.e2e.config.js e2e-tests",
+ "test-e2e": "playwright test",
"test-unit": "./script/build-sql-docs && ./script/build-grammars.mjs && npm run typecheck && npm run eslint && npm run sasslint && npm run prettify-check && jest --config jest.config.js --testPathIgnorePatterns='e2e-tests'",
"coverage": "jest --coverage src",
"update-snapshots": "jest -u --config jest.config.js",
@@ -30,8 +30,8 @@
"sasslint-fix": "npm run sasslint -- --fix",
"sasslint-changed-only": "git diff --diff-filter=ACMR --name-only | grep -E \\.scss$ | xargs ./node_modules/.bin/stylelint",
"sasslint-fix-changed-only": "npm run sasslint-changed-only -- --fix",
- "prettify": "prettier --write '{src,e2e-tests}/**/*.{ts,tsx,scss}' './*.js'",
- "prettify-check": "prettier --check '{src,e2e-tests}/**/*.{ts,tsx,scss}' './*.js'",
+ "prettify": "prettier --write '{src,e2e-tests}/**/*.{ts,tsx,scss}' './*.{js,ts}'",
+ "prettify-check": "prettier --check '{src,e2e-tests}/**/*.{ts,tsx,scss}' './*.{js,ts}'",
"generate-licenses-file": "license-checker --production --json --out licenses.json",
"check-licenses": "license-checker --production --onlyAllow 'Apache-1.1;Apache-2.0;BSD-2-Clause;BSD-3-Clause;0BSD;MIT;ISC;CC0-1.0;OFL-1.1' --summary",
"start": "webpack serve"
@@ -99,6 +99,7 @@
"@awesome-code-style/stylelint-config": "^6.0.2",
"@lezer/generator": "^1.8.1",
"@lizenz/checker": "^0.0.2",
+ "@playwright/test": "^1.63.0",
"@testing-library/dom": "^10.4.0",
"@testing-library/react": "^16.0.1",
"@types/d3-array": "^3.2.1",
diff --git a/web-console/jest.e2e.config.js b/web-console/playwright.config.ts
similarity index 51%
rename from web-console/jest.e2e.config.js
rename to web-console/playwright.config.ts
index 50c7ad09470e..b0395fab425d 100644
--- a/web-console/jest.e2e.config.js
+++ b/web-console/playwright.config.ts
@@ -16,9 +16,28 @@
* limitations under the License.
*/
-const common = require('./jest.common.config');
+import { defineConfig } from '@playwright/test';
-module.exports = Object.assign(common, {
- testMatch: ['**/?(*.)+(spec).ts?(x)'],
- testEnvironment: 'node',
+const UNIFIED_CONSOLE_PORT = process.env['DRUID_E2E_TEST_UNIFIED_CONSOLE_PORT'] || '8888';
+
+export default defineConfig({
+ testDir: 'e2e-tests',
+ // The tests share one Druid cluster (and its task slots)
+ workers: 1,
+ timeout: 5 * 60 * 1000,
+ expect: {
+ timeout: 30 * 1000,
+ // For polling the cluster state: a task finishing, segments loading
+ toPass: { timeout: 2 * 60 * 1000, intervals: [1000] },
+ },
+ globalSetup: './e2e-tests/util/global-setup.ts',
+ reporter: 'list',
+ use: {
+ browserName: 'chromium',
+ baseURL: `http://localhost:${UNIFIED_CONSOLE_PORT}`,
+ viewport: { width: 1250, height: 760 },
+ actionTimeout: 30 * 1000,
+ screenshot: 'only-on-failure',
+ trace: 'retain-on-failure',
+ },
});
diff --git a/web-console/src/components/console-table/table-filter-inputs.tsx b/web-console/src/components/console-table/table-filter-inputs.tsx
index cf86bf954af0..dc12f14ccb3f 100644
--- a/web-console/src/components/console-table/table-filter-inputs.tsx
+++ b/web-console/src/components/console-table/table-filter-inputs.tsx
@@ -153,8 +153,8 @@ export function suggestibleFilterInput(suggestions: string[]) {
TableFilter.combineModeAndNeedle(
'=',
valuesFilteredOn
- ? toggle(valuesFilteredOn, suggestion).join('|')
- : suggestion,
+ ? TableFilter.joinNeedle(toggle(valuesFilteredOn, suggestion))
+ : TableFilter.joinNeedle([suggestion]),
true,
),
)
diff --git a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx
index bbab1463c3b1..dc574407c678 100644
--- a/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx
+++ b/web-console/src/components/table-filterable-cell/table-filterable-cell.tsx
@@ -59,7 +59,7 @@ export const TableFilterableCell = React.memo(function TableFilterableCell(
icon={TableFilter.modeToIcon(mode)}
text={displayValue ?? value}
onClick={() =>
- onFiltersChange(filters.addOrUpdate(new TableFilter(field, mode, value)))
+ onFiltersChange(filters.addOrUpdate(new TableFilter(field, mode, [value])))
}
/>
))}
diff --git a/web-console/src/singletons/api.spec.ts b/web-console/src/singletons/api.spec.ts
index 1dda6d8611ef..34b5024feb8a 100644
--- a/web-console/src/singletons/api.spec.ts
+++ b/web-console/src/singletons/api.spec.ts
@@ -22,6 +22,7 @@ describe('Api', () => {
it('escapes stuff', () => {
expect(Api.encodePath('wikipedia')).toEqual('wikipedia');
expect(Api.encodePath(`wi%ki?pe#dia&'[;]`)).toEqual('wi%25ki%3Fpe%23dia%26%27%5B%3B%5D');
+ expect(Api.encodePath('wiki\\pe^di|a')).toEqual('wiki%5Cpe%5Edi%7Ca');
});
describe(`with BigInt`, () => {
diff --git a/web-console/src/singletons/api.ts b/web-console/src/singletons/api.ts
index c7b123df76c7..2f330568a5fd 100644
--- a/web-console/src/singletons/api.ts
+++ b/web-console/src/singletons/api.ts
@@ -71,8 +71,12 @@ export class Api {
};
}
+ /**
+ * Encodes a value (like a datasource name) for a URL path. Encodes what would change the URL's meaning, and what
+ * the browser sends as is but the server rejects (^ and |); the browser encodes everything else that needs it.
+ */
static encodePath(path: string): string {
- return path.replace(/[?#%;&'[\]\\]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase());
+ return path.replace(/[#%&';?[\\\]^|]/g, c => '%' + c.charCodeAt(0).toString(16).toUpperCase());
}
static isNetworkError(e: Error): boolean {
diff --git a/web-console/src/utils/hash-routing.spec.ts b/web-console/src/utils/hash-routing.spec.ts
index 1cc2002fa9c7..24e73cd8d881 100644
--- a/web-console/src/utils/hash-routing.spec.ts
+++ b/web-console/src/utils/hash-routing.spec.ts
@@ -42,6 +42,18 @@ describe('hash-routing', () => {
expect(TableFilters.fromString(route.param).toString()).toEqual(filters.toString());
});
+ it('keeps table filters with every sort of character', () => {
+ const filters = TableFilters.eq({
+ datasource: '<>|!@#$%^&`\'".,:;\\*()[]{}Україна 한국 中国!?~',
+ });
+ // As the browser has it in the URL, which percent-encodes the non-ASCII and some ASCII characters
+ const route = parseHashRoute(encodeURI(`datasources/${filters.toString()}`));
+ expect(route.view).toEqual('datasources');
+ expect(TableFilters.fromString(route.param).toArray()[0].values).toEqual(
+ filters.toArray()[0].values,
+ );
+ });
+
it('decodes non reserved characters', () => {
expect(parseHashRoute('segments/datasource=wiki%20pedia')).toEqual({
view: 'segments',
diff --git a/web-console/src/utils/table-filters/table-filter.spec.ts b/web-console/src/utils/table-filters/table-filter.spec.ts
index b7fdec612095..8895ed7c1914 100644
--- a/web-console/src/utils/table-filters/table-filter.spec.ts
+++ b/web-console/src/utils/table-filters/table-filter.spec.ts
@@ -84,6 +84,32 @@ describe('TableFilter', () => {
});
});
+ describe('splitNeedle / joinNeedle', () => {
+ it('splits on |', () => {
+ expect(TableFilter.splitNeedle('a|b')).toEqual(['a', 'b']);
+ expect(TableFilter.splitNeedle('a')).toEqual(['a']);
+ expect(TableFilter.splitNeedle('')).toEqual(['']);
+ });
+
+ it('unescapes \\| and \\\\, and keeps any other \\', () => {
+ expect(TableFilter.splitNeedle('a\\|b|c')).toEqual(['a|b', 'c']);
+ expect(TableFilter.splitNeedle('a\\\\|b')).toEqual(['a\\', 'b']);
+ expect(TableFilter.splitNeedle('C:\\dir')).toEqual(['C:\\dir']);
+ });
+
+ it('round trips any values', () => {
+ const values = ['a|b', 'c\\', '\\|', '', 'C:\\dir'];
+ expect(TableFilter.splitNeedle(TableFilter.joinNeedle(values))).toEqual(values);
+ });
+
+ it('keeps a literal value that contains | as one value', () => {
+ const filter = new TableFilter('datasource', '=', ['a|b']);
+ expect(filter.value).toEqual('a\\|b');
+ expect(TableFilter.fromFilter(filter.toFilter()).values).toEqual(['a|b']);
+ expect(String(filter.toSqlExpression())).toEqual(`"datasource" = 'a|b'`);
+ });
+ });
+
describe('equals', () => {
it('compares two filters', () => {
const filter1 = new TableFilter('x', '=', 'y');
diff --git a/web-console/src/utils/table-filters/table-filter.ts b/web-console/src/utils/table-filters/table-filter.ts
index 528ac4076b40..03442d3807fe 100644
--- a/web-console/src/utils/table-filters/table-filter.ts
+++ b/web-console/src/utils/table-filters/table-filter.ts
@@ -33,9 +33,41 @@ export class TableFilter {
public readonly mode: FilterMode;
public readonly values: string[];
+ /**
+ * Splits a needle into its values, which are separated by `|`. In a value, `\|` is a literal `|` and `\\` a literal
+ * `\` (any other `\` is literal too).
+ */
+ static splitNeedle(needle: string): string[] {
+ const values: string[] = [];
+ let current = '';
+ for (let i = 0; i < needle.length; i++) {
+ const char = needle[i];
+ const next = needle[i + 1];
+ if (char === '\\' && (next === '|' || next === '\\')) {
+ current += next;
+ i++;
+ } else if (char === '|') {
+ values.push(current);
+ current = '';
+ } else {
+ current += char;
+ }
+ }
+ values.push(current);
+ return values;
+ }
+
+ /**
+ * Joins values into a needle, the inverse of `splitNeedle`.
+ */
+ static joinNeedle(values: readonly string[]): string {
+ return values.map(value => value.replace(/[\\|]/g, '\\$&')).join('|');
+ }
+
static fromSingleTableFilterString(str: string): TableFilter | undefined {
+ // Undo the encoding of TableFilters.toString()
const m = /^(\w+)((?:~|=|!=|<(?!=)|<=|>(?!=)|>=).*)$/.exec(
- str.replace(/%2[56F]/g, decodeURIComponent),
+ str.replace(/%2[356F]|%3F/g, decodeURIComponent),
);
if (!m) return;
@@ -64,14 +96,18 @@ export class TableFilter {
return new TableFilter(filter.id, mode, value);
}
+ /**
+ * @param value the values to filter on, or a needle (as typed in a filter input) of values separated by `|` (see
+ * `splitNeedle`). To filter on one literal value, that may contain a `|`, pass it as `[value]`.
+ */
constructor(key: string, mode: FilterMode, value: string | string[]) {
this.key = key;
this.mode = mode;
- this.values = typeof value === 'string' ? value.split('|') : value;
+ this.values = typeof value === 'string' ? TableFilter.splitNeedle(value) : value;
}
public get value(): string {
- return this.values.join('|');
+ return TableFilter.joinNeedle(this.values);
}
public toFilter(): ColumnFilter {
@@ -208,7 +244,7 @@ export class TableFilter {
return {
mode,
needle,
- needleParts: needle.split('|'),
+ needleParts: TableFilter.splitNeedle(needle),
};
}
}
diff --git a/web-console/src/utils/table-filters/table-filters.spec.ts b/web-console/src/utils/table-filters/table-filters.spec.ts
index 1a9dfa2ef774..3b8b633d7bfe 100644
--- a/web-console/src/utils/table-filters/table-filters.spec.ts
+++ b/web-console/src/utils/table-filters/table-filters.spec.ts
@@ -28,6 +28,20 @@ describe('TableFilters', () => {
]);
expect(filters.toString()).toEqual('x~y&z=w%26%25%2F');
});
+
+ it('encodes # and ?, which end the hash route', () => {
+ expect(TableFilters.eq({ x: 'a#b?c' }).toString()).toEqual('x=a%23b%3Fc');
+ });
+ });
+
+ describe('toString and fromString', () => {
+ it('round trip values with every sort of character', () => {
+ const filters = TableFilters.eq({
+ datasource: '<>|!@#$%^&`\'".,:;\\*()[]{}Україна 한국 中国!?~',
+ });
+ const parsed = TableFilters.fromString(filters.toString());
+ expect(parsed.toArray()[0].values).toEqual(filters.toArray()[0].values);
+ });
});
describe('fromString', () => {
@@ -82,6 +96,10 @@ describe('TableFilters', () => {
expect(filterArray[1].mode).toBe('=');
expect(filterArray[1].value).toBe('index');
});
+
+ it('filters on a value that contains | (rather than on its parts)', () => {
+ expect(TableFilters.eq({ datasource: 'a|b' }).toArray()[0].values).toEqual(['a|b']);
+ });
});
describe('empty', () => {
diff --git a/web-console/src/utils/table-filters/table-filters.ts b/web-console/src/utils/table-filters/table-filters.ts
index ee9a827ebd98..6cb4972be131 100644
--- a/web-console/src/utils/table-filters/table-filters.ts
+++ b/web-console/src/utils/table-filters/table-filters.ts
@@ -40,7 +40,7 @@ export class TableFilters {
static eq(keyValue: Record): TableFilters {
const filters = Object.entries(keyValue).map(
- ([key, value]) => new TableFilter(key, '=', value),
+ ([key, value]) => new TableFilter(key, '=', [value]),
);
return new TableFilters(filters);
}
@@ -62,12 +62,11 @@ export class TableFilters {
}
toString(): string {
+ // Encodes what would end the filter (&), the hash route's param (/) or the hash route (# and ?)
return this.filters
.map(
filter =>
- `${filter.key}${filter.mode}${filter.values
- .join('|')
- .replace(/[&%/]/g, encodeURIComponent)}`,
+ `${filter.key}${filter.mode}${filter.value.replace(/[#%&/?]/g, encodeURIComponent)}`,
)
.join('&');
}
diff --git a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
index b3ab62be6183..1c135d657d8c 100644
--- a/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
+++ b/web-console/src/views/sql-data-loader-view/schema-step/schema-step.tsx
@@ -1004,7 +1004,7 @@ export const SchemaStep = function SchemaStep(props: SchemaStepProps) {