test: upgrade e2e test framework

.github/workflows/linuxUI.yml

@@ -44,35 +44,13 @@ jobs:
     - name: Build VSIX file
       run: vsce package
 
-    - name: UI Test
-      continue-on-error: true
-      id: test
-      run: DISPLAY=:99 npm run test-ui
+    - name: E2E Test (Playwright)
+      run: DISPLAY=:99 npm run test-e2e
 
-    - name: Retry UI Test 1
-      continue-on-error: true
-      if: steps.test.outcome=='failure'
-      id: retry1
-      run: |
-        git reset --hard
-        git clean -fd
-        DISPLAY=:99 npm run test-ui
-
-    - name: Retry UI Test 2
-      continue-on-error: true
-      if: steps.retry1.outcome=='failure'
-      id: retry2
-      run: |
-        git reset --hard
-        git clean -fd
-        DISPLAY=:99 npm run test-ui
-
-    - name: Set test status
-      if: ${{ steps.test.outcome=='failure' && steps.retry1.outcome=='failure' && steps.retry2.outcome=='failure' }}
-      run: |
-        echo "Tests failed"
-        exit 1
-
-    - name: Print language server Log
-      if: ${{ failure() }}
-      run: find ./test-resources/settings/User/workspaceStorage/*/redhat.java/jdt_ws/.metadata/.log -print -exec cat '{}' \;;
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-results-linux
+        path: test-results/
+        retention-days: 7

.github/workflows/windowsUI.yml

@@ -44,35 +44,13 @@ jobs:
     - name: Build VSIX file
       run: vsce package
 
-    - name: UI Test
-      continue-on-error: true
-      id: test
-      run: npm run test-ui
+    - name: E2E Test (Playwright)
+      run: npm run test-e2e
 
-    - name: Retry UI Test 1
-      continue-on-error: true
-      if: steps.test.outcome=='failure'
-      id: retry1
-      run: |
-        git reset --hard
-        git clean -fd
-        npm run test-ui
-
-    - name: Retry UI Test 2
-      continue-on-error: true
-      if: steps.retry1.outcome=='failure'
-      id: retry2
-      run: |
-        git reset --hard
-        git clean -fd
-        npm run test-ui
-
-    - name: Set test status
-      if: ${{ steps.test.outcome=='failure' && steps.retry1.outcome=='failure' && steps.retry2.outcome=='failure' }}
-      run: |
-        echo "Tests failed"
-        exit 1
-
-    - name: Print language server Log if job failed
-      if: ${{ failure() }}
-      run: Get-ChildItem -Path ./test-resources/settings/User/workspaceStorage/*/redhat.java/jdt_ws/.metadata/.log | cat
+    - name: Upload test results
+      if: ${{ always() }}
+      uses: actions/upload-artifact@v4
+      with:
+        name: e2e-results-windows
+        path: test-results/
+        retention-days: 7

package.json

@@ -1263,6 +1263,7 @@
     "watch": "webpack --mode development --watch",
     "test": "tsc -p . && webpack --config webpack.config.js --mode development && node ./dist/test/index.js",
     "test-ui": "tsc -p . && webpack --config webpack.config.js --mode development && node ./dist/test/ui/index.js",
+    "test-e2e": "npx playwright test --config test/e2e/playwright.config.ts",
     "build-server": "node scripts/buildJdtlsExt.js",
     "vscode:prepublish": "tsc -p ./ && webpack --mode production",
     "tslint": "tslint -t verbose --project tsconfig.json"
@@ -1284,6 +1285,7 @@
     "tslint": "^6.1.3",
     "typescript": "^4.9.4",
     "vscode-extension-tester": "^8.23.0",
+    "@playwright/test": "^1.50.0",
     "webpack": "^5.105.0",
     "webpack-cli": "^4.10.0"
   },

test/e2e/fixtures/baseTest.ts

@@ -0,0 +1,201 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+/**
+ * Playwright test fixture that launches VS Code via Electron,
+ * opens a temporary copy of a test project, and tears everything
+ * down after the test.
+ *
+ * Usage in test files:
+ *
+ *   import { test, expect } from "../fixtures/baseTest";
+ *
+ *   test("my test", async ({ page }) => {
+ *       // `page` is a Playwright Page attached to VS Code
+ *   });
+ */
+
+import { _electron, test as base, type Page } from "@playwright/test";
+import { downloadAndUnzipVSCode } from "@vscode/test-electron";
+import * as fs from "fs-extra";
+import * as os from "os";
+import * as path from "path";
+
+export { expect } from "@playwright/test";
+
+// Root of the extension source tree
+const EXTENSION_ROOT = path.join(__dirname, "..", "..", "..");
+// Root of the test data projects
+const TEST_DATA_ROOT = path.join(EXTENSION_ROOT, "test");
+
+export type TestOptions = {
+    /** VS Code version to download, default "stable" */
+    vscodeVersion: string;
+    /** Relative path under `test/` to the project to open (e.g. "maven") */
+    testProjectDir: string;
+};
+
+type TestFixtures = TestOptions & {
+    /** Playwright Page connected to the VS Code Electron window */
+    page: Page;
+};
+
+export const test = base.extend<TestFixtures>({
+    vscodeVersion: [process.env.VSCODE_VERSION || "stable", { option: true }],
+    testProjectDir: ["maven", { option: true }],
+
+    page: async ({ vscodeVersion, testProjectDir }, use, testInfo) => {
+        // 1. Create a temp directory and copy the test project into it.
+        const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "java-dep-e2e-"));
+        const projectName = path.basename(testProjectDir);
+        const projectDir = path.join(tmpDir, projectName);
+        fs.copySync(path.join(TEST_DATA_ROOT, testProjectDir), projectDir);
+
+        // Write VS Code settings to suppress telemetry prompts and notification noise
+        const vscodeDir = path.join(projectDir, ".vscode");
+        fs.ensureDirSync(vscodeDir);
+        const settingsPath = path.join(vscodeDir, "settings.json");
+        let existingSettings: Record<string, unknown> = {};
+        if (fs.existsSync(settingsPath)) {
+            // settings.json may contain JS-style comments (JSONC), strip them before parsing
+            const raw = fs.readFileSync(settingsPath, "utf-8");
+            const stripped = raw.replace(/\/\/.*$/gm, "").replace(/\/\*[\s\S]*?\*\//g, "");
+            try {
+                existingSettings = JSON.parse(stripped);
+            } catch {
+                // If still invalid, start fresh — our injected settings are more important
+                existingSettings = {};
+            }
+        }
+        const mergedSettings = {
+            ...existingSettings,
+            "telemetry.telemetryLevel": "off",
+            "redhat.telemetry.enabled": false,
+            "workbench.colorTheme": "Default Dark Modern",
+            "update.mode": "none",
+            "extensions.ignoreRecommendations": true,
+        };
+        fs.writeFileSync(settingsPath, JSON.stringify(mergedSettings, null, 4));
+
+        // 2. Resolve VS Code executable.
+        const vscodePath = await downloadAndUnzipVSCode(vscodeVersion);
+        // resolveCliArgsFromVSCodeExecutablePath returns CLI-specific args
+        // (e.g. --ms-enable-electron-run-as-node) that are unsuitable for
+        // Electron UI launch. Extract only --extensions-dir and --user-data-dir.
+        const vscodeTestDir = path.join(EXTENSION_ROOT, ".vscode-test");
+        const extensionsDir = path.join(vscodeTestDir, "extensions");
+        const userDataDir = path.join(vscodeTestDir, "user-data");
+
+        // 3. Launch VS Code as an Electron app.
+        const electronApp = await _electron.launch({
+            executablePath: vscodePath,
+            env: { ...process.env, NODE_ENV: "development" },
+            args: [
+                "--no-sandbox",
+                "--disable-gpu-sandbox",
+                "--disable-updates",
+                "--skip-welcome",
+                "--skip-release-notes",
+                "--disable-workspace-trust",
+                "--password-store=basic",
+                // Suppress notifications that block UI interactions
+                "--disable-telemetry",
+                `--extensions-dir=${extensionsDir}`,
+                `--user-data-dir=${userDataDir}`,
+                `--extensionDevelopmentPath=${EXTENSION_ROOT}`,
+                projectDir,
+            ],
+        });
+
+        const page = await electronApp.firstWindow();
+
+        // Auto-dismiss Electron native dialogs (e.g. redhat.java refactoring
+        // confirmation, delete file confirmation). These dialogs are outside
+        // the renderer DOM and cannot be handled via Playwright Page API.
+        // Monkey-patch dialog.showMessageBox to find and click the confirm
+        // button by label, falling back to the first button.
+        await electronApp.evaluate(({ dialog }) => {
+            const confirmLabels = /^(OK|Delete|Move to Recycle Bin|Move to Trash)$/i;
+            dialog.showMessageBox = async (_win: any, opts: any) => {
+                const options = opts || _win;
+                const buttons: string[] = options?.buttons || [];
+                let idx = buttons.findIndex((b: string) => confirmLabels.test(b));
+                if (idx < 0) idx = 0;
+                return { response: idx, checkboxChecked: true };
+            };
+            dialog.showMessageBoxSync = (_win: any, opts: any) => {
+                const options = opts || _win;
+                const buttons: string[] = options?.buttons || [];
+                let idx = buttons.findIndex((b: string) => confirmLabels.test(b));
+                if (idx < 0) idx = 0;
+                return idx;
+            };
+        });
+
+        // Dismiss any startup notifications/dialogs before handing off to tests
+        await page.waitForTimeout(3_000);
+        await dismissAllNotifications(page);
+
+        // 4. Optional tracing
+        if (testInfo.retry > 0 || !process.env.CI) {
+            await page.context().tracing.start({ screenshots: true, snapshots: true, title: testInfo.title });
+        }
+
+        // ---- hand off to the test ----
+        await use(page);
+
+        // ---- teardown ----
+        // Save trace on failure/retry
+        if (testInfo.status !== "passed" || testInfo.retry > 0) {
+            const tracePath = testInfo.outputPath("trace.zip");
+            try {
+                await page.context().tracing.stop({ path: tracePath });
+                testInfo.attachments.push({ name: "trace", path: tracePath, contentType: "application/zip" });
+            } catch {
+                // Tracing may not have been started
+            }
+        }
+
+        await electronApp.close();
+
+        // Clean up temp directory
+        try {
+            fs.rmSync(tmpDir, { force: true, recursive: true });
+        } catch (e) {
+            console.warn(`Warning: failed to clean up ${tmpDir}: ${e}`);
+        }
+    },
+});
+
+/**
+ * Dismiss all VS Code notification toasts (telemetry prompts, theme suggestions, etc.).
+ * These notifications can steal focus and block Quick Open / Command Palette interactions.
+ */
+async function dismissAllNotifications(page: Page): Promise<void> {
+    try {
+        // Click "Clear All Notifications" if the notification center button is visible
+        const clearAll = page.locator(".notifications-toasts .codicon-notifications-clear-all, .notification-toast .codicon-close");
+        let count = await clearAll.count().catch(() => 0);
+        while (count > 0) {
+            await clearAll.first().click();
+            await page.waitForTimeout(500);
+            count = await clearAll.count().catch(() => 0);
+        }
+
+        // Also try the command palette approach as a fallback
+        const notificationToasts = page.locator(".notification-toast");
+        if (await notificationToasts.count().catch(() => 0) > 0) {
+            // Use keyboard shortcut to clear all notifications
+            await page.keyboard.press("Control+Shift+P");
+            const input = page.locator(".quick-input-widget input.input");
+            if (await input.isVisible({ timeout: 3_000 }).catch(() => false)) {
+                await input.fill("Notifications: Clear All Notifications");
+                await page.waitForTimeout(500);
+                await input.press("Enter");
+                await page.waitForTimeout(500);
+            }
+        }
+    } catch {
+        // Best effort
+    }
+}

test/e2e/globalSetup.ts

@@ -0,0 +1,34 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+import { downloadAndUnzipVSCode, resolveCliArgsFromVSCodeExecutablePath } from "@vscode/test-electron";
+import * as childProcess from "child_process";
+
+/**
+ * Global setup runs once before all test files.
+ * It downloads VS Code and installs the redhat.java extension so that
+ * every test run starts from an identical, pre-provisioned state.
+ *
+ * Our own extension is loaded at launch time via --extensionDevelopmentPath
+ * (see baseTest.ts), so there is no need to install a VSIX here.
+ */
+export default async function globalSetup(): Promise<void> {
+    // Download VS Code stable (or the version configured via VSCODE_VERSION env).
+    const vscodeVersion = process.env.VSCODE_VERSION || "stable";
+    console.log(`[globalSetup] Downloading VS Code ${vscodeVersion}…`);
+    const vscodePath = await downloadAndUnzipVSCode(vscodeVersion);
+    const [cli, ...cliArgs] = resolveCliArgsFromVSCodeExecutablePath(vscodePath);
+
+    // On Windows, the CLI is a .cmd batch file which requires shell: true.
+    const isWindows = process.platform === "win32";
+    const execOptions: childProcess.ExecFileSyncOptions = {
+        encoding: "utf-8",
+        stdio: "inherit",
+        timeout: 120_000,
+        shell: isWindows,
+    };
+
+    // Install the Language Support for Java extension from the Marketplace.
+    console.log("[globalSetup] Installing redhat.java extension…");
+    childProcess.execFileSync(cli, [...cliArgs, "--install-extension", "redhat.java"], execOptions);
+}

test/e2e/playwright.config.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+import { defineConfig } from "@playwright/test";
+import * as path from "path";
+
+export default defineConfig({
+    testDir: path.join(__dirname, "tests"),
+    reporter: process.env.CI
+        ? [["list"], ["junit", { outputFile: path.join(__dirname, "..", "..", "test-results", "e2e-results.xml") }]]
+        : "list",
+    // Java Language Server can take 2-3 minutes to fully index on first run.
+    timeout: 240_000,
+    // Run tests sequentially — launching multiple VS Code instances is too resource-heavy.
+    workers: 1,
+    // Allow one retry in CI to handle transient environment issues.
+    retries: process.env.CI ? 1 : 0,
+    expect: {
+        timeout: 30_000,
+    },
+    globalSetup: path.join(__dirname, "globalSetup.ts"),
+    use: {
+        // Automatically take a screenshot when a test fails.
+        screenshot: "only-on-failure",
+        // Capture full trace on retry for deep debugging (includes screenshots, DOM snapshots, network).
+        trace: "on-first-retry",
+    },
+    outputDir: path.join(__dirname, "..", "..", "test-results", "e2e"),
+});