test(functional): Add test for auto-reload after failed load

When auto-reload is enabled, and the developer accidentally breaks the
extension, then web-ext will try to reload the extension, which causes
it to be disabled in the browser. In the case of Chrome, the new
CDP-based method can successfully reload the extension (Chrome 126+).

The fallback for older Chrome does not work. This patch adds test
coverage for that scenario, and updates the fake-chrome-binary to
accurately simulate that scenario.

Tested with real Chrome 75, 131, 138, 139.

CHROME_PATH=/path/to/chrome75/chromium TEST_WEBEXT_USE_REAL_CHROME=1 ./node_modules/.bin/mocha tests/functional/test.cli.run-target-chromium.js

Author: Rob--W

tests/functional/common.js

@@ -131,10 +131,11 @@ export function execWebExt(argv, spawnOptions) {
 export function monitorOutput(spawnedProcess) {
   const callbacks = new Set();
   let outputData = '';
+  let errorData = '';
   function checkCallbacks() {
     for (const callback of callbacks) {
       const { outputTestFunc, resolve } = callback;
-      if (outputTestFunc(outputData)) {
+      if (outputTestFunc(outputData, errorData)) {
         callbacks.delete(callback);
         resolve();
       }
@@ -144,6 +145,10 @@ export function monitorOutput(spawnedProcess) {
     outputData += data;
     checkCallbacks();
   });
+  spawnedProcess.stderr.on('data', (data) => {
+    errorData += data;
+    checkCallbacks();
+  });
 
   const waitUntilOutputMatches = (outputTestFunc) => {
     return new Promise((resolve) => {

tests/functional/fake-chrome-binary.js

@@ -1,6 +1,6 @@
 #!/usr/bin/env node
 
-import { createReadStream, createWriteStream } from 'node:fs';
+import { createReadStream, createWriteStream, existsSync } from 'node:fs';
 import { readFile } from 'node:fs/promises';
 import http from 'node:http';
 import net from 'node:net';
@@ -185,6 +185,16 @@ async function handleChromeDevtoolsProtocolMessage(rawRequest) {
     }
     // No further validation: Unknown keys in params are accepted by Chrome.
 
+    if (!existsSync(path.join(request.params.path, 'manifest.json'))) {
+      return {
+        id,
+        error: {
+          code: -32600,
+          message: 'Manifest file is missing or unreadable',
+        },
+      };
+    }
+
     const { extensionId } = await fakeLoadChromExtension(request.params.path);
     return { id, result: { id: extensionId } };
   }
@@ -198,6 +208,7 @@ async function handleChromeDevtoolsProtocolMessage(rawRequest) {
 }
 
 let isAttachingToTarget;
+let isDisabledDueToLoadFailure;
 async function simulateResponsesForReloadViaDeveloperPrivate(request) {
   // Supports reloadAllExtensionsFallbackForChrome125andEarlier. This is NOT
   // the full real protocol response; just enough for the simulation to work.
@@ -232,6 +243,29 @@ async function simulateResponsesForReloadViaDeveloperPrivate(request) {
     }
     const extensionsToReload = Array.from(loadedFakeExtensions);
     for (const dir of extensionsToReload) {
+      if (!existsSync(path.join(dir, 'manifest.json'))) {
+        isDisabledDueToLoadFailure = true;
+        // In reality an exception has many more fields, but for simplicity we
+        // omit anything we don't already rely on.
+        return {
+          id,
+          result: {
+            exceptionDetails: {
+              text: 'Uncaught (in promise) Error: Manifest file is missing or unreadable',
+            },
+          },
+        };
+      }
+      if (isDisabledDueToLoadFailure) {
+        // In theory, one would expect to be able to reload an extension that
+        // failed to load. In practice, that is not the case, the developer
+        // has to manually enter chrome://extensions/, use the file picker to
+        // select the original extension. This weird issue is tracked at
+        // https://crbug.com/792277 (but even if it were to be fixed, we are
+        // never going to encounter the fix, since this code path is only
+        // taken for Chrome 125 and earlier, which will never be fixed).
+        continue;
+      }
       await fakeLoadChromExtension(dir);
     }
     return { id, result: { result: { value: extensionsToReload.length } } };

tests/functional/test.cli.run-target-chromium.js

@@ -26,7 +26,7 @@
 
 import path from 'node:path';
 import { existsSync } from 'node:fs';
-import { writeFile } from 'node:fs/promises';
+import { writeFile, rename } from 'node:fs/promises';
 
 import { describe, it, beforeEach, afterEach } from 'mocha';
 import { assert, expect } from 'chai';
@@ -148,6 +148,46 @@ describe('web-ext run -t chromium', () => {
         assert.equal(testServer.requestCount, 1, 'No reload with --no-reload');
       }
 
+      // Now test what happens if the extension becomes invalid,
+      // then valid again.
+      let isUsingFallbackForChrome125andEarlier = false;
+      if (expectReload) {
+        const manifestPathOld = path.join(srcDir, 'manifest.json');
+        const manifestPathNew = path.join(srcDir, 'manifest.json.deleted');
+        await rename(manifestPathOld, manifestPathNew);
+        await outputMonitor.waitUntilOutputMatches((output, errout) => {
+          if (errout.includes('Cannot load extension via CDP, falling back')) {
+            isUsingFallbackForChrome125andEarlier = true;
+            return true;
+          }
+          return errout.includes('Manifest file is missing or unreadable');
+        });
+        await rename(manifestPathNew, manifestPathOld);
+        if (isUsingFallbackForChrome125andEarlier) {
+          // When an extension is disabled due to a bad manifest, there is
+          // unfortunately no way to revive it, due to https://crbug.com/792277
+          assert.equal(
+            testServer.requestCount,
+            2,
+            'No reload after bad reload when using --load-extension',
+          );
+        } else {
+          if (testServer.requestCount === 2) {
+            await testServer.waitForHelloFromExtension();
+          }
+          assert.equal(testServer.requestCount, 3, 'Reloaded after bad load');
+        }
+        // Sanity check: When using fake-chrome-binary, verify that it falls
+        // back to --load-extension when we expect it to.
+        if (!useRealChrome) {
+          assert.equal(
+            isUsingFallbackForChrome125andEarlier,
+            chromeVersion < 126,
+            'isUsingFallbackForChrome125andEarlier only true in Chrome < 126',
+          );
+        }
+      }
+
       // Must send SIGINT so that chrome-launcher (used by web-ext) has a
       // chance to terminate the browser that it spawned. Using plain kill
       // can cause Chrome processes to be left behind.
@@ -166,7 +206,9 @@ describe('web-ext run -t chromium', () => {
       // Example of showing stderr from the last minute:
       // find /tmp/ -mmin 1 -name chrome-err.log -exec cat {} \; ;echo
       await cmd.waitForExit;
-      if (expectReload) {
+      if (expectReload && !isUsingFallbackForChrome125andEarlier) {
+        assert.equal(testServer.requestCount, 3, 'No unexpected requests');
+      } else if (expectReload && isUsingFallbackForChrome125andEarlier) {
         assert.equal(testServer.requestCount, 2, 'No unexpected requests');
       } else {
         assert.equal(testServer.requestCount, 1, 'No unexpected requests');