From 37678e03a526b686123c0744e00b59482a7c8758 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 5 Oct 2023 11:27:23 +0200 Subject: [PATCH 1/3] Reboot emulator on e2e tests --- tests/e2e/config.local.js | 6 ++-- tests/e2e/measure/writeTestStats.js | 1 + tests/e2e/testRunner.js | 45 ++++++++++-------------- tests/e2e/utils/checkEmulatorIsBooted.js | 6 ++++ tests/e2e/utils/rebootEmulator.js | 10 ++++++ tests/e2e/utils/sleep.js | 6 ++++ 6 files changed, 43 insertions(+), 31 deletions(-) create mode 100644 tests/e2e/utils/checkEmulatorIsBooted.js create mode 100644 tests/e2e/utils/rebootEmulator.js create mode 100644 tests/e2e/utils/sleep.js diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js index 0c38c3f1056f..8c9ba5a69e94 100644 --- a/tests/e2e/config.local.js +++ b/tests/e2e/config.local.js @@ -1,8 +1,6 @@ module.exports = { - APP_PACKAGE: 'com.expensify.chat.dev', - - WARM_UP_RUNS: 1, - RUNS: 8, + APP_PACKAGE: 'com.expensify.chat.adhoc', + RUNS: 1, APP_PATHS: { baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', diff --git a/tests/e2e/measure/writeTestStats.js b/tests/e2e/measure/writeTestStats.js index c4074dbdd08f..3af867e536f6 100644 --- a/tests/e2e/measure/writeTestStats.js +++ b/tests/e2e/measure/writeTestStats.js @@ -25,6 +25,7 @@ module.exports = (stats, path = OUTPUT_FILE_CURRENT) => { try { const content = JSON.parse(fs.readFileSync(path, 'utf8')); const line = `${JSON.stringify(content.concat([stats]))}\n`; + console.log(`ROPO writting results to ${path}`); fs.writeFileSync(path, line); } catch (error) { console.error(`Error writing ${path}`, error); diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 2a5aee78715f..0d21432df5b8 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -13,7 +13,6 @@ const defaultConfig = require('./config'); const compare = require('./compare/compare'); const Logger = require('./utils/logger'); const execAsync = require('./utils/execAsync'); -const killApp = require('./utils/killApp'); const launchApp = require('./utils/launchApp'); const createServerInstance = require('./server'); const installApp = require('./utils/installApp'); @@ -22,6 +21,9 @@ const writeTestStats = require('./measure/writeTestStats'); const withFailTimeout = require('./utils/withFailTimeout'); const reversePort = require('./utils/androidReversePort'); const getCurrentBranchName = require('./utils/getCurrentBranchName'); +const rebootEmulator = require('./utils/rebootEmulator'); +const sleep = require('./utils/sleep'); +const checkEmulatorIsBooted = require('./utils/checkEmulatorIsBooted'); const args = process.argv.slice(2); @@ -71,9 +73,18 @@ if (isDevMode) { Logger.note(`Running in development mode. Set baseline branch to same as current ${baselineBranch}`); } -const restartApp = async () => { - Logger.log('Killing app …'); - await killApp('android', config.APP_PACKAGE); +const restartApp = async (appPath) => { + Logger.log('Rebooting Emulator …'); + await rebootEmulator('android'); + let emulatorIsBooted = await checkEmulatorIsBooted(); + while (emulatorIsBooted.trim() !== '1') { + await sleep(1000); + emulatorIsBooted = await checkEmulatorIsBooted(); + } + // Give it a bit more time to stabilize + await sleep(5000); + await installApp('android', config.APP_PACKAGE, appPath); + await reversePort(); Logger.log('Launching app …'); await launchApp('android', config.APP_PACKAGE); }; @@ -127,8 +138,8 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { // Install app and reverse port let progressLog = Logger.progressInfo('Installing app and reversing port'); - await installApp('android', config.APP_PACKAGE, appPath); - await reversePort(); + // await installApp('android', config.APP_PACKAGE, appPath); + // await reversePort(); progressLog.done(); // Start the HTTP server @@ -169,33 +180,13 @@ const runTestsOnBranch = async (baselineOrCompare, branch) => { server.setTestConfig(testConfig); - const warmupLogs = Logger.progressInfo(`Running test '${testConfig.name}'`); - for (let warmUpRuns = 0; warmUpRuns < config.WARM_UP_RUNS; warmUpRuns++) { - const progressText = `(${testIndex + 1}/${numOfTests}) Warmup for test '${testConfig.name}' (iteration ${warmUpRuns + 1}/${config.WARM_UP_RUNS})`; - warmupLogs.updateText(progressText); - - await restartApp(); - - await withFailTimeout( - new Promise((resolve) => { - const cleanup = server.addTestDoneListener(() => { - Logger.log(`Warmup ${warmUpRuns + 1} done!`); - cleanup(); - resolve(); - }); - }), - progressText, - ); - } - warmupLogs.done(); - // We run each test multiple time to average out the results const testLog = Logger.progressInfo(''); for (let i = 0; i < config.RUNS; i++) { const progressText = `(${testIndex + 1}/${numOfTests}) Running test '${testConfig.name}' (iteration ${i + 1}/${config.RUNS})`; testLog.updateText(progressText); - await restartApp(); + await restartApp(appPath); // Wait for a test to finish by waiting on its done call to the http server try { diff --git a/tests/e2e/utils/checkEmulatorIsBooted.js b/tests/e2e/utils/checkEmulatorIsBooted.js new file mode 100644 index 000000000000..8bf9bbcda4d9 --- /dev/null +++ b/tests/e2e/utils/checkEmulatorIsBooted.js @@ -0,0 +1,6 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + // Use adb to kill the app + return execAsync(`adb shell getprop sys.boot_completed | tr -d '\r'`); +}; diff --git a/tests/e2e/utils/rebootEmulator.js b/tests/e2e/utils/rebootEmulator.js new file mode 100644 index 000000000000..b88f579e9a38 --- /dev/null +++ b/tests/e2e/utils/rebootEmulator.js @@ -0,0 +1,10 @@ +const execAsync = require('./execAsync'); + +module.exports = function (platform = 'android') { + if (platform !== 'android') { + throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`); + } + + // Use adb to kill the app + return execAsync(`adb -e reboot`); +}; diff --git a/tests/e2e/utils/sleep.js b/tests/e2e/utils/sleep.js new file mode 100644 index 000000000000..b73cf4947448 --- /dev/null +++ b/tests/e2e/utils/sleep.js @@ -0,0 +1,6 @@ +module.exports = function (ms) { + // Use adb to kill the app + return new Promise((resolve) => { + setTimeout(resolve, ms); + }); +}; From edf4816fe7ccb93fa28069cc0a5f2c720fb8ff21 Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 5 Oct 2023 15:10:32 +0200 Subject: [PATCH 2/3] Try starting the emulator fresh everytime --- src/libs/E2E/tests/appStartTimeTest.e2e.js | 10 ++++---- tests/e2e/config.local.js | 4 +-- tests/e2e/measure/math.js | 13 ++++++---- tests/e2e/measure/writeTestStats.js | 1 - tests/e2e/testRunner.js | 30 +++++++++++++++++++--- tests/e2e/utils/getDevices.js | 5 ++++ tests/e2e/utils/killEmulator.js | 5 ++++ tests/e2e/utils/listEmulators.js | 9 +++++++ tests/e2e/utils/startEmulator.js | 5 ++++ 9 files changed, 65 insertions(+), 17 deletions(-) create mode 100644 tests/e2e/utils/getDevices.js create mode 100644 tests/e2e/utils/killEmulator.js create mode 100644 tests/e2e/utils/listEmulators.js create mode 100644 tests/e2e/utils/startEmulator.js diff --git a/src/libs/E2E/tests/appStartTimeTest.e2e.js b/src/libs/E2E/tests/appStartTimeTest.e2e.js index d44fd6ce1195..b0973a68f4d8 100644 --- a/src/libs/E2E/tests/appStartTimeTest.e2e.js +++ b/src/libs/E2E/tests/appStartTimeTest.e2e.js @@ -5,11 +5,11 @@ import E2EClient from '../client'; const test = () => { // check for login (if already logged in the action will simply resolve) - E2ELogin().then((neededLogin) => { - if (neededLogin) { - // we don't want to submit the first login to the results - return E2EClient.submitTestDone(); - } + E2ELogin().then(() => { + // if (neededLogin) { + // // we don't want to submit the first login to the results + // return E2EClient.submitTestDone(); + // } console.debug('[E2E] Logged in, getting metrics and submitting them…'); diff --git a/tests/e2e/config.local.js b/tests/e2e/config.local.js index 8c9ba5a69e94..f55139b5d301 100644 --- a/tests/e2e/config.local.js +++ b/tests/e2e/config.local.js @@ -1,6 +1,6 @@ module.exports = { - APP_PACKAGE: 'com.expensify.chat.adhoc', - RUNS: 1, + APP_PACKAGE: 'com.expensify.chat.dev', + RUNS: 30, APP_PATHS: { baseline: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', compare: './android/app/build/outputs/apk/e2e/release/app-e2e-release.apk', diff --git a/tests/e2e/measure/math.js b/tests/e2e/measure/math.js index 712e197d367b..457aaf45804f 100644 --- a/tests/e2e/measure/math.js +++ b/tests/e2e/measure/math.js @@ -34,15 +34,18 @@ const std = (arr) => { }; const getStats = (entries) => { - const cleanedEntries = filterOutliersViaIQR(entries); - const meanDuration = mean(cleanedEntries); - const stdevDuration = std(cleanedEntries); + // TBD I don't think outliers should be discarded. On certain data sets such as user collection it might make sense. + // But in theory the app is a defined system. Discarting outliers will effectively hide extreme cases where app might hang + // or be stuck in a race condition + // const cleanedEntries = filterOutliersViaIQR(entries); + const meanDuration = mean(entries); + const stdevDuration = std(entries); return { mean: meanDuration, stdev: stdevDuration, - runs: cleanedEntries.length, - entries: cleanedEntries, + runs: entries.length, + entries, }; }; diff --git a/tests/e2e/measure/writeTestStats.js b/tests/e2e/measure/writeTestStats.js index 3af867e536f6..c4074dbdd08f 100644 --- a/tests/e2e/measure/writeTestStats.js +++ b/tests/e2e/measure/writeTestStats.js @@ -25,7 +25,6 @@ module.exports = (stats, path = OUTPUT_FILE_CURRENT) => { try { const content = JSON.parse(fs.readFileSync(path, 'utf8')); const line = `${JSON.stringify(content.concat([stats]))}\n`; - console.log(`ROPO writting results to ${path}`); fs.writeFileSync(path, line); } catch (error) { console.error(`Error writing ${path}`, error); diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js index 0d21432df5b8..4e0a08ad13df 100644 --- a/tests/e2e/testRunner.js +++ b/tests/e2e/testRunner.js @@ -21,9 +21,12 @@ const writeTestStats = require('./measure/writeTestStats'); const withFailTimeout = require('./utils/withFailTimeout'); const reversePort = require('./utils/androidReversePort'); const getCurrentBranchName = require('./utils/getCurrentBranchName'); -const rebootEmulator = require('./utils/rebootEmulator'); +const startEmulator = require('./utils/startEmulator'); +const killEmulator = require('./utils/killEmulator'); +const getDevices = require('./utils/getDevices'); const sleep = require('./utils/sleep'); const checkEmulatorIsBooted = require('./utils/checkEmulatorIsBooted'); +const listEmulators = require('./utils/listEmulators'); const args = process.argv.slice(2); @@ -74,15 +77,34 @@ if (isDevMode) { } const restartApp = async (appPath) => { - Logger.log('Rebooting Emulator …'); - await rebootEmulator('android'); + let list = await listEmulators(); + list = list.split('\n'); + + const emulatorName = list[0]; + + const devices = await getDevices(); + Logger.log('DEVICES'); + Logger.log(devices.trim().split('\n')); + if (devices.trim().split('\n').length > 1) { + killEmulator(); + await sleep(10000); + } + + if (!emulatorName) { + throw new Error('No emulator found'); + } + + await startEmulator(emulatorName); + // await sleep(5000); + let emulatorIsBooted = await checkEmulatorIsBooted(); while (emulatorIsBooted.trim() !== '1') { await sleep(1000); emulatorIsBooted = await checkEmulatorIsBooted(); } - // Give it a bit more time to stabilize + // Give it a bit more time to finish booting await sleep(5000); + Logger.log(`Installing app from ${appPath} …`); await installApp('android', config.APP_PACKAGE, appPath); await reversePort(); Logger.log('Launching app …'); diff --git a/tests/e2e/utils/getDevices.js b/tests/e2e/utils/getDevices.js new file mode 100644 index 000000000000..984bb71e5a9a --- /dev/null +++ b/tests/e2e/utils/getDevices.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + return execAsync(`adb devices`); +}; diff --git a/tests/e2e/utils/killEmulator.js b/tests/e2e/utils/killEmulator.js new file mode 100644 index 000000000000..9209961461d5 --- /dev/null +++ b/tests/e2e/utils/killEmulator.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function () { + return execAsync(`adb emu kill`); +}; diff --git a/tests/e2e/utils/listEmulators.js b/tests/e2e/utils/listEmulators.js new file mode 100644 index 000000000000..32a637a41264 --- /dev/null +++ b/tests/e2e/utils/listEmulators.js @@ -0,0 +1,9 @@ +const execAsync = require('./execAsync'); + +module.exports = function (platform = 'android') { + if (platform !== 'android') { + throw new Error(`rebootEmulator() missing implementation for platform: ${platform}`); + } + + return execAsync(`emulator -list-avds`); +}; diff --git a/tests/e2e/utils/startEmulator.js b/tests/e2e/utils/startEmulator.js new file mode 100644 index 000000000000..fa51a0db60ee --- /dev/null +++ b/tests/e2e/utils/startEmulator.js @@ -0,0 +1,5 @@ +const execAsync = require('./execAsync'); + +module.exports = function (emulatorName) { + return execAsync(`emulator @${emulatorName} -no-snapstorage > /dev/null 2>&1 &`); +}; From 08ecec25e28d0c0b49b816ac9a1fef2b9cd6658a Mon Sep 17 00:00:00 2001 From: Oscar Franco Date: Thu, 5 Oct 2023 15:39:28 +0200 Subject: [PATCH 3/3] Comment filteroutliersviaIQR function --- tests/e2e/measure/math.js | 40 +++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/tests/e2e/measure/math.js b/tests/e2e/measure/math.js index 457aaf45804f..6aee0b775614 100644 --- a/tests/e2e/measure/math.js +++ b/tests/e2e/measure/math.js @@ -1,25 +1,25 @@ const _ = require('underscore'); -const filterOutliersViaIQR = (data) => { - let q1; - let q3; - - const values = data.slice().sort((a, b) => a - b); - - if ((values.length / 4) % 1 === 0) { - q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]); - q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]); - } else { - q1 = values[Math.floor(values.length / 4 + 1)]; - q3 = values[Math.ceil(values.length * (3 / 4) + 1)]; - } - - const iqr = q3 - q1; - const maxValue = q3 + iqr * 1.5; - const minValue = q1 - iqr * 1.5; - - return _.filter(values, (x) => x >= minValue && x <= maxValue); -}; +// const filterOutliersViaIQR = (data) => { +// let q1; +// let q3; + +// const values = data.slice().sort((a, b) => a - b); + +// if ((values.length / 4) % 1 === 0) { +// q1 = (1 / 2) * (values[values.length / 4] + values[values.length / 4 + 1]); +// q3 = (1 / 2) * (values[values.length * (3 / 4)] + values[values.length * (3 / 4) + 1]); +// } else { +// q1 = values[Math.floor(values.length / 4 + 1)]; +// q3 = values[Math.ceil(values.length * (3 / 4) + 1)]; +// } + +// const iqr = q3 - q1; +// const maxValue = q3 + iqr * 1.5; +// const minValue = q1 - iqr * 1.5; + +// return _.filter(values, (x) => x >= minValue && x <= maxValue); +// }; const mean = (arr) => _.reduce(arr, (a, b) => a + b, 0) / arr.length;