From 24012b7b48e5ad56bd34fa1eb15f1788f6521439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 21:08:10 +0200 Subject: [PATCH 1/2] perf(ci): experiment with two isolated iOS XCTest shards --- .github/workflows/ios.yml | 86 +++++++++++++++++-- .../ios-xctest-sharding-pilot-2026-10-18.md | 78 +++++++++++++++++ 2 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 docs/evidence/ios-xctest-sharding-pilot-2026-10-18.md diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index c138a600d9..9bcddb4fd2 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -137,6 +137,32 @@ jobs: runtime-version: ${{ env.IOS_RUNTIME_VERSION }} preferred-device-name: iPhone 17 Pro + - name: Boot second iOS XCTest simulator + id: ios-xctest-second-simulator + if: steps.xctest-impact.outputs.run != 'false' + env: + FIRST_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }} + run: | + set -euo pipefail + read -r RUNTIME DEVICE_TYPE < <( + xcrun simctl list devices -j | node -e ' + const fs = require("node:fs"); + const { devices } = JSON.parse(fs.readFileSync(0, "utf8")); + for (const [runtime, entries] of Object.entries(devices)) { + const device = entries.find((entry) => entry.udid === process.env.FIRST_UDID); + if (device?.deviceTypeIdentifier) { + console.log(`${runtime} ${device.deviceTypeIdentifier}`); + process.exit(0); + } + } + throw new Error("The primary XCTest simulator is missing from inventory"); + ' + ) + SECOND_UDID="$(xcrun simctl create agent-device-xctest-shard-2 "$DEVICE_TYPE" "$RUNTIME")" + echo "simulator-udid=$SECOND_UDID" >> "$GITHUB_OUTPUT" + xcrun simctl boot "$SECOND_UDID" + perl -e 'alarm shift; exec @ARGV' 300 xcrun simctl bootstatus "$SECOND_UDID" -b + - name: Run targeted iOS runner XCTest regressions id: ios-xctest if: steps.xctest-impact.outputs.run != 'false' @@ -144,16 +170,43 @@ jobs: set -euo pipefail XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)" test -n "$XCTESTRUN_PATH" - IOS_PR_TEST_ARGS=() IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)" - while IFS= read -r test_id; do - IOS_PR_TEST_ARGS+=("-only-testing:$test_id") - done <<< "$IOS_PR_TEST_IDS" - test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 - xcodebuild test-without-building \ - -xctestrun "$XCTESTRUN_PATH" \ - -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \ - "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log + test -n "$IOS_PR_TEST_IDS" + SIMULATORS=("${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.ios-xctest-second-simulator.outputs.simulator-udid }}") + LOG_DIR="$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Logs/PRXCTest" + mkdir -p "$LOG_DIR" + START_SECONDS=$SECONDS + PIDS=() + for SHARD_INDEX in 0 1; do + ( + IOS_PR_TEST_ARGS=() + TEST_INDEX=0 + while IFS= read -r test_id; do + if [ "$((TEST_INDEX % 2))" -eq "$SHARD_INDEX" ]; then + IOS_PR_TEST_ARGS+=("-only-testing:$test_id") + fi + TEST_INDEX=$((TEST_INDEX + 1)) + done <<< "$IOS_PR_TEST_IDS" + test "${#IOS_PR_TEST_ARGS[@]}" -gt 0 + xcodebuild test-without-building \ + -xctestrun "$XCTESTRUN_PATH" \ + -destination "platform=iOS Simulator,id=${SIMULATORS[$SHARD_INDEX]}" \ + -parallel-testing-enabled NO \ + -resultBundlePath "$LOG_DIR/shard-$SHARD_INDEX.xcresult" \ + "${IOS_PR_TEST_ARGS[@]}" + ) 2>&1 | tee "$LOG_DIR/shard-$SHARD_INDEX.log" & + PIDS+=("$!") + done + NATIVE_STATUS=0 + for SHARD_INDEX in 0 1; do + if wait "${PIDS[$SHARD_INDEX]}"; then SHARD_STATUS=0; else SHARD_STATUS=$?; fi + echo "$SHARD_STATUS" > "$LOG_DIR/shard-$SHARD_INDEX.exit" + echo "XCTest shard $SHARD_INDEX exited $SHARD_STATUS" + if [ "$NATIVE_STATUS" -eq 0 ] && [ "$SHARD_STATUS" -ne 0 ]; then NATIVE_STATUS=$SHARD_STATUS; fi + done + echo "XCTest phase wall time: $((SECONDS - START_SECONDS))s" | tee -a "$GITHUB_STEP_SUMMARY" + cat "$LOG_DIR/shard-0.log" "$LOG_DIR/shard-1.log" > /tmp/agent-device-runner-regressions.log + if [ "$NATIVE_STATUS" -ne 0 ]; then exit "$NATIVE_STATUS"; fi node --input-type=module -e ' import { readFileSync } from "node:fs"; const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); @@ -162,6 +215,21 @@ jobs: } ' + - name: Remove second iOS XCTest simulator + if: ${{ always() && steps.ios-xctest-second-simulator.outputs.simulator-udid != '' }} + env: + SECOND_UDID: ${{ steps.ios-xctest-second-simulator.outputs.simulator-udid }} + run: | + node --input-type=module -e ' + import { execFileSync } from "node:child_process"; + const id = process.env.SECOND_UDID; + const { devices } = JSON.parse(execFileSync("xcrun", ["simctl", "list", "devices", "-j"], { encoding: "utf8" })); + const device = Object.values(devices).flat().find((entry) => entry.udid === id); + if (!device) process.exit(0); + if (device.state === "Booted") execFileSync("xcrun", ["simctl", "shutdown", id], { stdio: "inherit" }); + execFileSync("xcrun", ["simctl", "delete", id], { stdio: "inherit" }); + ' + - name: Build CLI used by every smoke step run: pnpm gate build diff --git a/docs/evidence/ios-xctest-sharding-pilot-2026-10-18.md b/docs/evidence/ios-xctest-sharding-pilot-2026-10-18.md new file mode 100644 index 0000000000..1a2892e660 --- /dev/null +++ b/docs/evidence/ios-xctest-sharding-pilot-2026-10-18.md @@ -0,0 +1,78 @@ +# iOS XCTest sharding pilot + +The iOS runner PR lane executed 124 methods in 1,120.27 seconds of test-case time in +[this CI run](https://github.com/callstack/agent-device/actions/runs/39151402218/job/117745626510). +The complete XCTest step took 21 minutes 40 seconds. +The run failed on the existing [snapshot prerequisite blocker](https://github.com/callstack/agent-device/issues/3800). +These durations are timing evidence, not passing validation. + +## Local comparison + +The pilot used source `9beae21eaf1f041ddcff053cdbf0782f6133e023`, which contains the +separate text-entry test fix in [PR #3802](https://github.com/callstack/agent-device/pull/3802). +It used Xcode 26.2 (`17C52`), iOS 26.2, and two disposable iPhone 17 Pro simulators. +The host had 12 CPU cores and 48 GiB of memory. + +Both modes used the same compiled artifacts, tests, assertions, and timeouts. +Both simulators were booted before measurement. +Measurements include XCTest process startup and result writing, but exclude simulator boot. +Sequential execution used one process. Parallel execution used two independent processes, +with nested Xcode parallelism disabled and distinct simulator UDIDs and result bundles. + +| Run order | Mode | Wall time | Passed | Failed/skipped | Exact selected IDs | +| --- | --- | ---: | ---: | ---: | --- | +| 1 | Sequential | 223.434 s | 12 | 0 | Yes | +| 2 | Two processes | 128.093 s | 12 | 0 | Yes | +| 3 | Two processes | 118.726 s | 12 | 0 | Yes | +| 4 | Sequential | 196.802 s | 12 | 0 | Yes | + +The first pair had falling host load after fresh simulator boots. +The reverse-order pair started at one-minute loads of 5.47 and 7.16, respectively. +That pair reduced wall time by 39.7%. +Two runs per mode are a pilot, not a stable estimate for hosted CI or the full suite. + +## Unchanged pilot selection + +The 12 slowest methods from the cited CI run were divided into two groups using that run's durations. +Each group contained six methods. All use `AgentDeviceRunnerUITests/RunnerTests/`. + +| Process 0 | Process 1 | +| --- | --- | +| `testAlertAcceptDoesNotActivateAReplacementWithASharedButton` | `testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt` | +| `testSynthesizedReplacementLeavesAMaskedFieldUnverified` | `testSynthesizedReplacementClearsPastDroppedSelectAlls` | +| `testAlertActivationConfirmsBeforeDeadlineWithANotificationBanner` | `testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden` | +| `testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption` | `testAlertActivationDoesNotConsultInterruptionHandlingWithANotificationBanner` | +| `testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand` | `testFillIntoAutoSubmittingDigitCountFieldReportsNoEvidenceFromItsSuccessor` | +| `testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField` | `testSynthesizedReplacementPacesAnAppOwnedFieldAtItsAcknowledgeWindow` | + +## CI experiment + +The workflow experiment retains the complete generated PR selection. +It assigns alternating sorted identifiers to two processes; it stores no timing baseline or test-name allowlist. +Both workers must finish before the existing replay and daemon steps start. +Each worker has its own simulator, log, exit status, and result bundle. +The execution check rejects missing, unexpected, or repeated identifiers, including equally sized incorrect runs. +Failed and skipped outcomes remain separate from method coverage. + +Full-suite hosted timing, the extra simulator boot cost, and failure behavior still need CI evidence. +The local pilot does not justify relaxing the existing snapshot blocker or other assertions. + +## Execution checks + +The experiment branch starts from main `3d9190107e`, whose generated selection now contains 125 methods. +The new reporter accepts the 124 completed records from a successful historical CI run against +that run's source selection at `0fb25c474a`. +Replacing one record with a duplicate keeps the count at 124 but fails the identity check. +Replaying the old log against current source also fails because the new alert-title method is missing. + +Fault injection into the actual workflow shell confirms these outcomes: + +| Child outcomes | Workflow exit | Both children finish | +| --- | ---: | --- | +| Both succeed with complete execution | 0 | Yes | +| First child exits 17 | 17 | Yes | +| Second child exits 23 | 23 | Yes | +| Both exit 0 but omit one selected method | 1 | Yes | + +The second-simulator creation and cleanup commands also ran on a disposable local seed simulator. +They used its actual runtime and device type and deleted only the newly created simulator. From 027b43734b26134459be39310eeef0e61a1fd1ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Micha=C5=82=20Pierzcha=C5=82a?= Date: Sun, 18 Oct 2026 21:08:45 +0200 Subject: [PATCH 2/2] chore(gates): verify exact iOS XCTest shard execution --- .github/workflows/ios.yml | 4 + scripts/__tests__/xctest-run-summary.test.ts | 94 ++++++++++++++++++++ scripts/xctest-run-summary.ts | 81 ++++++++++++++++- 3 files changed, 177 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 9bcddb4fd2..72b6637778 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -206,7 +206,11 @@ jobs: done echo "XCTest phase wall time: $((SECONDS - START_SECONDS))s" | tee -a "$GITHUB_STEP_SUMMARY" cat "$LOG_DIR/shard-0.log" "$LOG_DIR/shard-1.log" > /tmp/agent-device-runner-regressions.log + SUMMARY_STATUS=0 + node --experimental-strip-types scripts/xctest-run-summary.ts --ios-pr-logs \ + "$LOG_DIR/shard-0.log" "$LOG_DIR/shard-1.log" || SUMMARY_STATUS=$? if [ "$NATIVE_STATUS" -ne 0 ]; then exit "$NATIVE_STATUS"; fi + test "$SUMMARY_STATUS" -eq 0 node --input-type=module -e ' import { readFileSync } from "node:fs"; const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8"); diff --git a/scripts/__tests__/xctest-run-summary.test.ts b/scripts/__tests__/xctest-run-summary.test.ts index 2905673da7..c0033ce604 100644 --- a/scripts/__tests__/xctest-run-summary.test.ts +++ b/scripts/__tests__/xctest-run-summary.test.ts @@ -3,7 +3,9 @@ import { describe, expect, test } from 'vitest'; import { + executionSelectionFailure, livenessFailure, + parseTestExecutions, MAX_FAILURE_TEXT, MAX_LISTED_FAILURES, renderSummary, @@ -147,3 +149,95 @@ describe('the job summary', () => { expect(rendered).toContain('`RunnerTests/testY()`'); }); }); + +describe('iOS PR XCTest execution coverage', () => { + const first = 'AgentDeviceRunnerUITests/RunnerTests/testFirst'; + const second = 'AgentDeviceRunnerUITests/RunnerTests/testSecond'; + + function completed(method: string, status: 'passed' | 'failed' | 'skipped' = 'passed') { + return `Test Case '-[AgentDeviceRunnerUITests.RunnerTests ${method}]' ${status} (1.234 seconds).`; + } + + test('reads completed method identities and outcomes without counting start or suite records', () => { + const log = [ + "Test Case '-[AgentDeviceRunnerUITests.RunnerTests testFirst]' started.", + completed('testFirst'), + completed('testSecond', 'skipped'), + completed('testThird', 'failed'), + "Test Suite 'RunnerTests' failed.", + ].join('\n'); + expect(parseTestExecutions(log)).toEqual([ + { identifier: first, status: 'passed', seconds: 1.234 }, + { identifier: second, status: 'skipped', seconds: 1.234 }, + { + identifier: 'AgentDeviceRunnerUITests/RunnerTests/testThird', + status: 'failed', + seconds: 1.234, + }, + ]); + }); + + test('accepts a complete disjoint execution regardless of shard or completion order', () => { + expect( + executionSelectionFailure( + [first, second], + [parseTestExecutions(completed('testSecond')), parseTestExecutions(completed('testFirst'))], + ), + ).toBeNull(); + }); + + test('rejects an equally sized run that replaced a selected method with an unexpected method', () => { + const failure = executionSelectionFailure( + [first, second], + [parseTestExecutions(completed('testFirst')), parseTestExecutions(completed('testOther'))], + ); + expect(failure).toContain(second); + expect(failure).toContain('AgentDeviceRunnerUITests/RunnerTests/testOther'); + }); + + test('rejects repeats across shards even when total execution count matches selection', () => { + expect( + executionSelectionFailure( + [first, second], + [parseTestExecutions(completed('testFirst')), parseTestExecutions(completed('testFirst'))], + ), + ).not.toBeNull(); + }); + + test('rejects repeated execution within a shard', () => { + expect( + executionSelectionFailure( + [first, second], + [ + parseTestExecutions(`${completed('testFirst')}\n${completed('testFirst')}`), + parseTestExecutions(completed('testSecond')), + ], + ), + ).not.toBeNull(); + }); + + test('rejects a shard whose xcodebuild matched no methods even if its exit was successful', () => { + expect( + executionSelectionFailure([first], [parseTestExecutions('** TEST EXECUTE SUCCEEDED **')]), + ).not.toBeNull(); + expect(executionSelectionFailure([first], [])).not.toBeNull(); + }); + + test('rejects an empty or duplicate generated selection', () => { + const observed = [parseTestExecutions(completed('testFirst'))]; + expect(executionSelectionFailure([], observed)).not.toBeNull(); + expect(executionSelectionFailure([first, first], observed)).not.toBeNull(); + }); + + test('keeps coverage distinct from pass status when a selected method fails or skips', () => { + expect( + executionSelectionFailure( + [first, second], + [ + parseTestExecutions(completed('testFirst', 'failed')), + parseTestExecutions(completed('testSecond', 'skipped')), + ], + ), + ).toBeNull(); + }); +}); diff --git a/scripts/xctest-run-summary.ts b/scripts/xctest-run-summary.ts index eaa14c63cd..363c521803 100644 --- a/scripts/xctest-run-summary.ts +++ b/scripts/xctest-run-summary.ts @@ -1,5 +1,4 @@ -// Renders a whole-bundle XCTest lane's job summary, and asserts the run executed exactly the -// tests the source says that lane reaches. +// Reports whole-bundle XCTest counts and exact method coverage across iOS PR shards. // // Two jobs in one place because they read the same file. `xcodebuild` exits 0 when a test // selection matches nothing, so "green" and "ran no tests" produce the same log tail: a @@ -105,7 +104,85 @@ export function livenessFailure(summary: ResultSummary, expected: number): strin ].join('\n'); } +type TestExecution = { + identifier: string; + status: 'passed' | 'failed' | 'skipped'; + seconds: number; +}; + +/** Read completed methods instead of trusting a successful xcodebuild exit. */ +export function parseTestExecutions(log: string): TestExecution[] { + const completed = + /Test Case '-\[([A-Za-z_]\w*)\.([A-Za-z_]\w*) ([A-Za-z_]\w*)\]' (passed|failed|skipped) \((\d+(?:\.\d+)?) seconds\)\./g; + return [...log.matchAll(completed)].map((match) => ({ + identifier: `${match[1]}/${match[2]}/${match[3]}`, + status: match[4] as TestExecution['status'], + seconds: Number(match[5]), + })); +} + +/** Count equality alone cannot detect a missing method replaced by a duplicate or another test. */ +export function executionSelectionFailure( + expected: readonly string[], + shards: readonly (readonly TestExecution[])[], +): string | null { + const selected = new Set(expected); + if (expected.length === 0 || selected.size !== expected.length) { + return 'The generated iOS XCTest selection must be nonempty and unique.'; + } + if (shards.length === 0 || shards.some((shard) => shard.length === 0)) { + return 'An iOS XCTest shard executed no identifiable test methods.'; + } + const counts = new Map(); + for (const execution of shards.flat()) { + counts.set(execution.identifier, (counts.get(execution.identifier) ?? 0) + 1); + } + const missing = expected.filter((identifier) => !counts.has(identifier)); + const unexpected = [...counts.keys()].filter((identifier) => !selected.has(identifier)); + const repeated = [...counts].filter(([, count]) => count > 1).map(([identifier]) => identifier); + const errors = [ + ['missing', missing], + ['unexpected', unexpected], + ['repeated', repeated], + ] as const; + const details = errors + .filter(([, identifiers]) => identifiers.length > 0) + .map( + ([kind, identifiers]) => + `${kind} (${identifiers.length}): ${identifiers.slice(0, 10).join(', ')}`, + ); + return details.length > 0 ? `iOS XCTest selection mismatch: ${details.join('; ')}` : null; +} + +function summarizePrShards(paths: readonly string[]): number { + const expected = [...loadReport().reach.pr]; + const shards = paths.map((file) => parseTestExecutions(fs.readFileSync(file, 'utf8'))); + const executions = shards.flat(); + const passed = executions.filter((entry) => entry.status === 'passed').length; + const failed = executions.filter((entry) => entry.status === 'failed').length; + const skipped = executions.filter((entry) => entry.status === 'skipped').length; + const failure = executionSelectionFailure(expected, shards); + const lines = [ + '### iOS runner XCTest shards', + '', + `- executed: ${executions.length} of ${expected.length} selected methods`, + `- passed: ${passed}; failed: ${failed}; skipped: ${skipped}`, + ...shards.map( + (entries, index) => + `- shard ${index}: ${entries.length} methods; ${entries.reduce((sum, entry) => sum + entry.seconds, 0).toFixed(1)}s in test cases`, + ), + `- exact method coverage: ${failure ? 'failed' : 'passed'}`, + '', + ]; + const rendered = lines.join('\n'); + process.stdout.write(rendered); + if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, rendered); + if (failure) process.stderr.write(`${failure}\n`); + return failure || failed > 0 ? 1 : 0; +} + function main(): number { + if (process.argv[2] === '--ios-pr-logs') return summarizePrShards(process.argv.slice(3)); const summaryPath = process.env.RESULT_SUMMARY_PATH; if (!summaryPath) throw new Error('RESULT_SUMMARY_PATH is not set.'); const current: Lane = lane(process.env.XCTEST_LANE ?? '');