perf(ci): experiment with two isolated iOS XCTest shards

.github/workflows/ios.yml

@@ -137,23 +137,80 @@ jobs:
           runtime-version: ${{ env.IOS_RUNTIME_VERSION }}
           preferred-device-name: iPhone 17 Pro
 
+      - name: Boot second iOS XCTest simulator
+        id: ios-xctest-second-simulator
+        if: steps.xctest-impact.outputs.run != 'false'
+        env:
+          FIRST_UDID: ${{ steps.ios-simulator.outputs.simulator-udid }}
+        run: |
+          set -euo pipefail
+          read -r RUNTIME DEVICE_TYPE < <(
+            xcrun simctl list devices -j | node -e '
+              const fs = require("node:fs");
+              const { devices } = JSON.parse(fs.readFileSync(0, "utf8"));
+              for (const [runtime, entries] of Object.entries(devices)) {
+                const device = entries.find((entry) => entry.udid === process.env.FIRST_UDID);
+                if (device?.deviceTypeIdentifier) {
+                  console.log(`${runtime} ${device.deviceTypeIdentifier}`);
+                  process.exit(0);
+                }
+              }
+              throw new Error("The primary XCTest simulator is missing from inventory");
+            '
+          )
+          SECOND_UDID="$(xcrun simctl create agent-device-xctest-shard-2 "$DEVICE_TYPE" "$RUNTIME")"
+          echo "simulator-udid=$SECOND_UDID" >> "$GITHUB_OUTPUT"
+          xcrun simctl boot "$SECOND_UDID"
+          perl -e 'alarm shift; exec @ARGV' 300 xcrun simctl bootstatus "$SECOND_UDID" -b
+
       - name: Run targeted iOS runner XCTest regressions
         id: ios-xctest
         if: steps.xctest-impact.outputs.run != 'false'
         run: |
           set -euo pipefail
           XCTESTRUN_PATH="$(find "$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Build/Products" -maxdepth 1 -name '*.xctestrun' -print -quit)"
           test -n "$XCTESTRUN_PATH"
-          IOS_PR_TEST_ARGS=()
           IOS_PR_TEST_IDS="$(node --experimental-strip-types scripts/check-xctest-selection.ts --ios-pr-tests)"
-          while IFS= read -r test_id; do
-            IOS_PR_TEST_ARGS+=("-only-testing:$test_id")
-          done <<< "$IOS_PR_TEST_IDS"
-          test "${#IOS_PR_TEST_ARGS[@]}" -gt 0
-          xcodebuild test-without-building \
-            -xctestrun "$XCTESTRUN_PATH" \
-            -destination "platform=iOS Simulator,id=${{ steps.ios-simulator.outputs.simulator-udid }}" \
-            "${IOS_PR_TEST_ARGS[@]}" 2>&1 | tee /tmp/agent-device-runner-regressions.log
+          test -n "$IOS_PR_TEST_IDS"
+          SIMULATORS=("${{ steps.ios-simulator.outputs.simulator-udid }}" "${{ steps.ios-xctest-second-simulator.outputs.simulator-udid }}")
+          LOG_DIR="$AGENT_DEVICE_IOS_RUNNER_DERIVED_PATH/Logs/PRXCTest"
+          mkdir -p "$LOG_DIR"
+          START_SECONDS=$SECONDS
+          PIDS=()
+          for SHARD_INDEX in 0 1; do
+            (
+              IOS_PR_TEST_ARGS=()
+              TEST_INDEX=0
+              while IFS= read -r test_id; do
+                if [ "$((TEST_INDEX % 2))" -eq "$SHARD_INDEX" ]; then
+                  IOS_PR_TEST_ARGS+=("-only-testing:$test_id")
+                fi
+                TEST_INDEX=$((TEST_INDEX + 1))
+              done <<< "$IOS_PR_TEST_IDS"
+              test "${#IOS_PR_TEST_ARGS[@]}" -gt 0
+              xcodebuild test-without-building \
+                -xctestrun "$XCTESTRUN_PATH" \
+                -destination "platform=iOS Simulator,id=${SIMULATORS[$SHARD_INDEX]}" \
+                -parallel-testing-enabled NO \
+                -resultBundlePath "$LOG_DIR/shard-$SHARD_INDEX.xcresult" \
+                "${IOS_PR_TEST_ARGS[@]}"
+            ) 2>&1 | tee "$LOG_DIR/shard-$SHARD_INDEX.log" &
+            PIDS+=("$!")
+          done
+          NATIVE_STATUS=0
+          for SHARD_INDEX in 0 1; do
+            if wait "${PIDS[$SHARD_INDEX]}"; then SHARD_STATUS=0; else SHARD_STATUS=$?; fi
+            echo "$SHARD_STATUS" > "$LOG_DIR/shard-$SHARD_INDEX.exit"
+            echo "XCTest shard $SHARD_INDEX exited $SHARD_STATUS"
+            if [ "$NATIVE_STATUS" -eq 0 ] && [ "$SHARD_STATUS" -ne 0 ]; then NATIVE_STATUS=$SHARD_STATUS; fi
+          done
+          echo "XCTest phase wall time: $((SECONDS - START_SECONDS))s" | tee -a "$GITHUB_STEP_SUMMARY"
+          cat "$LOG_DIR/shard-0.log" "$LOG_DIR/shard-1.log" > /tmp/agent-device-runner-regressions.log
+          SUMMARY_STATUS=0
+          node --experimental-strip-types scripts/xctest-run-summary.ts --ios-pr-logs \
+            "$LOG_DIR/shard-0.log" "$LOG_DIR/shard-1.log" || SUMMARY_STATUS=$?
+          if [ "$NATIVE_STATUS" -ne 0 ]; then exit "$NATIVE_STATUS"; fi
+          test "$SUMMARY_STATUS" -eq 0
           node --input-type=module -e '
             import { readFileSync } from "node:fs";
             const log = readFileSync("/tmp/agent-device-runner-regressions.log", "utf8");
@@ -162,6 +219,21 @@ jobs:
             }
           '
 
+      - name: Remove second iOS XCTest simulator
+        if: ${{ always() && steps.ios-xctest-second-simulator.outputs.simulator-udid != '' }}
+        env:
+          SECOND_UDID: ${{ steps.ios-xctest-second-simulator.outputs.simulator-udid }}
+        run: |
+          node --input-type=module -e '
+            import { execFileSync } from "node:child_process";
+            const id = process.env.SECOND_UDID;
+            const { devices } = JSON.parse(execFileSync("xcrun", ["simctl", "list", "devices", "-j"], { encoding: "utf8" }));
+            const device = Object.values(devices).flat().find((entry) => entry.udid === id);
+            if (!device) process.exit(0);
+            if (device.state === "Booted") execFileSync("xcrun", ["simctl", "shutdown", id], { stdio: "inherit" });
+            execFileSync("xcrun", ["simctl", "delete", id], { stdio: "inherit" });
+          '
+
       - name: Build CLI used by every smoke step
         run: pnpm gate build
 

docs/evidence/ios-xctest-sharding-pilot-2026-10-18.md

@@ -0,0 +1,78 @@
+# iOS XCTest sharding pilot
+
+The iOS runner PR lane executed 124 methods in 1,120.27 seconds of test-case time in
+[this CI run](https://github.com/callstack/agent-device/actions/runs/39151402218/job/117745626510).
+The complete XCTest step took 21 minutes 40 seconds.
+The run failed on the existing [snapshot prerequisite blocker](https://github.com/callstack/agent-device/issues/3800).
+These durations are timing evidence, not passing validation.
+
+## Local comparison
+
+The pilot used source `9beae21eaf1f041ddcff053cdbf0782f6133e023`, which contains the
+separate text-entry test fix in [PR #3802](https://github.com/callstack/agent-device/pull/3802).
+It used Xcode 26.2 (`17C52`), iOS 26.2, and two disposable iPhone 17 Pro simulators.
+The host had 12 CPU cores and 48 GiB of memory.
+
+Both modes used the same compiled artifacts, tests, assertions, and timeouts.
+Both simulators were booted before measurement.
+Measurements include XCTest process startup and result writing, but exclude simulator boot.
+Sequential execution used one process. Parallel execution used two independent processes,
+with nested Xcode parallelism disabled and distinct simulator UDIDs and result bundles.
+
+| Run order | Mode | Wall time | Passed | Failed/skipped | Exact selected IDs |
+| --- | --- | ---: | ---: | ---: | --- |
+| 1 | Sequential | 223.434 s | 12 | 0 | Yes |
+| 2 | Two processes | 128.093 s | 12 | 0 | Yes |
+| 3 | Two processes | 118.726 s | 12 | 0 | Yes |
+| 4 | Sequential | 196.802 s | 12 | 0 | Yes |
+
+The first pair had falling host load after fresh simulator boots.
+The reverse-order pair started at one-minute loads of 5.47 and 7.16, respectively.
+That pair reduced wall time by 39.7%.
+Two runs per mode are a pilot, not a stable estimate for hosted CI or the full suite.
+
+## Unchanged pilot selection
+
+The 12 slowest methods from the cited CI run were divided into two groups using that run's durations.
+Each group contained six methods. All use `AgentDeviceRunnerUITests/RunnerTests/`.
+
+| Process 0 | Process 1 |
+| --- | --- |
+| `testAlertAcceptDoesNotActivateAReplacementWithASharedButton` | `testAlertCannotProveAnIdenticalReplacementAndDoesNotActivateIt` |
+| `testSynthesizedReplacementLeavesAMaskedFieldUnverified` | `testSynthesizedReplacementClearsPastDroppedSelectAlls` |
+| `testAlertActivationConfirmsBeforeDeadlineWithANotificationBanner` | `testBareTypeUsesTappedInputWhenSoftwareKeyboardIsHidden` |
+| `testAbandonedTreeCaptureSkipsQuerySweepAndHonorsWarmupExemption` | `testAlertActivationDoesNotConsultInterruptionHandlingWithANotificationBanner` |
+| `testBareDelayedTypeFailsWhenTappedInputDisappearsMidCommand` | `testFillIntoAutoSubmittingDigitCountFieldReportsNoEvidenceFromItsSuccessor` |
+| `testPenalizedCoordinateFillAtAPointWithNoTextInputRefusesBeforeItsTapAndKeepsTheFocusedField` | `testSynthesizedReplacementPacesAnAppOwnedFieldAtItsAcknowledgeWindow` |
+
+## CI experiment
+
+The workflow experiment retains the complete generated PR selection.
+It assigns alternating sorted identifiers to two processes; it stores no timing baseline or test-name allowlist.
+Both workers must finish before the existing replay and daemon steps start.
+Each worker has its own simulator, log, exit status, and result bundle.
+The execution check rejects missing, unexpected, or repeated identifiers, including equally sized incorrect runs.
+Failed and skipped outcomes remain separate from method coverage.
+
+Full-suite hosted timing, the extra simulator boot cost, and failure behavior still need CI evidence.
+The local pilot does not justify relaxing the existing snapshot blocker or other assertions.
+
+## Execution checks
+
+The experiment branch starts from main `3d9190107e`, whose generated selection now contains 125 methods.
+The new reporter accepts the 124 completed records from a successful historical CI run against
+that run's source selection at `0fb25c474a`.
+Replacing one record with a duplicate keeps the count at 124 but fails the identity check.
+Replaying the old log against current source also fails because the new alert-title method is missing.
+
+Fault injection into the actual workflow shell confirms these outcomes:
+
+| Child outcomes | Workflow exit | Both children finish |
+| --- | ---: | --- |
+| Both succeed with complete execution | 0 | Yes |
+| First child exits 17 | 17 | Yes |
+| Second child exits 23 | 23 | Yes |
+| Both exit 0 but omit one selected method | 1 | Yes |
+
+The second-simulator creation and cleanup commands also ran on a disposable local seed simulator.
+They used its actual runtime and device type and deleted only the newly created simulator.

scripts/__tests__/xctest-run-summary.test.ts

@@ -3,7 +3,9 @@
 
 import { describe, expect, test } from 'vitest';
 import {
+  executionSelectionFailure,
   livenessFailure,
+  parseTestExecutions,
   MAX_FAILURE_TEXT,
   MAX_LISTED_FAILURES,
   renderSummary,
@@ -147,3 +149,95 @@ describe('the job summary', () => {
     expect(rendered).toContain('`RunnerTests/testY()`');
   });
 });
+
+describe('iOS PR XCTest execution coverage', () => {
+  const first = 'AgentDeviceRunnerUITests/RunnerTests/testFirst';
+  const second = 'AgentDeviceRunnerUITests/RunnerTests/testSecond';
+
+  function completed(method: string, status: 'passed' | 'failed' | 'skipped' = 'passed') {
+    return `Test Case '-[AgentDeviceRunnerUITests.RunnerTests ${method}]' ${status} (1.234 seconds).`;
+  }
+
+  test('reads completed method identities and outcomes without counting start or suite records', () => {
+    const log = [
+      "Test Case '-[AgentDeviceRunnerUITests.RunnerTests testFirst]' started.",
+      completed('testFirst'),
+      completed('testSecond', 'skipped'),
+      completed('testThird', 'failed'),
+      "Test Suite 'RunnerTests' failed.",
+    ].join('\n');
+    expect(parseTestExecutions(log)).toEqual([
+      { identifier: first, status: 'passed', seconds: 1.234 },
+      { identifier: second, status: 'skipped', seconds: 1.234 },
+      {
+        identifier: 'AgentDeviceRunnerUITests/RunnerTests/testThird',
+        status: 'failed',
+        seconds: 1.234,
+      },
+    ]);
+  });
+
+  test('accepts a complete disjoint execution regardless of shard or completion order', () => {
+    expect(
+      executionSelectionFailure(
+        [first, second],
+        [parseTestExecutions(completed('testSecond')), parseTestExecutions(completed('testFirst'))],
+      ),
+    ).toBeNull();
+  });
+
+  test('rejects an equally sized run that replaced a selected method with an unexpected method', () => {
+    const failure = executionSelectionFailure(
+      [first, second],
+      [parseTestExecutions(completed('testFirst')), parseTestExecutions(completed('testOther'))],
+    );
+    expect(failure).toContain(second);
+    expect(failure).toContain('AgentDeviceRunnerUITests/RunnerTests/testOther');
+  });
+
+  test('rejects repeats across shards even when total execution count matches selection', () => {
+    expect(
+      executionSelectionFailure(
+        [first, second],
+        [parseTestExecutions(completed('testFirst')), parseTestExecutions(completed('testFirst'))],
+      ),
+    ).not.toBeNull();
+  });
+
+  test('rejects repeated execution within a shard', () => {
+    expect(
+      executionSelectionFailure(
+        [first, second],
+        [
+          parseTestExecutions(`${completed('testFirst')}\n${completed('testFirst')}`),
+          parseTestExecutions(completed('testSecond')),
+        ],
+      ),
+    ).not.toBeNull();
+  });
+
+  test('rejects a shard whose xcodebuild matched no methods even if its exit was successful', () => {
+    expect(
+      executionSelectionFailure([first], [parseTestExecutions('** TEST EXECUTE SUCCEEDED **')]),
+    ).not.toBeNull();
+    expect(executionSelectionFailure([first], [])).not.toBeNull();
+  });
+
+  test('rejects an empty or duplicate generated selection', () => {
+    const observed = [parseTestExecutions(completed('testFirst'))];
+    expect(executionSelectionFailure([], observed)).not.toBeNull();
+    expect(executionSelectionFailure([first, first], observed)).not.toBeNull();
+  });
+
+  test('keeps coverage distinct from pass status when a selected method fails or skips', () => {
+    expect(
+      executionSelectionFailure(
+        [first, second],
+        [
+          parseTestExecutions(completed('testFirst', 'failed')),
+          parseTestExecutions(completed('testSecond', 'skipped')),
+        ],
+      ),
+    ).toBeNull();
+  });
+});

scripts/xctest-run-summary.ts

@@ -1,5 +1,4 @@
-// Renders a whole-bundle XCTest lane's job summary, and asserts the run executed exactly the
-// tests the source says that lane reaches.
+// Reports whole-bundle XCTest counts and exact method coverage across iOS PR shards.
 //
 // Two jobs in one place because they read the same file. `xcodebuild` exits 0 when a test
 // selection matches nothing, so "green" and "ran no tests" produce the same log tail: a
@@ -105,7 +104,85 @@ export function livenessFailure(summary: ResultSummary, expected: number): strin
   ].join('\n');
 }
 
+type TestExecution = {
+  identifier: string;
+  status: 'passed' | 'failed' | 'skipped';
+  seconds: number;
+};
+
+/** Read completed methods instead of trusting a successful xcodebuild exit. */
+export function parseTestExecutions(log: string): TestExecution[] {
+  const completed =
+    /Test Case '-\[([A-Za-z_]\w*)\.([A-Za-z_]\w*) ([A-Za-z_]\w*)\]' (passed|failed|skipped) \((\d+(?:\.\d+)?) seconds\)\./g;
+  return [...log.matchAll(completed)].map((match) => ({
+    identifier: `${match[1]}/${match[2]}/${match[3]}`,
+    status: match[4] as TestExecution['status'],
+    seconds: Number(match[5]),
+  }));
+}
+
+/** Count equality alone cannot detect a missing method replaced by a duplicate or another test. */
+export function executionSelectionFailure(
+  expected: readonly string[],
+  shards: readonly (readonly TestExecution[])[],
+): string | null {
+  const selected = new Set(expected);
+  if (expected.length === 0 || selected.size !== expected.length) {
+    return 'The generated iOS XCTest selection must be nonempty and unique.';
+  }
+  if (shards.length === 0 || shards.some((shard) => shard.length === 0)) {
+    return 'An iOS XCTest shard executed no identifiable test methods.';
+  }
+  const counts = new Map<string, number>();
+  for (const execution of shards.flat()) {
+    counts.set(execution.identifier, (counts.get(execution.identifier) ?? 0) + 1);
+  }
+  const missing = expected.filter((identifier) => !counts.has(identifier));
+  const unexpected = [...counts.keys()].filter((identifier) => !selected.has(identifier));
+  const repeated = [...counts].filter(([, count]) => count > 1).map(([identifier]) => identifier);
+  const errors = [
+    ['missing', missing],
+    ['unexpected', unexpected],
+    ['repeated', repeated],
+  ] as const;
+  const details = errors
+    .filter(([, identifiers]) => identifiers.length > 0)
+    .map(
+      ([kind, identifiers]) =>
+        `${kind} (${identifiers.length}): ${identifiers.slice(0, 10).join(', ')}`,
+    );
+  return details.length > 0 ? `iOS XCTest selection mismatch: ${details.join('; ')}` : null;
+}
+
+function summarizePrShards(paths: readonly string[]): number {
+  const expected = [...loadReport().reach.pr];
+  const shards = paths.map((file) => parseTestExecutions(fs.readFileSync(file, 'utf8')));
+  const executions = shards.flat();
+  const passed = executions.filter((entry) => entry.status === 'passed').length;
+  const failed = executions.filter((entry) => entry.status === 'failed').length;
+  const skipped = executions.filter((entry) => entry.status === 'skipped').length;
+  const failure = executionSelectionFailure(expected, shards);
+  const lines = [
+    '### iOS runner XCTest shards',
+    '',
+    `- executed: ${executions.length} of ${expected.length} selected methods`,
+    `- passed: ${passed}; failed: ${failed}; skipped: ${skipped}`,
+    ...shards.map(
+      (entries, index) =>
+        `- shard ${index}: ${entries.length} methods; ${entries.reduce((sum, entry) => sum + entry.seconds, 0).toFixed(1)}s in test cases`,
+    ),
+    `- exact method coverage: ${failure ? 'failed' : 'passed'}`,
+    '',
+  ];
+  const rendered = lines.join('\n');
+  process.stdout.write(rendered);
+  if (process.env.GITHUB_STEP_SUMMARY) fs.appendFileSync(process.env.GITHUB_STEP_SUMMARY, rendered);
+  if (failure) process.stderr.write(`${failure}\n`);
+  return failure || failed > 0 ? 1 : 0;
+}
+
 function main(): number {
+  if (process.argv[2] === '--ios-pr-logs') return summarizePrShards(process.argv.slice(3));
   const summaryPath = process.env.RESULT_SUMMARY_PATH;
   if (!summaryPath) throw new Error('RESULT_SUMMARY_PATH is not set.');
   const current: Lane = lane(process.env.XCTEST_LANE ?? '');