diff --git a/docs/mobilecore-dual-app-qa.md b/docs/mobilecore-dual-app-qa.md index a3bf6c6..9d72a3f 100644 --- a/docs/mobilecore-dual-app-qa.md +++ b/docs/mobilecore-dual-app-qa.md @@ -246,14 +246,20 @@ python3 scripts/run_mobilecore_dual_app_qa.py \ --require-model-switch ``` -The counted physical-device lane must opt into both property-based physical-device enforcement and a sustained offline inference workload: +The counted physical-device lane must opt into property-based physical-device +enforcement, host-controlled background restriction/recovery, and a sustained +offline inference workload: ```bash python3 scripts/run_mobilecore_dual_app_qa.py \ --serial \ --model-file \ + --expected-mobilecore-cert-sha256 <64-hex-fingerprint> \ + --expected-mobilecode-cert-sha256 <64-hex-fingerprint> \ + --expected-mobilecode-test-cert-sha256 <64-hex-fingerprint> \ --require-model-switch \ --require-physical-device \ + --require-background-recovery \ --require-thermal \ --thermal-duration-seconds 900 \ --thermal-sample-seconds 15 @@ -268,8 +274,12 @@ image/audio cases to pass while the device is offline: ```bash python3 scripts/run_mobilecore_dual_app_qa.py \ --serial \ + --expected-mobilecore-cert-sha256 <64-hex-fingerprint> \ + --expected-mobilecode-cert-sha256 <64-hex-fingerprint> \ + --expected-mobilecode-test-cert-sha256 <64-hex-fingerprint> \ --require-model-switch \ --require-physical-device \ + --require-background-recovery \ --require-thermal \ --thermal-duration-seconds 900 \ --require-verified-omni @@ -285,6 +295,30 @@ rate. It never stores fixture bytes, prompts, response text, or data URIs. The v2 runner verifies `ro.kernel.qemu`, `ro.boot.qemu`, hardware, and model properties instead of trusting the adb serial prefix. During the thermal lane it keeps the device offline, repeatedly performs bounded local inference, and records only numeric temperature/status samples plus aggregate request and failure counts. Raw `dumpsys` output and inference responses are never written. +The background-recovery lane uses host-side ADB app-ops only on the dedicated +QA device. It snapshots the existing background modes, applies a temporary +restriction, verifies that the stopped MobileCore loopback service is +unavailable, restores the original modes, and requires a visible foreground +MobileCore recovery to reach `model_loaded=true`. The production apps never +edit app-ops or Android secure settings. Evidence contains booleans and step +digests only, not raw app-op output. + +On 2026-08-07, this lane passed once on the Android 16 ARM64 emulator: +both restriction commands were accepted and observed, the stopped service was +unavailable while restricted, the original app-ops were restored, and a visible +MobileCore foreground recovery returned to `model_loaded=true`. This validates +the harness and recovery sequence only; it is not physical-device background +evidence. + +Before a counted physical run installs anything, the runner uses `apksigner` +to compare all three APK certificate SHA-256 fingerprints with the explicitly +pinned values. It records only public certificate fingerprints and match +booleans, never keystore paths or passwords. The MobileCode app and its +instrumentation APK must use a compatible controlled QA signing identity; +this lane does not relabel that pair as the production-signed Release APK. The +official Release APK keeps its separate download, signature, cold-launch, and +TuiMa pairing evidence above. + Raw screenshots and sanitized logcat remain under the ignored `.qa-artifacts/` directory. The runner's manifest contains APK/model hashes, step digests, safe metrics, environment class, and redaction state; it does not persist model filenames, prompts, responses, images, audio, credentials, cookies, tokens, raw UI text, raw system dumps, or host paths. ## Verification @@ -321,7 +355,9 @@ Raw screenshots and sanitized logcat remain under the ignored `.qa-artifacts/` d - MobileCore local API instrumentation passed 2/2 tests covering model-ID control, incompatible-projector rejection, no-path response, multimodal contract, request-body consumption, cancellation, serialized inference, `runtime_busy`, and metrics counters. - A post-fix real-GGUF smoke reported active-model preflight `625617760` required bytes versus `1096425472` available bytes, `runtime=llama.cpp`, two completed requests, zero failures, and a non-zero average decode rate. - MobileCode `pureDebug` APK and cross-app Android test APK built successfully. -- The dual-app runner privacy/classification/thermal workload passes five deterministic host-side unit tests. +- The dual-app runner privacy, classification, multimodal, thermal, + background-recovery, and APK-signing paths pass 12 deterministic host-side + unit tests. - ActionEvidence inference-to-device linking passes focused unit coverage, including idempotency and action-type rejection. ## Remaining Release Gates diff --git a/scripts/run_mobilecore_dual_app_qa.py b/scripts/run_mobilecore_dual_app_qa.py index e174c34..17c5980 100644 --- a/scripts/run_mobilecore_dual_app_qa.py +++ b/scripts/run_mobilecore_dual_app_qa.py @@ -15,7 +15,9 @@ import argparse import hashlib import json +import os import re +import shutil import subprocess import sys import time @@ -47,6 +49,7 @@ "com.mobilecode.app.MobileCoreCrossAppQaTest#" "controlledLocalAudioQualityTasks" ) +BACKGROUND_APP_OPS = ("RUN_IN_BACKGROUND", "RUN_ANY_IN_BACKGROUND") _SENSITIVE_SNAPSHOT_KEYS = { "absolute_path", @@ -107,6 +110,48 @@ def sha256_file(path: Path) -> str: return digest.hexdigest() +def normalize_certificate_sha256(value: str) -> str: + normalized = re.sub(r"[^0-9a-f]", "", value.lower()) + if len(normalized) != 64: + raise ValueError("certificate SHA-256 must contain exactly 64 hex digits") + return normalized + + +def parse_apksigner_certificate_sha256(raw: str) -> str | None: + match = re.search( + r"Signer #1 certificate SHA-256 digest:\s*([0-9a-f:]+)", + raw, + re.IGNORECASE, + ) + if match is None: + return None + try: + return normalize_certificate_sha256(match.group(1)) + except ValueError: + return None + + +def resolve_apksigner(command: str) -> str | None: + explicit = Path(command).expanduser() + if explicit.is_file(): + return str(explicit) + discovered = shutil.which(command) + if discovered is not None: + return discovered + for variable in ("ANDROID_HOME", "ANDROID_SDK_ROOT"): + root_value = os.environ.get(variable) + if not root_value: + continue + build_tools = Path(root_value).expanduser() / "build-tools" + if not build_tools.is_dir(): + continue + for version in sorted(build_tools.iterdir(), reverse=True): + candidate = version / "apksigner" + if candidate.is_file(): + return str(candidate) + return None + + def sanitize_evidence_value(value: object) -> object: """Recursively remove payloads, credentials, and local absolute paths.""" if isinstance(value, dict): @@ -177,6 +222,21 @@ def parse_thermal_service(raw: str) -> tuple[int | None, float | None, int | Non return service_status, max_temperature, max_sensor_status +def parse_app_op_mode(raw: str, operation: str) -> str | None: + """Return one public Android app-op mode without retaining raw dumps.""" + for line in raw.splitlines(): + if operation.lower() not in line.lower(): + continue + match = re.search( + r":\s*(allow|ignore|deny|foreground|default)\b", + line, + re.IGNORECASE, + ) + if match is not None: + return match.group(1).lower() + return None + + def instrumentation_succeeded( *, returncode: int, @@ -212,6 +272,24 @@ def __init__(self, args: argparse.Namespace) -> None: "status": "not_run", "required": args.require_thermal, } + self.background_recovery_summary: dict[str, object] = { + "status": "not_run", + "required": bool( + getattr(args, "require_background_recovery", False) + ), + } + signing_required = any( + getattr(args, name, None) + for name in ( + "expected_mobilecore_cert_sha256", + "expected_mobilecode_cert_sha256", + "expected_mobilecode_test_cert_sha256", + ) + ) + self.apk_signing_summary: dict[str, object] = { + "status": "not_run", + "required": signing_required, + } def run( self, @@ -374,6 +452,17 @@ def wait_for_health(self, *, require_model: bool, timeout: int = 120) -> dict[st time.sleep(1) raise RuntimeError(f"MobileCore health timeout ({last_error})") + def wait_for_health_unavailable(self, *, timeout: int = 20) -> bool: + """Confirm the loopback service stopped without persisting response data.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + try: + self.api("GET", "/health", timeout=2) + except (OSError, ValueError, urllib.error.URLError): + return True + time.sleep(0.5) + return False + def dump_ui(self) -> ET.Element: self.adb( "dump_mobilecore_ui", @@ -475,7 +564,79 @@ def inspect_device_environment(self) -> None: "Physical-device acceptance was requested, but Android properties identify a non-physical target" ) + def inspect_apk_signatures(self) -> None: + inputs = ( + ( + "mobilecore", + self.args.mobilecore_apk, + getattr(self.args, "expected_mobilecore_cert_sha256", None), + ), + ( + "mobilecode", + self.args.mobilecode_apk, + getattr(self.args, "expected_mobilecode_cert_sha256", None), + ), + ( + "mobilecodeTest", + self.args.mobilecode_test_apk, + getattr(self.args, "expected_mobilecode_test_cert_sha256", None), + ), + ) + if not any(expected for _, _, expected in inputs): + self.apk_signing_summary = { + "status": "not_required", + "required": False, + } + return + + results: dict[str, object] = {} + all_matched = True + for label, path, expected in inputs: + normalized_expected = ( + normalize_certificate_sha256(expected) + if expected is not None + else None + ) + result = self.run( + f"verify_{label.lower()}_apk_signature", + [ + self.args.apksigner, + "verify", + "--verbose", + "--print-certs", + str(path), + ], + timeout=60, + required=False, + ) + observed = parse_apksigner_certificate_sha256( + result.stdout.decode("utf-8", "replace") + ) + matched = ( + result.returncode == 0 + and observed is not None + and normalized_expected is not None + and observed == normalized_expected + ) + all_matched = all_matched and matched + results[label] = { + "certificateSha256": observed, + "matchedExpected": matched, + } + + self.apk_signing_summary = { + "status": "passed" if all_matched else "failed", + "required": True, + "artifacts": results, + } + self.snapshot("apk_signing_summary", self.apk_signing_summary) + if not all_matched: + raise RuntimeError( + "APK signing identity did not match the pinned physical-QA certificate" + ) + def install_and_prepare(self) -> None: + self.inspect_apk_signatures() self.adb("wait_for_device", "wait-for-device") self.inspect_device_environment() for name, path in ( @@ -720,6 +881,204 @@ def exercise_lifecycle(self) -> None: restarted = self.wait_for_health(require_model=True, timeout=self.args.ready_timeout) self.snapshot("health_after_process_restart", restarted) + def exercise_background_recovery(self) -> None: + required = bool( + getattr(self.args, "require_background_recovery", False) + ) + if not required: + return + started = time.monotonic() + + original_modes: dict[str, str | None] = {} + restriction_commands_ok = True + restriction_observed = False + service_unavailable = False + recovery_ready = False + app_ops_restored = False + + for operation in BACKGROUND_APP_OPS: + result = self.adb( + f"read_{operation.lower()}_before_restriction", + "shell", + "cmd", + "appops", + "get", + MOBILECORE_PACKAGE, + operation, + required=False, + ) + original_modes[operation] = parse_app_op_mode( + result.stdout.decode("utf-8", "replace"), + operation, + ) + + try: + for operation in BACKGROUND_APP_OPS: + result = self.adb( + f"restrict_{operation.lower()}", + "shell", + "cmd", + "appops", + "set", + MOBILECORE_PACKAGE, + operation, + "ignore", + required=False, + ) + restriction_commands_ok = ( + restriction_commands_ok and result.returncode == 0 + ) + + observed_modes: list[str | None] = [] + for operation in BACKGROUND_APP_OPS: + result = self.adb( + f"read_{operation.lower()}_while_restricted", + "shell", + "cmd", + "appops", + "get", + MOBILECORE_PACKAGE, + operation, + required=False, + ) + observed_modes.append( + parse_app_op_mode( + result.stdout.decode("utf-8", "replace"), + operation, + ) + ) + restriction_observed = ( + restriction_commands_ok + and bool(observed_modes) + and all(mode == "ignore" for mode in observed_modes) + ) + + if restriction_observed: + self.adb( + "show_mobilecode_while_mobilecore_restricted", + "shell", + "am", + "start", + "-W", + "-n", + MOBILECODE_ACTIVITY, + ) + self.adb( + "stop_mobilecore_while_background_restricted", + "shell", + "am", + "force-stop", + MOBILECORE_PACKAGE, + ) + service_unavailable = self.wait_for_health_unavailable() + finally: + restore_commands_ok = True + for operation in BACKGROUND_APP_OPS: + restore_mode = original_modes[operation] or "default" + result = self.adb( + f"restore_{operation.lower()}", + "shell", + "cmd", + "appops", + "set", + MOBILECORE_PACKAGE, + operation, + restore_mode, + required=False, + ) + restore_commands_ok = ( + restore_commands_ok and result.returncode == 0 + ) + + restored_modes: list[str | None] = [] + for operation in BACKGROUND_APP_OPS: + result = self.adb( + f"read_{operation.lower()}_after_restore", + "shell", + "cmd", + "appops", + "get", + MOBILECORE_PACKAGE, + operation, + required=False, + ) + restored_modes.append( + parse_app_op_mode( + result.stdout.decode("utf-8", "replace"), + operation, + ) + ) + app_ops_restored = restore_commands_ok and all( + ( + restored_mode != "ignore" + if original_modes[operation] is None + else restored_mode == original_modes[operation] + ) + for operation, restored_mode in zip( + BACKGROUND_APP_OPS, + restored_modes, + ) + ) + + if service_unavailable and app_ops_restored: + try: + # Recovery is deliberately a visible foreground action. The + # production app never edits app-ops or secure settings. + self.tap_mobilecore_load() + recovered = self.wait_for_health( + require_model=True, + timeout=self.args.ready_timeout, + ) + recovery_ready = recovered.get("model_loaded") is True + except (OSError, RuntimeError, ValueError, urllib.error.URLError): + recovery_ready = False + + passed = all( + ( + restriction_commands_ok, + restriction_observed, + service_unavailable, + app_ops_restored, + recovery_ready, + ) + ) + self.background_recovery_summary = { + "status": "passed" if passed else "failed", + "required": required, + "restrictionCommandsAccepted": restriction_commands_ok, + "backgroundRestrictionObserved": restriction_observed, + "serviceUnavailableWhileRestricted": service_unavailable, + "appOpsRestored": app_ops_restored, + "recoveryAction": "visible_mobilecore_foreground_start", + "modelReadyAfterRecovery": recovery_ready, + "productionSettingsMutation": False, + } + self.snapshot( + "background_recovery_summary", + self.background_recovery_summary, + ) + self.steps.append( + Step( + name="background_restricted_recovery", + status="passed" if passed else "failed", + duration_ms=round((time.monotonic() - started) * 1000), + result_digest=sha256_bytes( + json.dumps( + self.background_recovery_summary, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + ), + failure_kind=None + if passed + else "background_recovery_failed", + ) + ) + if not passed: + raise RuntimeError( + "Required background-restriction recovery did not complete" + ) + def _capture_pressure_sample(self, elapsed_seconds: int) -> dict[str, object]: def capture(*arguments: str) -> str: try: @@ -940,7 +1299,9 @@ def write_manifest(self, *, status: str, error: str | None = None) -> None: "required": self.args.require_model_switch, "status": self.model_switch_status, }, + "apk_signing": self.apk_signing_summary, "multimodal": self.multimodal_summary, + "background_recovery": self.background_recovery_summary, "thermal_workload": self.thermal_summary, "redaction": "raw_prompts_media_credentials_and_host_paths_omitted", "apks": { @@ -976,6 +1337,7 @@ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--serial", required=True) parser.add_argument("--adb", default="adb") + parser.add_argument("--apksigner", default="apksigner") parser.add_argument( "--mobilecore-apk", type=Path, @@ -992,6 +1354,9 @@ def parse_args() -> argparse.Namespace: default=repository / "mobile_agent/build/app/outputs/apk/androidTest/pure/debug/app-pure-debug-androidTest.apk", ) parser.add_argument("--model-file", type=Path) + parser.add_argument("--expected-mobilecore-cert-sha256") + parser.add_argument("--expected-mobilecode-cert-sha256") + parser.add_argument("--expected-mobilecode-test-cert-sha256") parser.add_argument( "--require-model-switch", action="store_true", @@ -1017,6 +1382,14 @@ def parse_args() -> argparse.Namespace: action="store_true", help="Fail before installation when Android properties identify an emulator.", ) + parser.add_argument( + "--require-background-recovery", + action="store_true", + help=( + "Require host-controlled Android background restriction, service " + "interruption, app-op restoration, and visible foreground recovery." + ), + ) parser.add_argument( "--require-thermal", action="store_true", @@ -1037,6 +1410,35 @@ def parse_args() -> argparse.Namespace: parser.error(f"{label.replace('_', '-')} not found: {path}") if args.model_file is not None and not args.model_file.is_file(): parser.error(f"model-file not found: {args.model_file}") + fingerprint_names = ( + "expected_mobilecore_cert_sha256", + "expected_mobilecode_cert_sha256", + "expected_mobilecode_test_cert_sha256", + ) + for name in fingerprint_names: + value = getattr(args, name) + if value is None: + continue + try: + setattr(args, name, normalize_certificate_sha256(value)) + except ValueError as error: + parser.error(f"{name.replace('_', '-')}: {error}") + provided_fingerprints = [getattr(args, name) for name in fingerprint_names] + if any(provided_fingerprints) and not all(provided_fingerprints): + parser.error( + "APK certificate pinning requires all three expected SHA-256 values" + ) + if args.require_physical_device and not all( + provided_fingerprints + ): + parser.error( + "--require-physical-device requires all three expected APK certificate SHA-256 values" + ) + if any(getattr(args, name) for name in fingerprint_names): + resolved_apksigner = resolve_apksigner(args.apksigner) + if resolved_apksigner is None: + parser.error("apksigner was not found; pass --apksigner or set ANDROID_HOME") + args.apksigner = resolved_apksigner if args.thermal_duration_seconds < 0: parser.error("thermal-duration-seconds must be >= 0") if not 1 <= args.thermal_sample_seconds <= 60: @@ -1049,6 +1451,10 @@ def parse_args() -> argparse.Namespace: parser.error("--require-thermal requires --thermal-duration-seconds > 0") if args.require_physical_device and not args.require_thermal: parser.error("--require-physical-device requires --require-thermal") + if args.require_physical_device and not args.require_background_recovery: + parser.error( + "--require-physical-device requires --require-background-recovery" + ) return args @@ -1059,6 +1465,7 @@ def main() -> int: runner.install_and_prepare() runner.run_cross_app_tasks() runner.exercise_lifecycle() + runner.exercise_background_recovery() runner.exercise_thermal_workload() runner.capture_artifacts() runner.write_manifest(status="passed") diff --git a/scripts/test_run_mobilecore_dual_app_qa.py b/scripts/test_run_mobilecore_dual_app_qa.py index 48967dd..a89bfab 100644 --- a/scripts/test_run_mobilecore_dual_app_qa.py +++ b/scripts/test_run_mobilecore_dual_app_qa.py @@ -72,6 +72,35 @@ def test_pressure_parsers_return_safe_numeric_telemetry(self) -> None: self.assertEqual(qa.parse_battery_temperature_c(battery), 38.7) self.assertEqual(qa.parse_thermal_service(thermal), (2, 42.25, 2)) + def test_app_op_parser_returns_only_public_mode(self) -> None: + raw = """ +RUN_IN_BACKGROUND: allow; time=+10s ago +RUN_ANY_IN_BACKGROUND: ignore +""" + self.assertEqual( + qa.parse_app_op_mode(raw, "RUN_IN_BACKGROUND"), + "allow", + ) + self.assertEqual( + qa.parse_app_op_mode(raw, "RUN_ANY_IN_BACKGROUND"), + "ignore", + ) + self.assertIsNone(qa.parse_app_op_mode("No operations.", "RUN_IN_BACKGROUND")) + + def test_apksigner_parser_normalizes_public_certificate_fingerprint(self) -> None: + fingerprint = "ab" * 32 + raw = f"Signer #1 certificate SHA-256 digest: {fingerprint}\n" + self.assertEqual( + qa.parse_apksigner_certificate_sha256(raw), + fingerprint, + ) + self.assertEqual( + qa.normalize_certificate_sha256(":".join(["ab"] * 32)), + fingerprint, + ) + with self.assertRaises(ValueError): + qa.normalize_certificate_sha256("too-short") + def test_instrumentation_result_parser_rejects_junit_failure_with_zero_exit(self) -> None: failed = b""" INSTRUMENTATION_STATUS_CODE: -2 @@ -270,6 +299,90 @@ def api(*_args, **_kwargs): self.assertNotIn("Controlled sustained local QA", raw) self.assertNotIn("model-public-id", raw) + def test_background_recovery_restores_app_ops_and_requires_visible_recovery(self) -> None: + with tempfile.TemporaryDirectory() as directory: + args = argparse.Namespace( + output_dir=Path(directory), + forward_port=18080, + require_thermal=False, + require_background_recovery=True, + ready_timeout=30, + ) + runner = qa.QaRunner(args) + modes = {operation: "allow" for operation in qa.BACKGROUND_APP_OPS} + calls: list[str] = [] + + def adb(name, *arguments, **_kwargs): + calls.append(name) + if "appops" in arguments and "get" in arguments: + operation = arguments[-1] + stdout = f"{operation}: {modes[operation]}\n".encode() + return subprocess.CompletedProcess([], 0, stdout, b"") + if "appops" in arguments and "set" in arguments: + operation = arguments[-2] + modes[operation] = arguments[-1] + return subprocess.CompletedProcess([], 0, b"", b"") + + runner.adb = adb + runner.wait_for_health_unavailable = lambda **_: True + runner.tap_mobilecore_load = lambda: calls.append( + "visible_mobilecore_foreground_start" + ) + runner.wait_for_health = lambda **_: {"model_loaded": True} + + runner.exercise_background_recovery() + raw = (Path(directory) / "background_recovery_summary.json").read_text() + + self.assertEqual(runner.background_recovery_summary["status"], "passed") + self.assertTrue(runner.background_recovery_summary["appOpsRestored"]) + self.assertTrue(runner.background_recovery_summary["modelReadyAfterRecovery"]) + self.assertEqual( + modes, + {operation: "allow" for operation in qa.BACKGROUND_APP_OPS}, + ) + self.assertIn("visible_mobilecore_foreground_start", calls) + self.assertNotIn("com.mobilecore.app", raw) + + def test_apk_signature_gate_pins_every_physical_qa_artifact(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + apks = [] + for name in ("core.apk", "code.apk", "code-test.apk"): + path = root / name + path.write_bytes(name.encode()) + apks.append(path) + fingerprint = "12" * 32 + args = argparse.Namespace( + output_dir=root / "evidence", + forward_port=18080, + require_thermal=False, + require_physical_device=True, + require_background_recovery=True, + mobilecore_apk=apks[0], + mobilecode_apk=apks[1], + mobilecode_test_apk=apks[2], + expected_mobilecore_cert_sha256=fingerprint, + expected_mobilecode_cert_sha256=fingerprint, + expected_mobilecode_test_cert_sha256=fingerprint, + apksigner="apksigner", + ) + runner = qa.QaRunner(args) + + def run(*_args, **_kwargs): + output = ( + f"Signer #1 certificate SHA-256 digest: {fingerprint}\n" + ).encode() + return subprocess.CompletedProcess([], 0, output, b"") + + runner.run = run + runner.inspect_apk_signatures() + raw = (root / "evidence/apk_signing_summary.json").read_text() + + self.assertEqual(runner.apk_signing_summary["status"], "passed") + self.assertTrue(runner.apk_signing_summary["required"]) + self.assertEqual(len(runner.apk_signing_summary["artifacts"]), 3) + self.assertNotIn("core.apk", raw) + if __name__ == "__main__": unittest.main()