From fe3048a006a588b264833728edf6bd7aa9bc3954 Mon Sep 17 00:00:00 2001 From: Harzva <49864959+Harzva@users.noreply.github.com> Date: Sun, 21 Jun 2026 00:39:42 -0700 Subject: [PATCH] docs: add P6 runtime benchmark contracts --- .../p60-p62-runtime-benchmark-contract.md | 103 + .../run.json | 2110 +++++++++++++++++ .../runtime_benchmark_upgrade_contract.json | 356 +++ .../runtime_benchmark_upgrade_scoreboard.csv | 19 + .../runtime_benchmark_upgrade_verifier.json | 35 + ...ulti_agent_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...agent_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + ...ngle_agent_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...agent_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + ...ngle_agent_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...agent_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + ...l_verifier_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...ifier_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + ...ulti_agent_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...agent_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + ...ulti_agent_P60-BENCHMARK-TAXONOMY-001.json | 23 + ...agent_P61-COMMUNICATION-SUBSTRATE-001.json | 23 + ...ent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json | 23 + .../summary.md | 28 + ...mp2026.6.20-p6-mobile-agent-coevolution.md | 191 ++ docs/mobile-harness-roadmp/roadmp2026.6.20.md | 11 + ...ategy_p60_p62_runtime_benchmark_upgrade.py | 592 +++++ 27 files changed, 3859 insertions(+) create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_scoreboard.csv create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json create mode 100644 docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md create mode 100644 docs/mobile-harness-roadmp/roadmp2026.6.20-p6-mobile-agent-coevolution.md create mode 100644 scripts/run_mobile_harness_strategy_p60_p62_runtime_benchmark_upgrade.py diff --git a/docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md b/docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md new file mode 100644 index 0000000..5c76dd4 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md @@ -0,0 +1,103 @@ +# P6.0-P6.2 Runtime Benchmark Upgrade Contract + +This contract upgrades MobileHarnessBench from a small strategy pilot into a +mobile runtime benchmark scaffold. It is a contract artifact only: +`counts_as_experiment=false` and `run_kind=strategy_pilot_not_counted` remain +mandatory until a later promotion gate verifies real model, tool, device, +runtime, and privacy evidence. + +## P6.0 Task Taxonomy + +| Task category | Purpose | Required runtime assertions | Evidence refs | +| --- | --- | --- | --- | +| `ui_artifact` | Generated HTML or Flutter UI artifact behaves on a phone viewport. | keyboard, tap, screenshot, error absence | screenshot, verifier JSON | +| `webview_artifact` | Artifact behaves inside MobileCode WebView. | keyboard, tap, set_text, localStorage, WebView state, screenshot | WebView state, screenshot | +| `phone_use_permission` | App-owned phone-use capability is permission gated. | Accessibility state, dry probe, blocked reason, action schema | UI XML, logcat, verifier JSON | +| `file_intake` | Open/share/import flow preserves source metadata and content. | content URI state, file hash, UI confirmation, recovery path | intake log, UI XML | +| `local_runtime` | Local helper/runtime can execute or report a clear blocker. | process state, stdout/stderr summary, timeout, exit code | runtime report, logcat | +| `network_boundary` | Network-dependent task respects offline, proxy, and secret boundaries. | request block/allow decision, redaction, retry state | network summary, verifier JSON | +| `recovery_task` | Strategy recovers from verifier failure or blocked tooling. | retry/replan, blocked reason, recovery action, final status | strategy trace, verifier JSON | +| `real_device_task` | Android/iOS device lane proves install, launch, and runtime evidence. | install, launch, focus, screenshot, UI XML, logcat/device logs | device evidence directory | + +## Runtime Assertion Matrix + +| Assertion | Contract | Applies to | +| --- | --- | --- | +| `keyboard` | Dispatch Arrow/WASD or shortcut keys and assert visible/state change. | UI artifact, WebView artifact, games, editors | +| `tap` | Tap coordinates or accessibility node and assert target state changed. | WebView, phone-use, device lane | +| `swipe` | Swipe from/to coordinates and assert scroll, drawer, or canvas state changed. | phone-use, UI screens | +| `set_text` | Input text through DOM, WebView, or accessibility path and assert value/state. | forms, Kanban, chat/task input | +| `localStorage` | Read/write/refresh persistence assertion without leaking raw user data. | Web artifacts, WebView artifacts | +| `ui_xml` | Capture sanitized UI hierarchy and assert stable labels or state markers. | Android emulator/real device | +| `screenshot` | Capture nonblank image and inspect for target UI, not launcher/error page. | all runtime lanes | +| `logcat` | Scan app-scoped logs for fatal, ANR, Flutter, and plugin errors. | Android emulator/real device | +| `webview_state` | Query app-owned WebView URL, JS state, DOM markers, and console errors. | MobileCode WebView | +| `focus_state` | Assert foreground package/activity or iOS process state. | real device lanes | + +## Score Dimensions + +Scores are dimension reports, not counted benchmark rankings, until the +promotion gate passes. + +| Dimension | Weight | What it measures | +| --- | ---: | --- | +| `quality` | 25 | Functional completeness and requirement coverage. | +| `runtime_correctness` | 25 | Real interaction assertions and state changes. | +| `phone_use_ability` | 15 | Permission-gated observe/tap/swipe/set_text/back/home ability. | +| `recovery` | 15 | Retry, replan, blocked reason quality, and recovered verifier failures. | +| `latency_token` | 10 | Wall time, steps, model tokens, and tool I/O efficiency. | +| `safety_privacy` | 10 | Non-counted boundary, redaction, scope limits, and raw transcript avoidance. | + +## P6.1 Communication Substrate + +| Contract | Purpose | Required fields | +| --- | --- | --- | +| `MailboxMessage` | Role-to-role work packet with scoped context. | message_id, from_role, to_role, task_id, allowed_tools, budget, input_filter, expected_return, evidence_refs | +| `EvidenceLedgerEntry` | Shared blackboard entry for artifacts and verifier outputs. | evidence_id, kind, path, producer_role, redaction_state, created_at, summary | +| `RuntimeEvent` | EventBus record for device/WebView/runtime interactions. | event_id, source, target, action, status, timestamp, evidence_id, redaction_state | +| `MemoryCommitProposal` | Proposal-only memory write after verification. | proposal_id, source_trace, content_summary, ttl, redaction_state, approval_required | + +Role contracts: + +| Role | Allowlist | Budget | Input filter | Return contract | +| --- | --- | --- | --- | --- | +| `CodeAgent` | read_file, apply_patch, format, unit_test | max_steps 8 | code files and scoped task | patch summary, tests, blockers | +| `RuntimeAgent` | adb, devicectl, browser_cdp, webview_probe | max_steps 8 | runtime target and evidence policy | runtime events, screenshots/log refs | +| `PreviewAgent` | screenshot, ui_xml, webview_state | max_steps 6 | app-owned UI only | visual state, UI markers, blockers | +| `VerifierAgent` | validators, static checks, runtime assertions | max_steps 8 | artifacts and evidence refs | pass/fail, score dimensions, missing evidence | +| `MemoryAgent` | memory_packet, redaction, proposal | max_steps 4 | summaries only | proposal or no-op with reason | +| `ReporterAgent` | evidence_ledger, markdown_summary | max_steps 4 | verified evidence refs | public-safe summary | + +## P6.2 Runtime Verifier Scaffold + +The scaffold reuses P5.6 browser runtime verification and extends the contract +toward Android/WebView evidence. A verifier output must include: + +- `schema_version` +- `run_kind` +- `counts_as_experiment=false` +- `task_id` +- `task_category` +- `assertion_results` +- `score_dimensions` +- `device_evidence_refs` +- `webview_evidence_refs` +- `privacy_boundary` +- `blocked_reason` + +A non-counted run must include: + +- `run_kind=strategy_pilot_not_counted` +- `counts_as_experiment=false` +- one result per strategy per contract task +- `evidence.boundary=pilot_not_counted` +- no model/provider claim unless real callbacks were used and redacted +- no raw transcript, raw UI text from third-party apps, secrets, or private paths + +## P6.3 Android Real Device Lane Recommendation + +The next execution lane should install the latest APK on a real Android device +or dedicated emulator, grant/verify Accessibility manually where needed, run the +Mobile Phone Use dry/action probe, capture screenshot/UI XML/logcat/focus state, +and attach WebView state assertions for generated artifacts. This should remain +non-counted until repeated task samples and promotion gates pass. diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json new file mode 100644 index 0000000..cbba1f4 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json @@ -0,0 +1,2110 @@ +{ + "benchmark": "MobileHarnessBench", + "run_id": "p60-p62-runtime-benchmark-upgrade", + "created_at": "2026-06-21T07:28:20Z", + "counts_as_experiment": false, + "counts_as_strategy_ablation_result": false, + "run_kind": "strategy_pilot_not_counted", + "schema_version": "0.1.0-p60-p62-runtime-benchmark-upgrade", + "strategy_family": "mixed_strategy_ablation", + "evidence_boundary": "pilot_not_counted:p60_p62_runtime_benchmark_upgrade_contract_not_counted", + "environment": { + "execution_tier": "P6.0-P6.2-contract-scaffold", + "mode": "strategy_pilot_not_counted", + "model_provider": "none", + "model_name": "none", + "runtime_backend": "contract_materializer_no_device_actions", + "credential_source": "none" + }, + "mode": { + "name": "P6.0-P6.2 runtime benchmark upgrade", + "mode": "strategy_pilot_not_counted", + "non_counted_reason": "Contract/scaffold materialization only; no formal strategy benchmark." + }, + "strategies": [ + { + "strategy_id": "react_single_agent", + "strategy_family": "single_agent_reasoning", + "description": "One agent alternates think, act, and observe until completion or block." + }, + { + "strategy_id": "plan_execute_verify_single_agent", + "strategy_family": "single_agent_reasoning", + "description": "One agent creates a short plan, executes a step, verifies it, and retries or replans as needed." + }, + { + "strategy_id": "react_with_final_verifier", + "strategy_family": "single_agent_with_verifier", + "description": "A ReAct actor completes the task and a separate verifier checks the final artifact and trace." + }, + { + "strategy_id": "supervisor_handoff_multi_agent", + "strategy_family": "multi_agent_handoff", + "description": "A Supervisor plans and delegates typed HandoffPacket work to specialist mobile coding roles." + }, + { + "strategy_id": "swarm_router_multi_agent", + "strategy_family": "multi_agent_swarm", + "description": "A router selects the best specialist swarm by task category, device/runtime profile, and load." + }, + { + "strategy_id": "hierarchical_swarm_multi_agent", + "strategy_family": "multi_agent_swarm", + "description": "A manager decomposes the task and delegates to workers, then reconciles judged outputs." + } + ], + "task_subset": { + "name": "p60-p62-runtime-benchmark-upgrade", + "task_count": 3, + "tasks": [ + { + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "title": "P6.0 benchmark task taxonomy, runtime assertion matrix, and score dimensions", + "max_score": 100 + }, + { + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "title": "P6.1 mailbox, evidence ledger, runtime event bus, and memory proposal contracts", + "max_score": 100 + }, + { + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "title": "P6.2 runtime verifier JSON and non-counted run contract scaffold", + "max_score": 100 + } + ] + }, + "contract_summary": { + "taxonomy_categories": 8, + "runtime_assertions": [ + "focus_state", + "keyboard", + "localStorage", + "logcat", + "screenshot", + "set_text", + "swipe", + "tap", + "ui_xml", + "webview_state" + ], + "score_dimensions": { + "quality": 25, + "runtime_correctness": 25, + "phone_use_ability": 15, + "recovery": 15, + "latency_token": 10, + "safety_privacy": 10 + }, + "communication_substrate_contracts": [ + "EvidenceLedgerEntry", + "MailboxMessage", + "MemoryCommitProposal", + "RuntimeEvent" + ], + "runtime_verifier_upgrade": [ + "ui_xml", + "screenshot", + "logcat", + "webview_state", + "focus_state", + "accessibility_state" + ] + }, + "results": [ + { + "strategy_id": "react_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "react_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "react_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "plan_execute_verify_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "plan_execute_verify_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "plan_execute_verify_single_agent", + "strategy_family": "single_agent_reasoning", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "react_with_final_verifier", + "strategy_family": "single_agent_with_verifier", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "react_with_final_verifier", + "strategy_family": "single_agent_with_verifier", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "react_with_final_verifier", + "strategy_family": "single_agent_with_verifier", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "supervisor_handoff_multi_agent", + "strategy_family": "multi_agent_handoff", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "supervisor_handoff_multi_agent", + "strategy_family": "multi_agent_handoff", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "supervisor_handoff_multi_agent", + "strategy_family": "multi_agent_handoff", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "swarm_router_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "swarm_router_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "swarm_router_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "hierarchical_swarm_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "hierarchical_swarm_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + }, + { + "strategy_id": "hierarchical_swarm_multi_agent", + "strategy_family": "multi_agent_swarm", + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "status": "passed", + "strategy_trace": { + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null + }, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0 + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0 + }, + "effect_metrics": { + "task_success": 1.0, + "verified_success": 1.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": null, + "human_intervention_count": 0, + "handoff_success_rate": null, + "memory_reuse_score": null, + "steps_to_completion": 1 + }, + "evidence": { + "boundary": "pilot_not_counted", + "artifact_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md", + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + ], + "trace_paths": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json" + ], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used." + ], + "verifier_outputs": [ + "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + ], + "transcript_paths": [], + "human_intervention_notes": [] + }, + "pilot_verifier": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "verifier_output": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + }, + "pilot_score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract_refs": { + "task_taxonomy": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "communication_substrate": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "runtime_verifier_scaffold": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json" + }, + "counts_as_strategy_ablation_result": false + } + ], + "summary": { + "total": 18, + "strategies": 6, + "tasks_per_strategy": 3, + "passed": 18, + "warning": 0, + "failed": 0, + "blocked": 0, + "average_pilot_score_not_counted": 100.0 + }, + "score_boundary": "pilot_p60_p62_contract_score_not_counted" +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json new file mode 100644 index 0000000..0acf2f7 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json @@ -0,0 +1,356 @@ +{ + "schema_version": "0.1.0", + "contract_id": "p60_p62_runtime_benchmark_upgrade_contract", + "run_kind": "strategy_pilot_not_counted", + "counts_as_experiment": false, + "counts_as_strategy_ablation_result": false, + "task_taxonomy": [ + { + "task_category": "ui_artifact", + "required_assertions": [ + "keyboard", + "tap", + "screenshot", + "logcat" + ], + "evidence_refs": [ + "screenshot", + "verifier_json" + ] + }, + { + "task_category": "webview_artifact", + "required_assertions": [ + "keyboard", + "tap", + "set_text", + "localStorage", + "webview_state", + "screenshot" + ], + "evidence_refs": [ + "webview_state", + "screenshot", + "verifier_json" + ] + }, + { + "task_category": "phone_use_permission", + "required_assertions": [ + "tap", + "swipe", + "set_text", + "ui_xml", + "logcat", + "focus_state" + ], + "evidence_refs": [ + "ui_xml", + "logcat", + "accessibility_state" + ] + }, + { + "task_category": "file_intake", + "required_assertions": [ + "tap", + "set_text", + "ui_xml", + "screenshot" + ], + "evidence_refs": [ + "intake_log", + "content_hash", + "ui_xml" + ] + }, + { + "task_category": "local_runtime", + "required_assertions": [ + "logcat", + "screenshot", + "focus_state" + ], + "evidence_refs": [ + "runtime_report", + "exit_code", + "log_excerpt" + ] + }, + { + "task_category": "network_boundary", + "required_assertions": [ + "logcat", + "screenshot" + ], + "evidence_refs": [ + "network_summary", + "redaction_report" + ] + }, + { + "task_category": "recovery_task", + "required_assertions": [ + "keyboard", + "tap", + "screenshot", + "logcat" + ], + "evidence_refs": [ + "strategy_trace", + "blocked_reason", + "recovery_action" + ] + }, + { + "task_category": "real_device_task", + "required_assertions": [ + "tap", + "swipe", + "set_text", + "ui_xml", + "screenshot", + "logcat", + "focus_state" + ], + "evidence_refs": [ + "install_log", + "launch_log", + "ui_xml", + "screenshot", + "logcat" + ] + } + ], + "assertion_catalog": { + "keyboard": { + "state_proof": "visible_or_structured_state_change", + "required_redaction": "none" + }, + "tap": { + "state_proof": "target_state_changed", + "required_redaction": "coordinate_only_or_app_owned_label" + }, + "swipe": { + "state_proof": "scroll_drawer_or_canvas_state_changed", + "required_redaction": "coordinate_only" + }, + "set_text": { + "state_proof": "value_or_app_state_changed", + "required_redaction": "test_string_only" + }, + "localStorage": { + "state_proof": "key_roundtrip_or_refresh_persistence", + "required_redaction": "no_user_payload" + }, + "ui_xml": { + "state_proof": "stable_app_owned_label_or_state_marker", + "required_redaction": "third_party_text_removed" + }, + "screenshot": { + "state_proof": "nonblank_target_ui_not_launcher_or_error", + "required_redaction": "no_sensitive_screen" + }, + "logcat": { + "state_proof": "fatal_anr_flutter_plugin_scan", + "required_redaction": "app_scoped_or_sanitized" + }, + "webview_state": { + "state_proof": "url_dom_console_or_js_state_marker", + "required_redaction": "app_owned_state_only" + }, + "focus_state": { + "state_proof": "foreground_package_activity_or_process", + "required_redaction": "device_id_hash" + } + }, + "score_dimensions": { + "quality": 25, + "runtime_correctness": 25, + "phone_use_ability": 15, + "recovery": 15, + "latency_token": 10, + "safety_privacy": 10 + }, + "communication_substrate": { + "MailboxMessage": [ + "message_id", + "from_role", + "to_role", + "task_id", + "allowed_tools", + "budget", + "input_filter", + "expected_return", + "evidence_refs" + ], + "EvidenceLedgerEntry": [ + "evidence_id", + "kind", + "path", + "producer_role", + "redaction_state", + "created_at", + "summary" + ], + "RuntimeEvent": [ + "event_id", + "source", + "target", + "action", + "status", + "timestamp", + "evidence_id", + "redaction_state" + ], + "MemoryCommitProposal": [ + "proposal_id", + "source_trace", + "content_summary", + "ttl", + "redaction_state", + "approval_required" + ] + }, + "role_contracts": { + "CodeAgent": { + "allowed_tools": [ + "read_file", + "apply_patch", + "format", + "unit_test" + ], + "budget": { + "max_steps": 8 + }, + "input_filter": "code_files_and_scoped_task_only", + "return_contract": [ + "patch_summary", + "tests", + "blockers" + ] + }, + "RuntimeAgent": { + "allowed_tools": [ + "adb", + "devicectl", + "browser_cdp", + "webview_probe" + ], + "budget": { + "max_steps": 8 + }, + "input_filter": "runtime_target_and_evidence_policy", + "return_contract": [ + "runtime_events", + "screenshots", + "logs" + ] + }, + "PreviewAgent": { + "allowed_tools": [ + "screenshot", + "ui_xml", + "webview_state" + ], + "budget": { + "max_steps": 6 + }, + "input_filter": "app_owned_ui_only", + "return_contract": [ + "visual_state", + "ui_markers", + "blockers" + ] + }, + "VerifierAgent": { + "allowed_tools": [ + "validators", + "static_checks", + "runtime_assertions" + ], + "budget": { + "max_steps": 8 + }, + "input_filter": "artifacts_and_evidence_refs", + "return_contract": [ + "pass_fail", + "score_dimensions", + "missing_evidence" + ] + }, + "MemoryAgent": { + "allowed_tools": [ + "memory_packet", + "redaction", + "proposal" + ], + "budget": { + "max_steps": 4 + }, + "input_filter": "summaries_only", + "return_contract": [ + "proposal_or_noop_reason" + ] + }, + "ReporterAgent": { + "allowed_tools": [ + "evidence_ledger", + "markdown_summary" + ], + "budget": { + "max_steps": 4 + }, + "input_filter": "verified_evidence_refs_only", + "return_contract": [ + "public_safe_summary" + ] + } + }, + "runtime_verifier_scaffold": { + "verifier_json_required_fields": [ + "schema_version", + "run_kind", + "counts_as_experiment", + "task_id", + "task_category", + "assertion_results", + "score_dimensions", + "device_evidence_refs", + "webview_evidence_refs", + "privacy_boundary", + "blocked_reason" + ], + "non_counted_run_required_fields": [ + "run_kind", + "counts_as_experiment", + "counts_as_strategy_ablation_result", + "evidence_boundary", + "results", + "summary" + ], + "android_webview_upgrade_assertions": [ + "ui_xml", + "screenshot", + "logcat", + "webview_state", + "focus_state", + "accessibility_state" + ] + }, + "next_android_real_device_lane_recommendation": { + "status": "recommended_next", + "required_evidence": [ + "apk_install", + "launch_focus_state", + "accessibility_authorization_state", + "screenshot", + "ui_xml", + "app_scoped_logcat", + "phone_use_dry_probe", + "phone_use_action_probe", + "webview_state_assertions" + ], + "boundary": "non_counted_until_repeated_task_samples_and_promotion_gate" + } +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_scoreboard.csv b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_scoreboard.csv new file mode 100644 index 0000000..cd11c63 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_scoreboard.csv @@ -0,0 +1,19 @@ +task_id,strategy_id,status,score,counts_as_experiment +P60-BENCHMARK-TAXONOMY-001,react_single_agent,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,react_single_agent,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,react_single_agent,passed,100.0,False +P60-BENCHMARK-TAXONOMY-001,plan_execute_verify_single_agent,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,plan_execute_verify_single_agent,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,plan_execute_verify_single_agent,passed,100.0,False +P60-BENCHMARK-TAXONOMY-001,react_with_final_verifier,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,react_with_final_verifier,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,react_with_final_verifier,passed,100.0,False +P60-BENCHMARK-TAXONOMY-001,supervisor_handoff_multi_agent,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,supervisor_handoff_multi_agent,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,supervisor_handoff_multi_agent,passed,100.0,False +P60-BENCHMARK-TAXONOMY-001,swarm_router_multi_agent,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,swarm_router_multi_agent,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,swarm_router_multi_agent,passed,100.0,False +P60-BENCHMARK-TAXONOMY-001,hierarchical_swarm_multi_agent,passed,100.0,False +P61-COMMUNICATION-SUBSTRATE-001,hierarchical_swarm_multi_agent,passed,100.0,False +P62-RUNTIME-VERIFIER-SCAFFOLD-001,hierarchical_swarm_multi_agent,passed,100.0,False diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json new file mode 100644 index 0000000..0711ec6 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json @@ -0,0 +1,35 @@ +{ + "schema_version": "0.1.0", + "verifier_id": "p60_p62_runtime_benchmark_upgrade_verifier", + "created_at": "2026-06-21T07:28:20Z", + "status": "passed", + "run_kind": "strategy_pilot_not_counted", + "counts_as_experiment": false, + "counts_as_strategy_ablation_result": false, + "scope": "P6.0 taxonomy, P6.1 communication substrate, and P6.2 runtime verifier scaffold contract only.", + "score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0, + "max_score": 100, + "checks": { + "contract_doc_exists": true, + "taxonomy_has_required_categories": true, + "runtime_assertion_matrix_complete": true, + "score_dimensions_complete": true, + "score_weights_sum_100": true, + "mailbox_contract_present": true, + "evidence_ledger_contract_present": true, + "runtime_event_bus_contract_present": true, + "memory_commit_proposal_contract_present": true, + "role_contracts_cover_six_roles": true, + "runtime_verifier_json_contract_present": true, + "non_counted_run_contract_present": true, + "android_webview_upgrade_assertions_present": true, + "next_android_real_device_lane_recommendation_present": true, + "non_counted_boundary": true + }, + "blocked_reason": null + }, + "contract": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json", + "contract_doc": "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md" +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..83570b7 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..59fbc15 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..08d5d52 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "hierarchical_swarm_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_hierarchical_swarm_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for hierarchical_swarm_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..a0e4562 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..9ead3bf --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..895ae98 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "plan_execute_verify_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_plan_execute_verify_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for plan_execute_verify_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..85815ae --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..51fd69c --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..ca77114 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "react_single_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_single_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for react_single_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..d3e189b --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..d11e049 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..d27566d --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "react_with_final_verifier", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_react_with_final_verifier_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for react_with_final_verifier.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..1d46844 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..faeddbe --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..e2fab0c --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "supervisor_handoff_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_supervisor_handoff_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for supervisor_handoff_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001.json new file mode 100644 index 0000000..77068ee --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P60-BENCHMARK-TAXONOMY-001", + "summary": "P6.0 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json new file mode 100644 index 0000000..4a82bb1 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P61-COMMUNICATION-SUBSTRATE-001", + "summary": "P6.1 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json new file mode 100644 index 0000000..db0a66b --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/strategy_traces/swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001.json @@ -0,0 +1,23 @@ +{ + "trace_id": "strace_p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "strategy_id": "swarm_router_multi_agent", + "trace_status": "pilot_not_counted", + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": "2026-06-21T07:28:20Z", + "ended_at": "2026-06-21T07:28:20Z", + "tool_name": "p60_p62_contract_materializer", + "evidence_id": "p60-p62-runtime-benchmark-upgrade_swarm_router_multi_agent_P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "summary": "P6.2 contract scaffold materialized for swarm_router_multi_agent.", + "artifact_path": "docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json" + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": null +} diff --git a/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md new file mode 100644 index 0000000..45c7ce5 --- /dev/null +++ b/docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md @@ -0,0 +1,28 @@ +# P6.0-P6.2 Runtime Benchmark Upgrade + +- run_id: `p60-p62-runtime-benchmark-upgrade` +- run_kind: `strategy_pilot_not_counted` +- counts_as_experiment: `false` +- counts_as_strategy_ablation_result: `false` +- status: `passed` +- score_boundary: `pilot_p60_p62_contract_score_not_counted` +- total_score_not_counted: `100.0` + +## Evidence + +- Contract doc: `docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md` +- Contract JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json` +- Verifier JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json` +- Scoreboard: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_scoreboard.csv` +- Run JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json` + +## Scope + +- P6.0 defines task taxonomy, runtime assertions, and score dimensions. +- P6.1 defines mailbox, EvidenceLedger, RuntimeEventBus, MemoryCommitProposal, and role contracts. +- P6.2 defines runtime verifier JSON and non-counted run contracts, with Android/WebView evidence hooks. +- This run is a contract/scaffold proof only, not a formal benchmark. + +## Next P6.3 Android Real Device Lane + +Install the latest APK on a real Android device or dedicated emulator, verify Accessibility state, run Mobile Phone Use dry/action probes, capture screenshot/UI XML/logcat/focus state, and add WebView state assertions for generated artifacts. Keep the run non-counted until repeated samples and promotion gates pass. diff --git a/docs/mobile-harness-roadmp/roadmp2026.6.20-p6-mobile-agent-coevolution.md b/docs/mobile-harness-roadmp/roadmp2026.6.20-p6-mobile-agent-coevolution.md new file mode 100644 index 0000000..b6c1b5e --- /dev/null +++ b/docs/mobile-harness-roadmp/roadmp2026.6.20-p6-mobile-agent-coevolution.md @@ -0,0 +1,191 @@ +# P6 Mobile Agent Co-Evolution Roadmp + +目标：将 Mobile Harness 从单次策略对比推进到 mobile-first 智能体共进化系统，让 benchmark、推理策略、多智能体协同、phone-use/runtime verifier 和 MobileCode framework 一起升级。 + +## 范围 + +- 本文件是 P6 规划，不是已完成证据。 +- P6 结果在具备真实模型、真实工具、真实设备或 emulator、runtime verifier、截图/UI XML/logcat、token/model logs 前，仍必须保持 non-counted。 +- P6 可以参考 `https://github.com/Just-Agent/Oh-Reflective-loop-skills` 的 reflective loop 思想，但不得把外部仓库内容直接当成本项目已实现能力。 + +## 三个主方向 + +P6 需要同时推进三个正交方向。它们可以单独 ablation，也可以组合成完整策略。 + +| 方向 | 目标 | 当前基线 | P6 要补齐 | +| --- | --- | --- | --- | +| Single-agent 推理，纵向 | 提升单个 agent 的计划、行动、验证、恢复能力。 | ReAct、Plan-Execute-Verify、ReAct + FinalVerifier 已有 runner 和 dry trace。 | 接入真实 model/tool callback、runtime verifier、token/time 记录、失败恢复评分。 | +| Multi-agent 协同，横向 | 提升多个角色之间的分工、交接、互检能力。 | Supervisor/Handoff、SwarmRouter、HierarchicalSwarm 已有发布级 scaffold；Swarm 仍 feature-gated。 | 统一 mailbox、blackboard、event bus、judge/verifier、role budget 和协同评分。 | +| Loop engineering，闭环 | 让 benchmark、策略、framework 根据失败样本共同进化。 | P5.5/P5.6/P5.7/P5.8 已形成 non-counted pilot 和 phone-use gate。 | 建立 failure bank、strategy tournament、holdout eval、regression gate 和策略晋升规则。 | + +## 组合关系 + +P6 策略不再只用一个 strategy id 表达，而应拆成可组合配置： + +```text +MobileStrategy = + reasoning_mode + + coordination_topology + + loop_policy + + memory_policy + + tool_policy + + device_policy +``` + +示例组合： + +- `ReAct + no_handoff + simple_retry + redacted_memory + browser_tool + emulator_gate` +- `PEV + supervisor_mailbox + verifier_retry + proposal_memory + phone_use_tool + emulator_gate` +- `PEV/ReAct hybrid + hierarchical_swarm + reflective_failure_loop + evidence_ledger + phone_use/WebView tools + real_device_gate` + +P6 ablation 必须先控制变量，再测试组合策略： + +- [ ] 固定模型、任务、预算，只比较 single-agent 推理模式。 +- [ ] 固定推理模式，只比较 multi-agent 协同拓扑。 +- [ ] 固定推理和协同，只比较 loop policy。 +- [ ] 最后测试组合策略是否在 holdout tasks 上稳定胜出。 + +## 多智能体通信策略 + +P6 推荐使用混合通信层，不只采用 shared blackboard 或 mailbox。 + +| 通信方案 | 用途 | P6 规则 | +| --- | --- | --- | +| Mailbox / HandoffPacket | Supervisor 给 CodeAgent、RuntimeAgent、PreviewAgent、VerifierAgent、MemoryAgent、ReporterAgent 派发原子任务。 | 每次 handoff 必须有 typed packet、input filter、allowed tools、budget、return contract。 | +| Shared Blackboard / EvidenceLedger | 共享 artifact、screenshot、UI XML、logcat、verifier output、runtime state。 | 只放摘要和 evidence refs，不放 raw transcript、secret、private path。 | +| Runtime EventBus | 记录 device/emulator、Accessibility、adb、WebView、artifact verifier 的事件流。 | 每个事件必须有 schema、timestamp、source、status、evidence id。 | +| MemoryCommitProposal | 将失败模式、项目事实、用户偏好写入可审查提案。 | 默认 proposal-only；不得自动持久化 raw memory。 | +| Judge / Debate | 对多个候选 artifact 或修复方案做质量裁决。 | Judge 只读取 filtered trace summary 和 evidence refs。 | + +推荐默认拓扑： + +```text +Supervisor + -> mailbox handoff to specialist agents + -> blackboard/evidence ledger for shared proof + -> runtime event bus for phone-use/device state + -> verifier/judge for gate + -> memory proposal for durable learning +``` + +## Benchmark 与策略共进化 + +P6 采用 benchmark-adversarial co-evolution 思路： + +```text +Benchmark 生成更难的 mobile tasks + -> strategies 在相同模型/预算下执行 + -> runtime verifier 捕获真实失败 + -> failure bank 归类失败模式 + -> strategy designer 提出 mobile-first 改进 + -> MobileCode framework 补工具、权限、证据链和 UI + -> regression/holdout gate 验证是否真提升 +``` + +必须防止 benchmark overfitting： + +- [ ] Public eval 用于复现和回归。 +- [ ] Private holdout 用于策略晋升，不得被策略调参直接看到。 +- [ ] Pilot run 保持 `counts_as_experiment=false`。 +- [ ] Counted benchmark 只允许在 promotion gate 通过后生成。 +- [ ] 每次策略改动都要跑旧任务 regression，确认没有能力退化。 + +## P6 阶段计划 + +### P6.0 Benchmark Taxonomy Upgrade + +- [x] 增加 mobile task taxonomy：UI artifact、WebView artifact、phone-use permission、file intake、local runtime、network boundary、recovery task、real device task。 + - Evidence: `docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md` and `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json`。 +- [x] 为每类任务定义 runtime assertions：keyboard、tap、swipe、set_text、localStorage、UI XML、screenshot、logcat、WebView state。 + - Evidence: `runtime_benchmark_upgrade_contract.json` assertion catalog contains keyboard、tap、swipe、set_text、localStorage、UI XML、screenshot、logcat、WebView state、focus_state。 +- [x] 增加评分维度：quality、runtime correctness、phone-use ability、recovery、latency/token、safety/privacy。 + - Evidence: score dimensions weights sum to 100 in `runtime_benchmark_upgrade_contract.json`。 + +### P6.1 Communication Substrate + +- [x] 将 HandoffPacket 升级为 mailbox contract。 + - Evidence: `MailboxMessage` contract in `runtime_benchmark_upgrade_contract.json`。 +- [x] 增加 EvidenceLedger blackboard contract。 + - Evidence: `EvidenceLedgerEntry` contract in `runtime_benchmark_upgrade_contract.json`。 +- [x] 增加 RuntimeEventBus contract。 + - Evidence: `RuntimeEvent` contract in `runtime_benchmark_upgrade_contract.json`。 +- [x] 增加 MemoryCommitProposal contract。 + - Evidence: `MemoryCommitProposal` contract in `runtime_benchmark_upgrade_contract.json`。 +- [x] 为每个 role 固定 tool allowlist、budget、input filter、return contract。 + - Evidence: CodeAgent、RuntimeAgent、PreviewAgent、VerifierAgent、MemoryAgent、ReporterAgent role contracts in `runtime_benchmark_upgrade_contract.json`。 + +### P6.2 Runtime Interaction Verifier + +- [x] 定义 P6.2 runtime verifier scaffold，复用 P5.6 artifact verifier 并补齐 Android/WebView runtime assertion contract。 + - Evidence: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json`。 +- [x] 输出 verifier JSON contract 和 non-counted run contract。 + - Evidence: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json` validator passed，6 strategies、18 results、`run_kind=strategy_pilot_not_counted`、`counts_as_experiment=false`。 +- [ ] 对 Snake 验证 Arrow/WASD、移动、得分、暂停、重启。 +- [ ] 对 Kanban 验证输入任务、快捷键、timer、刷新后 localStorage。 +- [ ] 对 Maze 验证点击设墙、起点终点、Solve、路径长度变化。 +- [ ] 输出真实交互 screenshots、UI XML、logcat、WebView state 和 runtime score。 + - Note: P6.0-P6.2 只完成 contract/scaffold；真实 Android/WebView 交互证据进入 P6.3/P6.4 gate。 + +### P6.3 Real Mac / Real Android Device Lane + +- [ ] 建立 device registry：serial、device tier、Android version、screen size、permission status。 +- [ ] 支持 APK install、launch、Accessibility 授权检查、screenshot、UI dump、logcat、focus state。 +- [ ] 明确 emulator evidence 和 real-device evidence 的区别，不得互相替代。 +- [ ] 真机授权可先采用人工步骤，但必须保存授权状态和复测证据。 + +### P6.4 Strategy Tournament + +- [ ] 固定同一模型、同一任务集、同一预算。 +- [ ] 跑 ReAct、PEV、ReAct + FinalVerifier、Supervisor/Handoff、SwarmRouter、HierarchicalSwarm。 +- [ ] 同时报告质量分、runtime 分、phone-use 分、恢复分、效率分。 +- [ ] 输出 strategy matrix、efficiency scoreboard、failure taxonomy、evidence package。 + +### P6.5 Mobile-First Strategy Proposal + +- [ ] 基于失败样本提出 Mobile Evidence-Gated Reflective Swarm。 +- [ ] 默认结构：PEV outer loop、ReAct inner loop、Supervisor mailbox、EvidenceLedger blackboard、runtime verifier gate、Memory proposal。 +- [ ] 针对 mobile 优化：权限先验、UI tree grounding、tap/text/swipe recovery、WebView/localStorage verifier、real-device drift handling。 +- [ ] 只有 holdout 胜出且安全边界完整，才允许作为候选默认策略。 + +### P6.6 Promotion Gate + +- [ ] 检查 `counts_as_experiment=false` 是否只用于 pilot。 +- [ ] 检查 counted result 是否具备 model logs、token records、tool evidence、device evidence、verifier outputs、screenshots。 +- [ ] secret/path/raw transcript scan 必须通过。 +- [ ] Swarm 相关策略在未通过 gate 前继续 feature-gated。 + +## Evidence / 已完成证据 + +- 2026-06-20 P5.8 已完成 Android phone-use runtime gate；它只证明 phone-use runtime gate 和证据链，不证明六个推理框架质量差异。 +- 2026-06-20 P5.8 `phone_use_runtime_verifier.json` 为 non-counted runtime score。 +- 2026-06-21 P6.0-P6.2 runtime benchmark upgrade contract 已生成： + - Contract doc: `docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md`。 + - Contract JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json`。 + - Verifier JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json`。 + - Run JSON: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json`。 + - Summary: `docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md`。 + - 2026-06-21 strategy validator passed：6 strategies、18 results、`run_kind=strategy_pilot_not_counted`。 + - Boundary: `counts_as_experiment=false`、`counts_as_strategy_ablation_result=false`；这是 contract/scaffold proof，不是正式 benchmark。 + +## Open Questions + +- [ ] P6 是否先以 emulator 作为 primary lane，真机作为 release gate。 +- [ ] P6 strategy tournament 使用哪个固定模型和预算。 +- [ ] Private holdout tasks 放在公开仓库还是本地受控目录。 +- [ ] Real-device Accessibility 授权是否允许半自动化，还是必须人工确认。 + +## Test Plan + +- `flutter test test/services/ test/widgets/strategy_mode_card_test.dart test/widgets/phone_use_mode_card_test.dart` +- `flutter analyze` 针对 P6 新增 Dart 文件。 +- `python3 -m py_compile` 针对 P6 新增 verifier/runner scripts。 +- Strategy validator 覆盖 P5.8、P6 pilot、promotion gate。 +- Android emulator QA 覆盖 install、launch、Tools 页、phone-use dry/action probe。 +- Real Android device QA 覆盖 install、launch、Accessibility、runtime verifier、evidence package。 + +## Assumptions + +- P6 继续遵守 non-counted 边界。 +- 真机和 emulator 都由外部 harness 通过 adb 或等价 device channel 控制。 +- App 内部 phone-use 必须受 Accessibility 权限门控。 +- MobileCode 不提交 secret、private path、raw transcript、credential dump。 diff --git a/docs/mobile-harness-roadmp/roadmp2026.6.20.md b/docs/mobile-harness-roadmp/roadmp2026.6.20.md index 442f67d..29de069 100644 --- a/docs/mobile-harness-roadmp/roadmp2026.6.20.md +++ b/docs/mobile-harness-roadmp/roadmp2026.6.20.md @@ -255,3 +255,14 @@ git diff --check - `window-strategy-swarm-details.xml` - App-scoped logcat：`logcat-app-after-details.txt`，未发现 `FATAL EXCEPTION`、`E/flutter`、`ANR`、`MissingPluginException`。 - 该 QA 结果不是正式 benchmark，不得写入 counted strategy ablation result。 + +2026-06-21 P6.0-P6.2 runtime benchmark upgrade evidence: +- P6.0 taxonomy、P6.1 communication substrate、P6.2 verifier scaffold 已生成 contract/scaffold 证据。 +- Contract doc：`docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md`。 +- Contract JSON：`docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_contract.json`。 +- Verifier JSON：`docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/runtime_benchmark_upgrade_verifier.json`。 +- Run JSON：`docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/run.json`。 +- Summary：`docs/mobile-harness-benchmark/strategy-ablation/runs/p60-p62-runtime-benchmark-upgrade/summary.md`。 +- 2026-06-21 strategy validator passed：6 strategies、18 results、`run_kind=strategy_pilot_not_counted`。 +- Boundary：`counts_as_experiment=false`、`counts_as_strategy_ablation_result=false`；本轮不是正式 benchmark。 +- P6.3 Android real device lane 下一步：安装最新 APK 到独立 Android emulator 或真机，授权/核验 MobileCode Phone Use Accessibility service，跑 dry/action probe，并保存 screenshot、UI XML、logcat、focus state、WebView state assertions；promotion gate 前继续 non-counted。 diff --git a/scripts/run_mobile_harness_strategy_p60_p62_runtime_benchmark_upgrade.py b/scripts/run_mobile_harness_strategy_p60_p62_runtime_benchmark_upgrade.py new file mode 100644 index 0000000..40c65a2 --- /dev/null +++ b/scripts/run_mobile_harness_strategy_p60_p62_runtime_benchmark_upgrade.py @@ -0,0 +1,592 @@ +#!/usr/bin/env python3 +"""Generate the P6.0-P6.2 runtime benchmark upgrade contract run. + +This runner materializes taxonomy, communication-substrate, and runtime +verifier scaffold contracts as a validator-compatible, non-counted strategy +pilot. It does not call a model, drive a device, or claim a formal benchmark. +""" + +from __future__ import annotations + +import argparse +import csv +import json +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import run_mobile_harness_strategy_real_pilot as p5 + + +ROOT = p5.ROOT +REGISTRY_PATH = p5.REGISTRY_PATH +RUN_KIND = p5.RUN_KIND +BOUNDARY = p5.BOUNDARY +RUN_ID = "p60-p62-runtime-benchmark-upgrade" +TASKS = [ + { + "task_id": "P60-BENCHMARK-TAXONOMY-001", + "task_category": "benchmark_taxonomy_contract", + "title": "P6.0 benchmark task taxonomy, runtime assertion matrix, and score dimensions", + "phase": "P6.0", + }, + { + "task_id": "P61-COMMUNICATION-SUBSTRATE-001", + "task_category": "communication_substrate_contract", + "title": "P6.1 mailbox, evidence ledger, runtime event bus, and memory proposal contracts", + "phase": "P6.1", + }, + { + "task_id": "P62-RUNTIME-VERIFIER-SCAFFOLD-001", + "task_category": "runtime_verifier_scaffold_contract", + "title": "P6.2 runtime verifier JSON and non-counted run contract scaffold", + "phase": "P6.2", + }, +] +OUTPUT_DIR = ROOT / "docs/mobile-harness-benchmark/strategy-ablation/runs" / RUN_ID +TRACES_DIR = OUTPUT_DIR / "strategy_traces" +CONTRACT_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_contract.json" +VERIFIER_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_verifier.json" +SCOREBOARD_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_scoreboard.csv" +SUMMARY_PATH = OUTPUT_DIR / "summary.md" +RUN_PATH = OUTPUT_DIR / "run.json" +CONTRACT_DOC = ROOT / "docs/mobile-harness-benchmark/strategy-ablation/p60-p62-runtime-benchmark-contract.md" + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def rel(path: Path) -> str: + return p5.relative_to_root(path) + + +def load_json(path: Path) -> dict[str, Any]: + data = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError(f"Expected JSON object: {path}") + return data + + +def contract_payload() -> dict[str, Any]: + assertion_catalog = { + "keyboard": {"state_proof": "visible_or_structured_state_change", "required_redaction": "none"}, + "tap": {"state_proof": "target_state_changed", "required_redaction": "coordinate_only_or_app_owned_label"}, + "swipe": {"state_proof": "scroll_drawer_or_canvas_state_changed", "required_redaction": "coordinate_only"}, + "set_text": {"state_proof": "value_or_app_state_changed", "required_redaction": "test_string_only"}, + "localStorage": {"state_proof": "key_roundtrip_or_refresh_persistence", "required_redaction": "no_user_payload"}, + "ui_xml": {"state_proof": "stable_app_owned_label_or_state_marker", "required_redaction": "third_party_text_removed"}, + "screenshot": {"state_proof": "nonblank_target_ui_not_launcher_or_error", "required_redaction": "no_sensitive_screen"}, + "logcat": {"state_proof": "fatal_anr_flutter_plugin_scan", "required_redaction": "app_scoped_or_sanitized"}, + "webview_state": {"state_proof": "url_dom_console_or_js_state_marker", "required_redaction": "app_owned_state_only"}, + "focus_state": {"state_proof": "foreground_package_activity_or_process", "required_redaction": "device_id_hash"}, + } + taxonomy = [ + { + "task_category": "ui_artifact", + "required_assertions": ["keyboard", "tap", "screenshot", "logcat"], + "evidence_refs": ["screenshot", "verifier_json"], + }, + { + "task_category": "webview_artifact", + "required_assertions": ["keyboard", "tap", "set_text", "localStorage", "webview_state", "screenshot"], + "evidence_refs": ["webview_state", "screenshot", "verifier_json"], + }, + { + "task_category": "phone_use_permission", + "required_assertions": ["tap", "swipe", "set_text", "ui_xml", "logcat", "focus_state"], + "evidence_refs": ["ui_xml", "logcat", "accessibility_state"], + }, + { + "task_category": "file_intake", + "required_assertions": ["tap", "set_text", "ui_xml", "screenshot"], + "evidence_refs": ["intake_log", "content_hash", "ui_xml"], + }, + { + "task_category": "local_runtime", + "required_assertions": ["logcat", "screenshot", "focus_state"], + "evidence_refs": ["runtime_report", "exit_code", "log_excerpt"], + }, + { + "task_category": "network_boundary", + "required_assertions": ["logcat", "screenshot"], + "evidence_refs": ["network_summary", "redaction_report"], + }, + { + "task_category": "recovery_task", + "required_assertions": ["keyboard", "tap", "screenshot", "logcat"], + "evidence_refs": ["strategy_trace", "blocked_reason", "recovery_action"], + }, + { + "task_category": "real_device_task", + "required_assertions": ["tap", "swipe", "set_text", "ui_xml", "screenshot", "logcat", "focus_state"], + "evidence_refs": ["install_log", "launch_log", "ui_xml", "screenshot", "logcat"], + }, + ] + score_dimensions = { + "quality": 25, + "runtime_correctness": 25, + "phone_use_ability": 15, + "recovery": 15, + "latency_token": 10, + "safety_privacy": 10, + } + communication_substrate = { + "MailboxMessage": [ + "message_id", + "from_role", + "to_role", + "task_id", + "allowed_tools", + "budget", + "input_filter", + "expected_return", + "evidence_refs", + ], + "EvidenceLedgerEntry": [ + "evidence_id", + "kind", + "path", + "producer_role", + "redaction_state", + "created_at", + "summary", + ], + "RuntimeEvent": [ + "event_id", + "source", + "target", + "action", + "status", + "timestamp", + "evidence_id", + "redaction_state", + ], + "MemoryCommitProposal": [ + "proposal_id", + "source_trace", + "content_summary", + "ttl", + "redaction_state", + "approval_required", + ], + } + role_contracts = { + "CodeAgent": { + "allowed_tools": ["read_file", "apply_patch", "format", "unit_test"], + "budget": {"max_steps": 8}, + "input_filter": "code_files_and_scoped_task_only", + "return_contract": ["patch_summary", "tests", "blockers"], + }, + "RuntimeAgent": { + "allowed_tools": ["adb", "devicectl", "browser_cdp", "webview_probe"], + "budget": {"max_steps": 8}, + "input_filter": "runtime_target_and_evidence_policy", + "return_contract": ["runtime_events", "screenshots", "logs"], + }, + "PreviewAgent": { + "allowed_tools": ["screenshot", "ui_xml", "webview_state"], + "budget": {"max_steps": 6}, + "input_filter": "app_owned_ui_only", + "return_contract": ["visual_state", "ui_markers", "blockers"], + }, + "VerifierAgent": { + "allowed_tools": ["validators", "static_checks", "runtime_assertions"], + "budget": {"max_steps": 8}, + "input_filter": "artifacts_and_evidence_refs", + "return_contract": ["pass_fail", "score_dimensions", "missing_evidence"], + }, + "MemoryAgent": { + "allowed_tools": ["memory_packet", "redaction", "proposal"], + "budget": {"max_steps": 4}, + "input_filter": "summaries_only", + "return_contract": ["proposal_or_noop_reason"], + }, + "ReporterAgent": { + "allowed_tools": ["evidence_ledger", "markdown_summary"], + "budget": {"max_steps": 4}, + "input_filter": "verified_evidence_refs_only", + "return_contract": ["public_safe_summary"], + }, + } + verifier_scaffold = { + "verifier_json_required_fields": [ + "schema_version", + "run_kind", + "counts_as_experiment", + "task_id", + "task_category", + "assertion_results", + "score_dimensions", + "device_evidence_refs", + "webview_evidence_refs", + "privacy_boundary", + "blocked_reason", + ], + "non_counted_run_required_fields": [ + "run_kind", + "counts_as_experiment", + "counts_as_strategy_ablation_result", + "evidence_boundary", + "results", + "summary", + ], + "android_webview_upgrade_assertions": [ + "ui_xml", + "screenshot", + "logcat", + "webview_state", + "focus_state", + "accessibility_state", + ], + } + return { + "schema_version": "0.1.0", + "contract_id": "p60_p62_runtime_benchmark_upgrade_contract", + "run_kind": RUN_KIND, + "counts_as_experiment": False, + "counts_as_strategy_ablation_result": False, + "task_taxonomy": taxonomy, + "assertion_catalog": assertion_catalog, + "score_dimensions": score_dimensions, + "communication_substrate": communication_substrate, + "role_contracts": role_contracts, + "runtime_verifier_scaffold": verifier_scaffold, + "next_android_real_device_lane_recommendation": { + "status": "recommended_next", + "required_evidence": [ + "apk_install", + "launch_focus_state", + "accessibility_authorization_state", + "screenshot", + "ui_xml", + "app_scoped_logcat", + "phone_use_dry_probe", + "phone_use_action_probe", + "webview_state_assertions", + ], + "boundary": "non_counted_until_repeated_task_samples_and_promotion_gate", + }, + } + + +def verifier_payload(contract: dict[str, Any], created_at: str) -> dict[str, Any]: + required_assertions = { + "keyboard", + "tap", + "swipe", + "set_text", + "localStorage", + "ui_xml", + "screenshot", + "logcat", + "webview_state", + } + assertion_names = set(contract["assertion_catalog"]) + checks = { + "contract_doc_exists": CONTRACT_DOC.exists(), + "taxonomy_has_required_categories": len(contract["task_taxonomy"]) >= 8, + "runtime_assertion_matrix_complete": required_assertions.issubset(assertion_names), + "score_dimensions_complete": set(contract["score_dimensions"]) == { + "quality", + "runtime_correctness", + "phone_use_ability", + "recovery", + "latency_token", + "safety_privacy", + }, + "score_weights_sum_100": sum(contract["score_dimensions"].values()) == 100, + "mailbox_contract_present": "MailboxMessage" in contract["communication_substrate"], + "evidence_ledger_contract_present": "EvidenceLedgerEntry" in contract["communication_substrate"], + "runtime_event_bus_contract_present": "RuntimeEvent" in contract["communication_substrate"], + "memory_commit_proposal_contract_present": "MemoryCommitProposal" in contract["communication_substrate"], + "role_contracts_cover_six_roles": len(contract["role_contracts"]) == 6, + "runtime_verifier_json_contract_present": bool(contract["runtime_verifier_scaffold"]["verifier_json_required_fields"]), + "non_counted_run_contract_present": bool(contract["runtime_verifier_scaffold"]["non_counted_run_required_fields"]), + "android_webview_upgrade_assertions_present": { + "ui_xml", + "logcat", + "webview_state", + }.issubset(set(contract["runtime_verifier_scaffold"]["android_webview_upgrade_assertions"])), + "next_android_real_device_lane_recommendation_present": bool( + contract["next_android_real_device_lane_recommendation"]["required_evidence"] + ), + "non_counted_boundary": contract["run_kind"] == RUN_KIND + and contract["counts_as_experiment"] is False + and contract["counts_as_strategy_ablation_result"] is False, + } + status = "passed" if all(checks.values()) else "blocked" + return { + "schema_version": "0.1.0", + "verifier_id": "p60_p62_runtime_benchmark_upgrade_verifier", + "created_at": created_at, + "status": status, + "run_kind": RUN_KIND, + "counts_as_experiment": False, + "counts_as_strategy_ablation_result": False, + "scope": "P6.0 taxonomy, P6.1 communication substrate, and P6.2 runtime verifier scaffold contract only.", + "score": { + "score_boundary": "pilot_p60_p62_contract_score_not_counted", + "total_score": 100.0 if status == "passed" else 0.0, + "max_score": 100, + "checks": checks, + "blocked_reason": None if status == "passed" else "p60_p62_contract_checks_failed", + }, + "contract": rel(CONTRACT_PATH), + "contract_doc": rel(CONTRACT_DOC), + } + + +def build_trace(strategy: dict[str, Any], task: dict[str, str], created_at: str) -> dict[str, Any]: + return { + "trace_id": f"strace_{RUN_ID}_{strategy['strategy_id']}_{task['task_id']}", + "strategy_id": strategy["strategy_id"], + "trace_status": BOUNDARY, + "events": [ + { + "event_id": "evt_001", + "type": "contract_verify", + "role": "RuntimeBenchmarkContractVerifier", + "step_id": "step_001", + "started_at": created_at, + "ended_at": created_at, + "tool_name": "p60_p62_contract_materializer", + "evidence_id": f"{RUN_ID}_{strategy['strategy_id']}_{task['task_id']}", + "summary": f"{task['phase']} contract scaffold materialized for {strategy['strategy_id']}.", + "artifact_path": rel(VERIFIER_PATH), + } + ], + "handoff_count": 0, + "planning_revisions": 0, + "verification_failures_recovered": 0, + "failure_kind": None, + } + + +def build_run(contract: dict[str, Any], verifier: dict[str, Any], registry: dict[str, Any]) -> dict[str, Any]: + created_at = verifier["created_at"] + strategies = registry["strategies"] + results: list[dict[str, Any]] = [] + TRACES_DIR.mkdir(parents=True, exist_ok=True) + for strategy in strategies: + for task in TASKS: + trace = build_trace(strategy, task, created_at) + trace_path = TRACES_DIR / f"{strategy['strategy_id']}_{task['task_id']}.json" + trace_path.write_text(json.dumps(trace, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") + result = { + "strategy_id": strategy["strategy_id"], + "strategy_family": strategy["strategy_family"], + "task_id": task["task_id"], + "task_category": task["task_category"], + "status": verifier["status"], + "strategy_trace": trace, + "time_metrics": { + "planning_ms": 0, + "execution_ms": 0, + "verification_ms": 0, + "reporting_ms": 0, + "wall_ms": 0, + }, + "token_metrics": { + "prompt_tokens": 0, + "completion_tokens": 0, + "estimated_tool_io_tokens": 0, + "total_tokens": 0, + "estimated_cost_usd": 0, + "tokens_per_verified_success": 0, + }, + "effect_metrics": { + "task_success": 1.0 if verifier["status"] == "passed" else 0.0, + "verified_success": 1.0 if verifier["status"] == "passed" else 0.0, + "trace_completeness": 1.0, + "artifact_availability": 1.0, + "recovery_rate": None, + "human_intervention_count": 0, + "handoff_success_rate": None, + "memory_reuse_score": None, + "steps_to_completion": 1, + }, + "evidence": { + "boundary": BOUNDARY, + "artifact_paths": [rel(CONTRACT_DOC), rel(CONTRACT_PATH)], + "trace_paths": [rel(trace_path)], + "screenshot_paths": [], + "logs": [ + "P6.0-P6.2 runtime benchmark upgrade contract materialized.", + "Run is non-counted and must not be cited as a formal benchmark.", + "No model callback, device action, raw transcript, or secret material was used.", + ], + "verifier_outputs": [rel(VERIFIER_PATH)], + "transcript_paths": [], + "human_intervention_notes": [], + }, + "pilot_verifier": { + "score_boundary": verifier["score"]["score_boundary"], + "verifier_output": rel(VERIFIER_PATH), + }, + "pilot_score": verifier["score"], + "contract_refs": { + "task_taxonomy": rel(CONTRACT_PATH), + "communication_substrate": rel(CONTRACT_PATH), + "runtime_verifier_scaffold": rel(CONTRACT_PATH), + }, + "counts_as_strategy_ablation_result": False, + } + results.append(result) + summary = { + "total": len(results), + "strategies": len(strategies), + "tasks_per_strategy": len(TASKS), + "passed": sum(1 for item in results if item["status"] == "passed"), + "warning": sum(1 for item in results if item["status"] == "warning"), + "failed": sum(1 for item in results if item["status"] == "failed"), + "blocked": sum(1 for item in results if item["status"] == "blocked"), + "average_pilot_score_not_counted": verifier["score"]["total_score"], + } + return { + "benchmark": "MobileHarnessBench", + "run_id": RUN_ID, + "created_at": created_at, + "counts_as_experiment": False, + "counts_as_strategy_ablation_result": False, + "run_kind": RUN_KIND, + "schema_version": "0.1.0-p60-p62-runtime-benchmark-upgrade", + "strategy_family": "mixed_strategy_ablation", + "evidence_boundary": "pilot_not_counted:p60_p62_runtime_benchmark_upgrade_contract_not_counted", + "environment": { + "execution_tier": "P6.0-P6.2-contract-scaffold", + "mode": RUN_KIND, + "model_provider": "none", + "model_name": "none", + "runtime_backend": "contract_materializer_no_device_actions", + "credential_source": "none", + }, + "mode": { + "name": "P6.0-P6.2 runtime benchmark upgrade", + "mode": RUN_KIND, + "non_counted_reason": "Contract/scaffold materialization only; no formal strategy benchmark.", + }, + "strategies": [ + { + "strategy_id": item["strategy_id"], + "strategy_family": item["strategy_family"], + "description": item["description"], + } + for item in strategies + ], + "task_subset": { + "name": RUN_ID, + "task_count": len(TASKS), + "tasks": [ + { + "task_id": task["task_id"], + "task_category": task["task_category"], + "title": task["title"], + "max_score": 100, + } + for task in TASKS + ], + }, + "contract_summary": { + "taxonomy_categories": len(contract["task_taxonomy"]), + "runtime_assertions": sorted(contract["assertion_catalog"].keys()), + "score_dimensions": contract["score_dimensions"], + "communication_substrate_contracts": sorted(contract["communication_substrate"].keys()), + "runtime_verifier_upgrade": contract["runtime_verifier_scaffold"]["android_webview_upgrade_assertions"], + }, + "results": results, + "summary": summary, + "score_boundary": verifier["score"]["score_boundary"], + } + + +def write_scoreboard(run: dict[str, Any]) -> None: + with SCOREBOARD_PATH.open("w", encoding="utf-8", newline="") as handle: + writer = csv.DictWriter( + handle, + fieldnames=["task_id", "strategy_id", "status", "score", "counts_as_experiment"], + lineterminator="\n", + ) + writer.writeheader() + for result in run["results"]: + writer.writerow( + { + "task_id": result["task_id"], + "strategy_id": result["strategy_id"], + "status": result["status"], + "score": result["pilot_score"]["total_score"], + "counts_as_experiment": False, + } + ) + + +def write_summary(run: dict[str, Any], verifier: dict[str, Any]) -> None: + lines = [ + "# P6.0-P6.2 Runtime Benchmark Upgrade", + "", + f"- run_id: `{run['run_id']}`", + f"- run_kind: `{run['run_kind']}`", + "- counts_as_experiment: `false`", + "- counts_as_strategy_ablation_result: `false`", + f"- status: `{verifier['status']}`", + f"- score_boundary: `{verifier['score']['score_boundary']}`", + f"- total_score_not_counted: `{verifier['score']['total_score']}`", + "", + "## Evidence", + "", + f"- Contract doc: `{rel(CONTRACT_DOC)}`", + f"- Contract JSON: `{rel(CONTRACT_PATH)}`", + f"- Verifier JSON: `{rel(VERIFIER_PATH)}`", + f"- Scoreboard: `{rel(SCOREBOARD_PATH)}`", + f"- Run JSON: `{rel(RUN_PATH)}`", + "", + "## Scope", + "", + "- P6.0 defines task taxonomy, runtime assertions, and score dimensions.", + "- P6.1 defines mailbox, EvidenceLedger, RuntimeEventBus, MemoryCommitProposal, and role contracts.", + "- P6.2 defines runtime verifier JSON and non-counted run contracts, with Android/WebView evidence hooks.", + "- This run is a contract/scaffold proof only, not a formal benchmark.", + "", + "## Next P6.3 Android Real Device Lane", + "", + "Install the latest APK on a real Android device or dedicated emulator, verify Accessibility state, run Mobile Phone Use dry/action probes, capture screenshot/UI XML/logcat/focus state, and add WebView state assertions for generated artifacts. Keep the run non-counted until repeated samples and promotion gates pass.", + "", + ] + SUMMARY_PATH.write_text("\n".join(lines), encoding="utf-8") + + +def main() -> int: + global OUTPUT_DIR, TRACES_DIR, CONTRACT_PATH, VERIFIER_PATH, SCOREBOARD_PATH, SUMMARY_PATH, RUN_PATH + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", default=str(OUTPUT_DIR)) + args = parser.parse_args() + output_dir = Path(args.output) + OUTPUT_DIR = output_dir if output_dir.is_absolute() else ROOT / output_dir + TRACES_DIR = OUTPUT_DIR / "strategy_traces" + CONTRACT_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_contract.json" + VERIFIER_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_verifier.json" + SCOREBOARD_PATH = OUTPUT_DIR / "runtime_benchmark_upgrade_scoreboard.csv" + SUMMARY_PATH = OUTPUT_DIR / "summary.md" + RUN_PATH = OUTPUT_DIR / "run.json" + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + + registry = load_json(REGISTRY_PATH) + created_at = utc_now() + contract = contract_payload() + CONTRACT_PATH.write_text(json.dumps(contract, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") + verifier = verifier_payload(contract, created_at) + VERIFIER_PATH.write_text(json.dumps(verifier, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") + run = build_run(contract, verifier, registry) + RUN_PATH.write_text(json.dumps(run, indent=2, ensure_ascii=True) + "\n", encoding="utf-8") + write_scoreboard(run) + write_summary(run, verifier) + print(f"Wrote {rel(CONTRACT_PATH)}") + print(f"Wrote {rel(VERIFIER_PATH)}") + print(f"Wrote {rel(RUN_PATH)}") + print(f"Status: {verifier['status']} score={verifier['score']['total_score']}") + return 0 if verifier["status"] == "passed" else 1 + + +if __name__ == "__main__": + raise SystemExit(main())