From e8f1bccef8c579e17b7ebfb81c005dbc0f1dca71 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 1 Jul 2026 20:28:03 +0800 Subject: [PATCH] feat: add record and replay baseline --- .editorconfig | 15 + .github/ISSUE_TEMPLATE/bug_report.yml | 27 + .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.yml | 27 + .github/PULL_REQUEST_TEMPLATE.md | 12 + .github/dependency-review-config.yml | 7 + .github/workflows/ci.yml | 32 + .github/workflows/docs-check.yml | 26 + .github/workflows/repo-hygiene.yml | 32 + .github/workflows/supply-chain-security.yml | 42 + .markdownlint.json | 4 + CONTRIBUTING.md | 1 + Makefile | 141 +- README.md | 172 +- README.zh-CN.md | 118 +- .../OpenComputerUse/MCPAppRuntime.swift | 6 +- .../OpenComputerUse/MacOSAppAgentProxy.swift | 60 +- .../OpenComputerUse/OpenComputerUseMain.swift | 14 + docs/ARCHITECTURE.md | 51 +- docs/CICD.md | 64 + docs/QUALITY_SCORE.md | 162 +- docs/design-docs/index.md | 18 +- docs/design-docs/record-and-replay-handoff.md | 272 ++ ...cord-and-replay-official-golden-capture.md | 317 ++ .../record-and-replay-replication.md | 247 ++ ...20260626-record-and-replay-event-stream.md | 578 +++ ...26-1737-document-record-and-replay-plan.md | 608 ++++ ...0627-0112-record-and-replay-doc-handoff.md | 915 +++++ ...-1825-record-and-replay-official-golden.md | 31 + docs/references/README.md | 1 + .../README.md | 10 + ...d-replay-event-stream-surface-1.0.857.json | 64 + ...fficial-no-active-status-stop-1.0.857.json | 29 + ...ay-official-raw-start-timeout-1.0.857.json | 138 + .../ocu-simple-action-stop/README.md | 10 + .../ocu-simple-action-stop/events.jsonl | 5 + .../fixture-manifest.json | 101 + .../mcp-transcript.json | 587 +++ .../ocu-simple-action-stop/metadata.json | 14 + .../ocu-simple-action-stop/session.json | 7 + .../ocu-simple-action-stop/suppressed.jsonl | 0 .../README.md | 11 + .../events.jsonl | 13 + .../fixture-manifest.json | 102 + .../mcp-transcript.json | 250 ++ .../metadata.json | 14 + .../session.json | 7 + .../suppressed.jsonl | 0 .../record-and-replay-event-stream.md | 377 ++ .../AccessibilitySnapshot.swift | 14 +- .../EventStreamMCPServer.swift | 306 ++ .../EventStreamRecordingControls.swift | 190 + .../EventStreamRecordingSummary.swift | 1006 +++++ .../EventStreamRecordingValidation.swift | 696 ++++ .../EventStreamService.swift | 2799 ++++++++++++++ .../EventStreamSkillScaffold.swift | 501 +++ .../EventStreamToolDefinitions.swift | 41 + .../OpenComputerUseKit/MCPServer.swift | 7 +- .../OpenComputerUseCLI.swift | 567 ++- .../OpenComputerUseKitTests.swift | 3236 ++++++++++++++++- ...uild-record-and-replay-baseline-summary.py | 1266 +++++++ scripts/check-docs.sh | 198 + .../check-event-stream-golden-readiness.py | 697 ++++ ...-event-stream-official-fixture-coverage.py | 286 ++ ...check-event-stream-official-fixture-set.py | 246 ++ ...heck-record-and-replay-baseline-summary.py | 890 +++++ scripts/ci.sh | 33 + scripts/compare-event-stream-no-active.py | 242 ++ scripts/compare-event-stream-recordings.py | 1017 ++++++ scripts/compare-event-stream-surface.py | 282 ++ ...cord-and-replay-official-capture-packet.py | 331 ++ scripts/import-event-stream-fixture.py | 501 +++ .../ingest-ocu-record-and-replay-candidate.py | 489 +++ ...gest-official-record-and-replay-fixture.py | 530 +++ .../install-codex-record-and-replay-mcp.sh | 38 + scripts/install-config-helper.mjs | 24 +- scripts/npm/build-packages.mjs | 135 +- scripts/package-skill.sh | 166 +- ...record-and-replay-ocu-candidate-pairing.py | 308 ++ ...cord-and-replay-official-golden-capture.py | 1571 ++++++++ scripts/probe-event-stream-recording.py | 521 +++ .../record_and_replay_baseline_contract.py | 206 ++ scripts/record_and_replay_scenarios.py | 190 + scripts/run-event-stream-smoke-matrix.sh | 34 + scripts/run-event-stream-smoke-tests.sh | 1803 +++++++++ .../run-record-and-replay-baseline-smoke.sh | 517 +++ scripts/scaffold-event-stream-skill.py | 496 +++ .../scaffold-record-and-replay-skill-repo.py | 3197 ++++++++++++++++ scripts/summarize-event-stream-recording.py | 846 +++++ ...test-codex-record-and-replay-installer.mjs | 57 + scripts/test-event-stream-fixture-import.py | 498 +++ scripts/test-event-stream-golden-readiness.py | 965 +++++ scripts/test-event-stream-local-probe.py | 196 + ...-event-stream-official-fixture-coverage.py | 370 ++ .../test-event-stream-official-fixture-set.py | 531 +++ scripts/test-event-stream-probe-fixtures.py | 117 + .../test-event-stream-recording-compare.py | 726 ++++ scripts/test-event-stream-recording-probe.py | 281 ++ scripts/test-event-stream-skill-scaffold.py | 1212 ++++++ ...-record-and-replay-skill-repo-scaffold.mjs | 1265 +++++++ ...-ocu-record-and-replay-candidate-ingest.py | 398 ++ ...ficial-record-and-replay-fixture-ingest.py | 751 ++++ ...-and-replay-baseline-audit-make-targets.py | 137 + ...est-record-and-replay-baseline-contract.py | 81 + ...and-replay-baseline-runner-summary-json.py | 307 ++ ...ecord-and-replay-baseline-summary-audit.py | 888 +++++ ...test-record-and-replay-baseline-summary.py | 2273 ++++++++++++ ...-replay-ocu-candidate-pairing-preflight.py | 267 ++ ...d-and-replay-official-capture-finalizer.py | 249 ++ ...eplay-official-golden-capture-preflight.py | 965 +++++ ...t-record-and-replay-skill-repo-scaffold.py | 1525 ++++++++ scripts/validate-event-stream-recording.py | 543 +++ .../SKILL.md | 146 + skills/open-computer-use/SKILL.md | 4 +- .../references/record-and-replay.md | 200 + 115 files changed, 47003 insertions(+), 116 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/dependency-review-config.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docs-check.yml create mode 100644 .github/workflows/repo-hygiene.yml create mode 100644 .github/workflows/supply-chain-security.yml create mode 100644 .markdownlint.json create mode 100644 docs/design-docs/record-and-replay-handoff.md create mode 100644 docs/design-docs/record-and-replay-official-golden-capture.md create mode 100644 docs/design-docs/record-and-replay-replication.md create mode 100644 docs/exec-plans/active/20260626-record-and-replay-event-stream.md create mode 100644 docs/histories/2026-06/20260626-1737-document-record-and-replay-plan.md create mode 100644 docs/histories/2026-06/20260627-0112-record-and-replay-doc-handoff.md create mode 100644 docs/histories/2026-07/20260701-1825-record-and-replay-official-golden.md create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/README.md create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/events.jsonl create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/fixture-manifest.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/mcp-transcript.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/metadata.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/session.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/suppressed.jsonl create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/README.md create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/events.jsonl create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/fixture-manifest.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/mcp-transcript.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/metadata.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/session.json create mode 100644 docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/suppressed.jsonl create mode 100644 docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamMCPServer.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingControls.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingSummary.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingValidation.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamService.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamSkillScaffold.swift create mode 100644 packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamToolDefinitions.swift create mode 100755 scripts/build-record-and-replay-baseline-summary.py create mode 100755 scripts/check-event-stream-golden-readiness.py create mode 100755 scripts/check-event-stream-official-fixture-coverage.py create mode 100755 scripts/check-event-stream-official-fixture-set.py create mode 100755 scripts/check-record-and-replay-baseline-summary.py create mode 100755 scripts/compare-event-stream-no-active.py create mode 100755 scripts/compare-event-stream-recordings.py create mode 100755 scripts/compare-event-stream-surface.py create mode 100755 scripts/finalize-record-and-replay-official-capture-packet.py create mode 100755 scripts/import-event-stream-fixture.py create mode 100755 scripts/ingest-ocu-record-and-replay-candidate.py create mode 100755 scripts/ingest-official-record-and-replay-fixture.py create mode 100755 scripts/install-codex-record-and-replay-mcp.sh create mode 100755 scripts/prepare-record-and-replay-ocu-candidate-pairing.py create mode 100755 scripts/prepare-record-and-replay-official-golden-capture.py create mode 100755 scripts/probe-event-stream-recording.py create mode 100644 scripts/record_and_replay_baseline_contract.py create mode 100644 scripts/record_and_replay_scenarios.py create mode 100755 scripts/run-event-stream-smoke-matrix.sh create mode 100755 scripts/run-event-stream-smoke-tests.sh create mode 100755 scripts/run-record-and-replay-baseline-smoke.sh create mode 100755 scripts/scaffold-event-stream-skill.py create mode 100755 scripts/scaffold-record-and-replay-skill-repo.py create mode 100755 scripts/summarize-event-stream-recording.py create mode 100755 scripts/test-codex-record-and-replay-installer.mjs create mode 100755 scripts/test-event-stream-fixture-import.py create mode 100755 scripts/test-event-stream-golden-readiness.py create mode 100755 scripts/test-event-stream-local-probe.py create mode 100755 scripts/test-event-stream-official-fixture-coverage.py create mode 100755 scripts/test-event-stream-official-fixture-set.py create mode 100755 scripts/test-event-stream-probe-fixtures.py create mode 100755 scripts/test-event-stream-recording-compare.py create mode 100755 scripts/test-event-stream-recording-probe.py create mode 100755 scripts/test-event-stream-skill-scaffold.py create mode 100755 scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs create mode 100755 scripts/test-ocu-record-and-replay-candidate-ingest.py create mode 100755 scripts/test-official-record-and-replay-fixture-ingest.py create mode 100755 scripts/test-record-and-replay-baseline-audit-make-targets.py create mode 100755 scripts/test-record-and-replay-baseline-contract.py create mode 100755 scripts/test-record-and-replay-baseline-runner-summary-json.py create mode 100755 scripts/test-record-and-replay-baseline-summary-audit.py create mode 100755 scripts/test-record-and-replay-baseline-summary.py create mode 100755 scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py create mode 100755 scripts/test-record-and-replay-official-capture-finalizer.py create mode 100755 scripts/test-record-and-replay-official-golden-capture-preflight.py create mode 100755 scripts/test-record-and-replay-skill-repo-scaffold.py create mode 100755 scripts/validate-event-stream-recording.py create mode 100644 skills/open-computer-use-record-and-replay/SKILL.md create mode 100644 skills/open-computer-use/references/record-and-replay.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..2568cd18 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.swift] +indent_size = 4 + +[Makefile] +indent_style = tab diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 00000000..a81123ae --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,27 @@ +name: Bug report +description: Report a reproducible problem in open-computer-use. +title: "[Bug]: " +labels: + - bug +body: + - type: textarea + id: summary + attributes: + label: Summary + description: What went wrong? + validations: + required: true + - type: textarea + id: steps + attributes: + label: Reproduction + description: List the smallest steps or command that reproduces the problem. + validations: + required: true + - type: textarea + id: environment + attributes: + label: Environment + description: Include OS, open-computer-use version, and relevant agent/client. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..68cc9f22 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: true +contact_links: + - name: Discussions + url: https://github.com/iFurySt/open-codex-computer-use/discussions + about: Ask usage questions or discuss ideas that are not ready as issues. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 00000000..674be8e7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,27 @@ +name: Feature request +description: Propose a new capability or compatibility improvement. +title: "[Feature]: " +labels: + - enhancement +body: + - type: textarea + id: problem + attributes: + label: Problem + description: What workflow or compatibility gap should this solve? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Proposal + description: Describe the behavior you want. + validations: + required: true + - type: textarea + id: validation + attributes: + label: Validation + description: How should we verify this works? + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..81c2accf --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,12 @@ +## Summary + +- + +## Validation + +- [ ] `make check-docs` +- [ ] `make ci` + +## Notes + +- diff --git a/.github/dependency-review-config.yml b/.github/dependency-review-config.yml new file mode 100644 index 00000000..d3fbfa00 --- /dev/null +++ b/.github/dependency-review-config.yml @@ -0,0 +1,7 @@ +fail-on-severity: high +allow-licenses: + - MIT + - Apache-2.0 + - BSD-2-Clause + - BSD-3-Clause + - ISC diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..bf0b18fe --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,32 @@ +name: ci + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + ci: + runs-on: macos-26 + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: "24" + + - name: Set up Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c + with: + go-version: "1.22.x" + + - name: Run repository CI + run: ./scripts/ci.sh diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml new file mode 100644 index 00000000..204e77cc --- /dev/null +++ b/.github/workflows/docs-check.yml @@ -0,0 +1,26 @@ +name: docs-check + +on: + pull_request: + paths: + - "AGENTS.md" + - "docs/**" + - "README.md" + - "CONTRIBUTING.md" + - "scripts/check-docs.sh" + - ".github/workflows/docs-check.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + docs-check: + runs-on: macos-26 + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Run docs check + run: ./scripts/check-docs.sh diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml new file mode 100644 index 00000000..9902a44c --- /dev/null +++ b/.github/workflows/repo-hygiene.yml @@ -0,0 +1,32 @@ +name: repo-hygiene + +on: + pull_request: + paths: + - ".github/**" + - ".gitattributes" + - ".gitignore" + - ".editorconfig" + - "CODEOWNERS" + - "CONTRIBUTING.md" + - "README.md" + - "SECURITY.md" + - "scripts/check-action-pinning.sh" + - "scripts/check-repo-hygiene.sh" + workflow_dispatch: + +permissions: + contents: read + +jobs: + repo-hygiene: + runs-on: macos-26 + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Run repo hygiene checks + run: | + ./scripts/check-repo-hygiene.sh + ./scripts/check-action-pinning.sh diff --git a/.github/workflows/supply-chain-security.yml b/.github/workflows/supply-chain-security.yml new file mode 100644 index 00000000..529bcd5f --- /dev/null +++ b/.github/workflows/supply-chain-security.yml @@ -0,0 +1,42 @@ +name: supply-chain-security + +on: + pull_request: + paths: + - ".github/workflows/**" + - "package.json" + - "package-lock.json" + - "Package.swift" + - "apps/**/go.mod" + - "apps/**/go.sum" + - "scripts/check-action-pinning.sh" + - ".github/dependency-review-config.yml" + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + action-pinning: + runs-on: macos-26 + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Check GitHub Action pinning + run: ./scripts/check-action-pinning.sh + + dependency-review: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Dependency review + uses: actions/dependency-review-action@da24556b548a50705dd671f47852072ea4c105d9 + with: + config-file: .github/dependency-review-config.yml diff --git a/.markdownlint.json b/.markdownlint.json new file mode 100644 index 00000000..26593a22 --- /dev/null +++ b/.markdownlint.json @@ -0,0 +1,4 @@ +{ + "MD013": false, + "MD033": false +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ea22328..7cf9ea28 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,6 +12,7 @@ ## 发起 Pull Request 之前 - 运行 `make check-docs`。 +- 如果本次改动涉及代码、脚本或仓库流程，运行 `make ci`。 - 如果本次改动涉及代码或仓库流程，补齐或更新对应 history。 - 如果变更对用户可感知，补齐 release note。 - 确认示例、脚本、说明文档和当前实现一致。 diff --git a/Makefile b/Makefile index 20dd4ef9..5bd0c8e0 100644 --- a/Makefile +++ b/Makefile @@ -2,8 +2,15 @@ PROJECT ?= SLUG ?= AGENTS ?= claude,codex SCENARIO ?= list-apps +RNR_SCENARIO ?= simple-action-stop +RNR_PACKET_DIR ?= +RNR_FIXTURE_ROOT ?= +RNR_OFFICIAL_PLUGIN_ROOT ?= +RNR_ALLOW_MISSING_OFFICIAL_PLUGIN ?= 0 +RNR_BASELINE_SUMMARY_JSON ?= dist/record-and-replay-baseline-summary.json +RNR_OFFICIAL_GOLDEN_SUMMARY_JSON ?= dist/record-and-replay-official-golden-gate-summary.json -.PHONY: init build app test smoke stress agent-smoke check-docs check-repo ci release-package npm-build npm-publish new-history new-plan +.PHONY: init build app test smoke event-stream-smoke event-stream-action-smoke event-stream-smoke-matrix event-stream-surface-smoke event-stream-no-active-smoke event-stream-probe event-stream-local-probe-smoke event-stream-official-probe event-stream-official-start-probe event-stream-probe-fixture-smoke event-stream-fixture-smoke event-stream-compare-smoke event-stream-golden-readiness-smoke event-stream-official-fixture-coverage-smoke event-stream-official-fixture-set-smoke event-stream-official-fixture-ingest-smoke event-stream-ocu-candidate-ingest-smoke event-stream-skill-scaffold-smoke record-and-replay-baseline-contract-smoke record-and-replay-baseline-summary-smoke record-and-replay-baseline-summary-audit-smoke record-and-replay-baseline-audit-targets-smoke record-and-replay-official-golden-capture-preflight record-and-replay-official-golden-capture-packet record-and-replay-official-golden-capture-packet-set record-and-replay-official-golden-capture-preflight-smoke record-and-replay-official-capture-finalizer-smoke record-and-replay-official-golden-fixture-gate record-and-replay-ocu-candidate-pairing-preflight record-and-replay-ocu-candidate-pairing-preflight-smoke record-and-replay-skill-repo-smoke npm-record-and-replay-skill-repo-smoke record-and-replay-baseline-smoke record-and-replay-baseline-audit record-and-replay-official-golden-gate record-and-replay-official-golden-gate-audit codex-record-and-replay-installer-smoke stress agent-smoke check-docs check-repo ci release-package npm-build npm-publish new-history new-plan init: @if [ -z "$(PROJECT)" ]; then echo "用法: make init PROJECT=项目名"; exit 1; fi @@ -21,6 +28,138 @@ test: smoke: ./scripts/run-tool-smoke-tests.sh +event-stream-smoke: + ./scripts/run-event-stream-smoke-tests.sh + +event-stream-action-smoke: + OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 ./scripts/run-event-stream-smoke-tests.sh + +event-stream-smoke-matrix: + ./scripts/run-event-stream-smoke-matrix.sh + +event-stream-surface-smoke: + ./scripts/compare-event-stream-surface.py + +event-stream-no-active-smoke: + ./scripts/compare-event-stream-no-active.py + +event-stream-probe: + @./scripts/probe-event-stream-recording.py --target local --start-stop + +event-stream-local-probe-smoke: + ./scripts/test-event-stream-local-probe.py + +event-stream-official-probe: + @./scripts/probe-event-stream-recording.py --target official + +event-stream-official-start-probe: + @./scripts/probe-event-stream-recording.py --target official --start-stop --allow-start-timeout + +event-stream-probe-fixture-smoke: + ./scripts/test-event-stream-probe-fixtures.py + ./scripts/test-event-stream-recording-probe.py + +event-stream-fixture-smoke: + ./scripts/test-event-stream-fixture-import.py + +event-stream-compare-smoke: + ./scripts/test-event-stream-recording-compare.py + +event-stream-golden-readiness-smoke: + ./scripts/test-event-stream-golden-readiness.py + +event-stream-official-fixture-coverage-smoke: + ./scripts/test-event-stream-official-fixture-coverage.py + +event-stream-official-fixture-set-smoke: + ./scripts/test-event-stream-official-fixture-set.py + +event-stream-official-fixture-ingest-smoke: + ./scripts/test-official-record-and-replay-fixture-ingest.py + +event-stream-ocu-candidate-ingest-smoke: + ./scripts/test-ocu-record-and-replay-candidate-ingest.py + +event-stream-skill-scaffold-smoke: + ./scripts/test-event-stream-skill-scaffold.py + +record-and-replay-baseline-contract-smoke: + ./scripts/test-record-and-replay-baseline-contract.py + +record-and-replay-baseline-summary-smoke: + ./scripts/test-record-and-replay-baseline-summary.py + +record-and-replay-baseline-summary-audit-smoke: + ./scripts/test-record-and-replay-baseline-summary-audit.py + +record-and-replay-baseline-audit-targets-smoke: + ./scripts/test-record-and-replay-baseline-audit-make-targets.py + +record-and-replay-official-golden-capture-preflight: + @./scripts/prepare-record-and-replay-official-golden-capture.py + +record-and-replay-official-golden-capture-packet: + @packet_dir="$(RNR_PACKET_DIR)"; \ + if [ -z "$${packet_dir}" ]; then packet_dir="$${TMPDIR:-/tmp}/ocu-rnr-official-$(RNR_SCENARIO)"; fi; \ + set --; \ + if [ -n "$(RNR_FIXTURE_ROOT)" ]; then set -- "$$@" --fixture-root "$(RNR_FIXTURE_ROOT)"; fi; \ + if [ -n "$(RNR_OFFICIAL_PLUGIN_ROOT)" ]; then set -- "$$@" --official-plugin-root "$(RNR_OFFICIAL_PLUGIN_ROOT)"; fi; \ + if [ "$(RNR_ALLOW_MISSING_OFFICIAL_PLUGIN)" = "1" ]; then set -- "$$@" --allow-missing-official-plugin; fi; \ + ./scripts/prepare-record-and-replay-official-golden-capture.py \ + "$$@" \ + --scenario "$(RNR_SCENARIO)" \ + --capture-packet-dir "$${packet_dir}" + +record-and-replay-official-golden-capture-packet-set: + @packet_dir="$(RNR_PACKET_DIR)"; \ + if [ -z "$${packet_dir}" ]; then packet_dir="$${TMPDIR:-/tmp}/ocu-rnr-official-golden-packets"; fi; \ + set --; \ + if [ -n "$(RNR_FIXTURE_ROOT)" ]; then set -- "$$@" --fixture-root "$(RNR_FIXTURE_ROOT)"; fi; \ + if [ -n "$(RNR_OFFICIAL_PLUGIN_ROOT)" ]; then set -- "$$@" --official-plugin-root "$(RNR_OFFICIAL_PLUGIN_ROOT)"; fi; \ + if [ "$(RNR_ALLOW_MISSING_OFFICIAL_PLUGIN)" = "1" ]; then set -- "$$@" --allow-missing-official-plugin; fi; \ + ./scripts/prepare-record-and-replay-official-golden-capture.py \ + "$$@" \ + --capture-packet-dir "$${packet_dir}" \ + --capture-packet-recommended-scenarios + +record-and-replay-official-golden-capture-preflight-smoke: + ./scripts/test-record-and-replay-official-golden-capture-preflight.py + +record-and-replay-official-capture-finalizer-smoke: + ./scripts/test-record-and-replay-official-capture-finalizer.py + +record-and-replay-official-golden-fixture-gate: + @./scripts/check-event-stream-official-fixture-coverage.py --require-readiness + +record-and-replay-ocu-candidate-pairing-preflight: + @./scripts/prepare-record-and-replay-ocu-candidate-pairing.py + +record-and-replay-ocu-candidate-pairing-preflight-smoke: + ./scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py + +record-and-replay-skill-repo-smoke: + ./scripts/test-record-and-replay-skill-repo-scaffold.py + +npm-record-and-replay-skill-repo-smoke: + node ./scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs + +record-and-replay-baseline-smoke: + ./scripts/run-record-and-replay-baseline-smoke.sh + +record-and-replay-baseline-audit: + @mkdir -p "$$(dirname "$(RNR_BASELINE_SUMMARY_JSON)")" + ./scripts/run-record-and-replay-baseline-smoke.sh --summary-json "$(RNR_BASELINE_SUMMARY_JSON)" + +record-and-replay-official-golden-gate: + ./scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden + +record-and-replay-official-golden-gate-audit: + @mkdir -p "$$(dirname "$(RNR_OFFICIAL_GOLDEN_SUMMARY_JSON)")" + ./scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden --summary-json "$(RNR_OFFICIAL_GOLDEN_SUMMARY_JSON)" + +codex-record-and-replay-installer-smoke: + node ./scripts/test-codex-record-and-replay-installer.mjs + stress: ./scripts/run-tool-stress-tests.sh diff --git a/README.md b/README.md index 3f1cd14a..27ad2555 100644 --- a/README.md +++ b/README.md @@ -75,7 +75,7 @@ Or add it to your own client manually: ### Skill -Install the skill directly: +Install the general Computer Use skill directly: ```bash # Install for Codex @@ -83,20 +83,76 @@ npx skills add iFurySt/open-codex-computer-use -g -a codex --skill open-computer npx skills ls -g -a codex | rg 'open-computer-use' ``` +Install the dedicated Record & Replay workflow skill when you only want the +macOS recording-to-skill flow: + +```bash +npx skills add iFurySt/open-codex-computer-use -g -a codex --skill open-computer-use-record-and-replay -y +``` + Install for Claude Code: ```bash npx skills add iFurySt/open-codex-computer-use -g -a claude-code --skill open-computer-use -y +npx skills add iFurySt/open-codex-computer-use -g -a claude-code --skill open-computer-use-record-and-replay -y ``` -Update an existing global install, including the Codex install created above: +Update an existing global install, including the Codex installs created above: ```bash npx skills update open-computer-use -g -y +npx skills update open-computer-use-record-and-replay -g -y ``` You can also manually download and install the -[`open-computer-use` skill](./skills/open-computer-use). +[`open-computer-use` skill](./skills/open-computer-use) or the +[`open-computer-use-record-and-replay` skill](./skills/open-computer-use-record-and-replay). + +### Record & Replay Quick Start + +Use the dedicated Record & Replay surface when you want a user to demonstrate +a macOS workflow and turn it into a reusable skill: + +```bash +# Install the official-compatible 3-tool Record & Replay MCP surface into Codex +open-computer-use install-codex-record-and-replay-mcp + +# Optional: install the thin workflow skill +npx skills add iFurySt/open-codex-computer-use -g -a codex --skill open-computer-use-record-and-replay -y +``` + +The MCP surface intentionally exposes only: + +```text +event_stream_start +event_stream_status +event_stream_stop +``` + +OCU-specific lifecycle helpers stay in the CLI layer: + +```bash +# Start a recording through the OCU app agent and show local controls +open-computer-use event-stream start --json + +# Wait for Done / Discard from an independent wrapper +open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' + +# Validate and turn a completed session into a first-draft skill +open-computer-use event-stream validate --json --strict-ocu --require-skill-draft +open-computer-use event-stream summarize --json +open-computer-use event-stream scaffold-skill --json \ + --skill-name \ + --description "" \ + --output-dir +``` + +To split Record & Replay into its own thin skill repo: + +```bash +open-computer-use scaffold-record-and-replay-skill-repo --output-dir ./open-computer-use-rnr-skill-repo +(cd ./open-computer-use-rnr-skill-repo && ./scripts/check.sh) +``` ## More @@ -107,6 +163,9 @@ Besides the MCP JSON config above, you can also use the built-in commands: open-computer-use install-codex-mcp ocu install-codex-mcp +# Install the Record & Replay MCP surface into Codex separately +open-computer-use install-codex-record-and-replay-mcp + # Install as a Codex plugin, mainly for Codex App open-computer-use install-codex-plugin @@ -135,8 +194,115 @@ open-computer-use call --calls-file examples/textedit-overlay-seq.json --sleep 0 # Check permissions; onboarding only opens when something is missing open-computer-use doctor +# Record & Replay-compatible event stream +open-computer-use install-codex-record-and-replay-mcp +open-computer-use event-stream mcp +open-computer-use event-stream start --json +open-computer-use event-stream status --json +open-computer-use event-stream stop --json +open-computer-use event-stream cancel --json +open-computer-use event-stream wait --json --session-id --timeout 30 +# wait --json includes OCU extension fields waitTimedOut and optional notification +open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' +open-computer-use event-stream validate --json --strict-ocu +open-computer-use event-stream validate --json --require-skill-draft +open-computer-use event-stream summarize --json +open-computer-use event-stream scaffold-skill --json --skill-name --description "" --output-dir +scripts/validate-event-stream-recording.py --strict-ocu +scripts/validate-event-stream-recording.py --require-skill-draft +scripts/summarize-event-stream-recording.py +scripts/scaffold-event-stream-skill.py --skill-name --description "" --output-dir +scripts/scaffold-record-and-replay-skill-repo.py --output-dir /tmp/open-computer-use-rnr-skill-repo +# The installed standalone repo scaffold helper uses Python 3; set PYTHON=/path/to/python3 if needed +open-computer-use scaffold-record-and-replay-skill-repo --output-dir /tmp/open-computer-use-rnr-skill-repo +(cd /tmp/open-computer-use-rnr-skill-repo && ./scripts/check.sh) +scripts/import-event-stream-fixture.py --name --source official +# Inspect hosted official JSON first without creating a fixture. +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --mcp-transcript --require-mcp-transcript-evidence --inspect-only +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --mcp-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# Or generate a capture packet first; replace the placeholder hosted JSON, then run its wrappers. +make record-and-replay-official-golden-capture-packet RNR_SCENARIO=simple-action-stop RNR_PACKET_DIR= +scripts/finalize-record-and-replay-official-capture-packet.py --packet-dir --start-json --status-json --stop-json --final-status-json +(cd && ./verify-inputs.sh && ./inspect-only.sh && ./import-fixture.sh) +(cd && ./strict-golden-gate.sh) +# Only use this while a strict gate failure is expected because required official golden is missing/not-ready. +(cd && ./strict-expected-failure-audit.sh) +# Generate required + recommended capture packets and batch wrappers. +make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR= +(cd && ./verify-all.sh && ./inspect-all.sh && ./import-all.sh) +# Optional calibration: imports same-scenario OCU candidates and prints pairing/fixture-set commands +(cd && ./ingest-ocu-candidates.sh) +(cd && ./strict-expected-failure-audit.sh) +# Or pipe a separate transcript JSON while reading the status JSON from a file +cat | scripts/ingest-official-record-and-replay-fixture.py --status-json --mcp-transcript - --name --scenario simple-action-stop --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# Or pipe a hosted event_stream_stop/status JSON directly; use --status-json-base-dir if it contains relative paths +cat | scripts/ingest-official-record-and-replay-fixture.py --status-json - --status-json-base-dir --name --scenario simple-action-stop --use-status-json-as-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# --check-coverage reports required scenario coverage and, when present, required fixture readiness. +# --require-coverage also requires the required scenario to pass readiness. +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --require-coverage +# --smoke-json can consume mcpTranscriptPath from action smoke output. +# Keep the smoke temp dir when saving stdout for later import: +# OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP=1 make event-stream-action-smoke > /tmp/action-smoke.jsonl +# Action smoke defaults to mixed-action-stop; set OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop +# or drag-stop to sample a specific real-input candidate scenario. +# Or let the importer run action smoke and retain the evidence until import finishes: +# scripts/ingest-ocu-record-and-replay-candidate.py --run-action-smoke --scenario drag-stop ... +scripts/ingest-ocu-record-and-replay-candidate.py --smoke-json --name --scenario simple-action-stop --official-root --check-fixture-set +scripts/compare-event-stream-recordings.py --require-same-event-sequence --require-same-schema --require-ax-diff-evidence +scripts/probe-event-stream-recording.py --target local --start-stop +scripts/probe-event-stream-recording.py --target official + # Run local validation from a source checkout +make ci make smoke +make event-stream-surface-smoke +make event-stream-probe +make event-stream-probe-fixture-smoke +make event-stream-official-probe +make event-stream-smoke +make event-stream-smoke-matrix +make event-stream-fixture-smoke +make event-stream-official-fixture-ingest-smoke +make event-stream-official-fixture-coverage-smoke +make event-stream-ocu-candidate-ingest-smoke +make event-stream-compare-smoke +make event-stream-skill-scaffold-smoke +make record-and-replay-skill-repo-smoke +# Requires local dist/ native artifacts; verifies npm-installed repo scaffold command +make npm-record-and-replay-skill-repo-smoke +# Opt-in baseline: proves the current OCU baseline is usable and reports official golden scenario coverage +make record-and-replay-baseline-smoke +# Release / standalone evidence: writes dist/record-and-replay-baseline-summary.json +make record-and-replay-baseline-audit +# Fast fixture-only gate: requires imported official successful recordings to pass readiness +make record-and-replay-official-golden-fixture-gate +# Read-only next-step planner for same-scenario OCU candidate capture/import after official fixture import +make record-and-replay-ocu-candidate-pairing-preflight +# Strict release gate: same baseline checks, plus required official successful recording readiness +make record-and-replay-official-golden-gate +# Strict release evidence: writes dist/record-and-replay-official-golden-gate-summary.json. +# If required official golden is missing/not-ready, audit that expected failure explicitly: +make record-and-replay-official-golden-gate-audit +scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing +# Force non-interactive start approval for local automation +OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=approve make event-stream-smoke +# Verify host-side MCP elicitation approval without changing the 3-tool surface +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=deny make event-stream-smoke +# App-agent wait smoke also verifies wait --notify-command callback delivery +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1 make event-stream-smoke +# Optional matrix additions +OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_SCREENSHOTS=1 make event-stream-smoke-matrix +OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_OFFICIAL=1 make event-stream-smoke-matrix +# Optional official non-recording surface drift check +./scripts/compare-event-stream-surface.py --use-default-official +# Optional official raw start/status/stop probe; currently expected to depend on host/runtime state +make event-stream-official-start-probe OPEN_COMPUTER_USE_STRESS_LOOPS=20 make stress make agent-smoke make agent-smoke SCENARIO=fixture-full diff --git a/README.zh-CN.md b/README.zh-CN.md index 2cd60f82..969931b8 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -82,18 +82,71 @@ npx skills add iFurySt/open-codex-computer-use -g -a codex --skill open-computer npx skills ls -g -a codex | rg 'open-computer-use' ``` +如果只需要 macOS 录制工作流并生成可复用 skill，可以安装独立的 +Record & Replay workflow skill： + +```bash +npx skills add iFurySt/open-codex-computer-use -g -a codex --skill open-computer-use-record-and-replay -y +``` + 安装到 Claude Code ``` npx skills add iFurySt/open-codex-computer-use -g -a claude-code --skill open-computer-use -y +npx skills add iFurySt/open-codex-computer-use -g -a claude-code --skill open-computer-use-record-and-replay -y ``` 更新已有的全局安装，包括上面安装到 Codex 的那份： ```bash npx skills update open-computer-use -g -y +npx skills update open-computer-use-record-and-replay -g -y +``` + +也可以手动下载 [`open-computer-use` skill](./skills/open-computer-use) 或 +[`open-computer-use-record-and-replay` skill](./skills/open-computer-use-record-and-replay) 安装 + +### Record & Replay 快速开始 + +当你想让用户演示一个 macOS 工作流，并把录制结果转成可复用 skill 时，使用独立的 +Record & Replay MCP surface： + +```bash +# 安装官方兼容的 Record & Replay 三件套 MCP surface 到 Codex +open-computer-use install-codex-record-and-replay-mcp +``` + +这个 MCP surface 只暴露官方兼容的三个工具： + +```text +event_stream_start +event_stream_status +event_stream_stop +``` + +OCU 自己的控制条、等待和回调能力放在 CLI 扩展层： + +```bash +# 通过 OCU app agent 开始录制，并显示本地控制条 +open-computer-use event-stream start --json + +# 独立 wrapper 可以等待用户点击 Done / Discard 后继续 +open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' + +# 校验完成态 session，并生成第一版 skill 草稿 +open-computer-use event-stream validate --json --strict-ocu --require-skill-draft +open-computer-use event-stream summarize --json +open-computer-use event-stream scaffold-skill --json \ + --skill-name \ + --description "" \ + --output-dir ``` -也可以手动下载 [`open-computer-use` skill](./skills/open-computer-use) 安装 +如果要把 Record & Replay 拆成独立 thin skill repo： + +```bash +open-computer-use scaffold-record-and-replay-skill-repo --output-dir ./open-computer-use-rnr-skill-repo +(cd ./open-computer-use-rnr-skill-repo && ./scripts/check.sh) +``` ## 更多 @@ -103,6 +156,9 @@ npx skills update open-computer-use -g -y # 一键安装到 Codex，写到 ~/.codex/config.toml 中 open-computer-use install-codex-mcp +# 单独安装 Record & Replay MCP surface，写到 ~/.codex/config.toml 中 +open-computer-use install-codex-record-and-replay-mcp + # 一键安装到 Codex 插件，主要方便在 Codex App 中使用 open-computer-use install-codex-plugin @@ -131,6 +187,66 @@ open-computer-use call --calls-file examples/textedit-overlay-seq.json --sleep 0 # 检查权限；只有缺失时才会拉起引导，已全部授权则只打印状态并退出 open-computer-use doctor +# Record & Replay 兼容 event stream +open-computer-use event-stream mcp +open-computer-use event-stream start --json +open-computer-use event-stream status --json +open-computer-use event-stream stop --json +open-computer-use event-stream cancel --json +open-computer-use event-stream wait --json --session-id --timeout 30 +open-computer-use event-stream validate --json --strict-ocu +open-computer-use event-stream scaffold-skill --json --skill-name --description "" --output-dir + +# 导入官方 hosted Record & Replay 返回的录制样本，生成脱敏 fixture 并跑 readiness +# 建议先 inspect，不创建 fixture，只确认 hosted JSON 能解析出 handoff path 和 transcript 证据 +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --mcp-transcript --require-mcp-transcript-evidence --inspect-only +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --mcp-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# 也可以先生成 capture packet；替换其中的 hosted JSON 占位文件后，按 wrapper 顺序校验、inspect、导入和刷新 strict artifact +make record-and-replay-official-golden-capture-packet RNR_SCENARIO=simple-action-stop RNR_PACKET_DIR= +scripts/finalize-record-and-replay-official-capture-packet.py --packet-dir --start-json --status-json --stop-json --final-status-json +(cd && ./verify-inputs.sh && ./inspect-only.sh && ./import-fixture.sh) +(cd && ./strict-golden-gate.sh) +# 只有 required official golden 缺失或 readiness 未过时，才审计 strict 预期失败 artifact +(cd && ./strict-expected-failure-audit.sh) +# 生成 required + recommended capture packet set 和批量 wrapper +make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR= +(cd && ./verify-all.sh && ./inspect-all.sh && ./import-all.sh) +# 可选校准：导入同场景 OCU candidate，并打印后续 pairing / fixture-set 命令 +(cd && ./ingest-ocu-candidates.sh) +(cd && ./strict-expected-failure-audit.sh) +# 也可以从 stdin 传入单独的 transcript JSON，此时 status JSON 需要来自文件 +cat | scripts/ingest-official-record-and-replay-fixture.py --status-json --mcp-transcript - --name --scenario simple-action-stop --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# 也可以直接通过 stdin 管道传入 hosted event_stream_stop/status JSON；如果里面是相对路径，用 --status-json-base-dir 指定解析基准目录 +cat | scripts/ingest-official-record-and-replay-fixture.py --status-json - --status-json-base-dir --name --scenario simple-action-stop --use-status-json-as-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage +# --check-coverage 会报告 required 场景覆盖，并在样本存在时同步跑 required fixture readiness +# 采集 required 场景后可以用 --require-coverage，让导入后覆盖率或 readiness 不满足时直接失败 +scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario simple-action-stop --require-coverage +# 单独检查仓库里的 required official successful recording fixture 覆盖；排查缺失时可用 --allow-missing 只输出报告 +scripts/check-event-stream-official-fixture-coverage.py --allow-missing --check-readiness +# 导入 OCU action smoke 或本地 recording，生成同 scenario 的 candidate fixture +# --smoke-json 会自动消费 action smoke 输出里的 mcpTranscriptPath。 +# 如果要保存 stdout 后再导入，需要保留 smoke 临时目录： +# OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP=1 make event-stream-action-smoke > /tmp/action-smoke.jsonl +# action smoke 默认跑 mixed-action-stop；设置 OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop +# 或 drag-stop 可以采样指定真实输入 candidate 场景。 +# 也可以让导入脚本自己跑 action smoke，并在导入完成前保留证据： +# scripts/ingest-ocu-record-and-replay-candidate.py --run-action-smoke --scenario drag-stop ... +scripts/ingest-ocu-record-and-replay-candidate.py --smoke-json --name --scenario simple-action-stop --official-root --check-fixture-set +# opt-in baseline：证明当前 OCU baseline 可用，并报告官方 golden 场景覆盖 +make record-and-replay-baseline-smoke +# release / standalone 留证入口：写出 dist/record-and-replay-baseline-summary.json +make record-and-replay-baseline-audit +# 快速 fixture-only gate：要求入库的官方 successful recording 通过 readiness +make record-and-replay-official-golden-fixture-gate +# 只读下一步规划：官方 fixture 入库后，生成同 scenario OCU candidate 采样/导入/配对命令 +make record-and-replay-ocu-candidate-pairing-preflight +# 严格 release gate：同样跑 baseline 检查，并要求 required 官方 successful recording readiness 通过 +make record-and-replay-official-golden-gate +# 严格 release 留证入口：写出 dist/record-and-replay-official-golden-gate-summary.json； +# required 官方 golden 缺失或未 ready 时，用 allow-missing 模式确认这是预期失败而不是 baseline 回归 +make record-and-replay-official-golden-gate-audit +scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing + # 查看帮助 open-computer-use -h ocu -h diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MCPAppRuntime.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MCPAppRuntime.swift index cc6d6576..dd388d82 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/MCPAppRuntime.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MCPAppRuntime.swift @@ -3,16 +3,16 @@ import Foundation import OpenComputerUseKit final class MCPAppRuntime: NSObject, NSApplicationDelegate { - private let server: StdioMCPServer + private let server: LineBasedMCPServer private var runtimeError: Error? private var turnEndedObserver: NSObjectProtocol? - private init(server: StdioMCPServer) { + private init(server: LineBasedMCPServer) { self.server = server } @MainActor - static func run(server: StdioMCPServer) throws { + static func run(server: LineBasedMCPServer) throws { let application = NSApplication.shared application.setActivationPolicy(.accessory) diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift index d51fa228..4a712829 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift @@ -37,7 +37,10 @@ enum MacOSAppAgentProxy { switch command { case .mcp: - try proxyMCP(client: client) + try proxyMCP(kind: "mcp", client: client) + return EXIT_SUCCESS + case .eventStream(.mcp): + try proxyMCP(kind: "eventStreamMCP", client: client) return EXIT_SUCCESS default: let response = try sendCLIRequest(arguments: arguments, client: client) @@ -107,16 +110,21 @@ enum MacOSAppAgentProxy { throw OpenComputerUseCLIError(message: "Timed out waiting for Open Computer Use.app agent to start.") } - private static func proxyMCP(client: AppAgentSocketClient) throws { + private static func proxyMCP(kind: String, client: AppAgentSocketClient) throws { while let line = readLine(strippingNewline: true) { guard !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue } - let response = try client.request([ - "kind": "mcp", + var request: [String: Any] = [ + "kind": kind, "line": line, - ]) + ] + if kind == "eventStreamMCP" { + request["environment"] = proxiedEnvironment() + } + + let response = try client.request(request) if let responseLine = response["response"] as? String { FileHandle.standardOutput.write(Data((responseLine + "\n").utf8)) @@ -292,6 +300,7 @@ private final class AppAgentSocketListener: @unchecked Sendable { private final class AppAgentConnection: @unchecked Sendable { private let fileDescriptor: Int32 private let server = StdioMCPServer() + private lazy var eventStreamServer = EventStreamMCPServer(service: .shared) init(fileDescriptor: Int32) { self.fileDescriptor = fileDescriptor @@ -337,12 +346,20 @@ private final class AppAgentConnection: @unchecked Sendable { return ["response": response] } return ["response": NSNull()] - case "cli": - let arguments = request["arguments"] as? [String] ?? [] + case "eventStreamMCP": + let line = request["line"] as? String ?? "" let environment = request["environment"] as? [String: String] ?? [:] let response = AppAgentEnvironment.withOverrides(environment) { - runCLI(arguments: arguments) + eventStreamServer.handle(line: line) } + if let response { + return ["response": response] + } + return ["response": NSNull()] + case "cli": + let arguments = request["arguments"] as? [String] ?? [] + let environment = request["environment"] as? [String: String] ?? [:] + let response = runCLI(arguments: arguments, environment: environment) return [ "stdout": response.stdout, "stderr": response.stderr, @@ -357,7 +374,7 @@ private final class AppAgentConnection: @unchecked Sendable { } } - private func runCLI(arguments: [String]) -> CLIProxyResponse { + private func runCLI(arguments: [String], environment: [String: String]) -> CLIProxyResponse { do { let command = try parseOpenComputerUseCLI(arguments: arguments) @@ -390,7 +407,17 @@ private final class AppAgentConnection: @unchecked Sendable { return CLIProxyResponse(stdout: text + "\n", stderr: "", exitCode: EXIT_SUCCESS) case let .call(invocation): - let output = try runOpenComputerUseCall(invocation) + let output = try AppAgentEnvironment.withOverrides(environment) { + try runOpenComputerUseCall(invocation) + } + return CLIProxyResponse( + stdout: try output.jsonText() + "\n", + stderr: "", + exitCode: output.hasToolError ? EXIT_FAILURE : EXIT_SUCCESS + ) + + case let .eventStream(invocation): + let output = try runEventStreamCLI(invocation: invocation, environment: environment) return CLIProxyResponse( stdout: try output.jsonText() + "\n", stderr: "", @@ -405,6 +432,19 @@ private final class AppAgentConnection: @unchecked Sendable { return CLIProxyResponse(stdout: "", stderr: message + "\n", exitCode: EXIT_FAILURE) } } + + private func runEventStreamCLI(invocation: EventStreamCLIInvocation, environment: [String: String]) throws -> OpenComputerUseCallOutput { + switch invocation { + case .start: + return try AppAgentEnvironment.withOverrides(environment) { + try runOpenComputerUseEventStream(invocation) + } + case .status, .stop, .cancel, .wait, .summarize, .validate, .scaffoldSkill: + return try runOpenComputerUseEventStream(invocation) + case .mcp: + throw OpenComputerUseCLIError(message: "event-stream mcp is a server command", helpCommand: "event-stream") + } + } } private enum AppAgentEnvironment { diff --git a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift index cef6d82e..87feda2e 100644 --- a/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift +++ b/apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift @@ -48,6 +48,20 @@ enum OpenComputerUseMain { } else { try server.run() } + case let .eventStream(invocation): + switch invocation { + case .mcp: + let server = EventStreamMCPServer(service: .shared) + try MainActor.assumeIsolated { + try MCPAppRuntime.run(server: server) + } + default: + let output = try runOpenComputerUseEventStream(invocation) + print(try output.jsonText()) + if output.hasToolError { + exit(EXIT_FAILURE) + } + } case .doctor: let permissions = PermissionDiagnostics.current() print(permissions.summary) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 234452bf..cb358a96 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,7 +5,7 @@ ## 当前目录结构 - `apps/OpenComputerUse` - 主入口，负责 `mcp`、`doctor`、`list-apps`、`snapshot`、`call`、`turn-ended` 等 CLI 命令，以及 `-h` / `--help` / `-v` / `--version` 这类全局参数；不带参数启动时会先检查权限，只有缺失时才进入无 Dock 图标的 app 模式权限引导窗口，`doctor` 也只会在检测到缺失权限时拉起这套 onboarding UI。 + 主入口，负责 `mcp`、`event-stream`、`doctor`、`list-apps`、`snapshot`、`call`、`turn-ended` 等 CLI 命令，以及 `-h` / `--help` / `-v` / `--version` 这类全局参数；不带参数启动时会先检查权限，只有缺失时才进入无 Dock 图标的 app 模式权限引导窗口，`doctor` 也只会在检测到缺失权限时拉起这套 onboarding UI。 - `apps/OpenComputerUseFixture` 本地 GUI fixture app，用来承载低风险、可预测的点击/输入/滚动/拖拽验证路径。 - `apps/OpenComputerUseSmokeSuite` @@ -29,7 +29,7 @@ - `scripts/` 仓库级自动化命令，包括 smoke test、`.app` 打包入口、Windows `.exe` / Linux binary 构建入口、npm 分发脚本，以及 `scripts/computer-use-cli/` 这个用于探测官方 bundled `computer-use` 的 Go helper。 - `skills/` - 面向 agent runtime 的可安装 skill。当前 `skills/open-computer-use/SKILL.md` 只作为轻量入口和目录，安装、MCP/CLI 使用、排障等细节拆到相邻 `references/` 文件里按需加载；`scripts/package-skill.sh` 负责校验并打包 `.zip` / `.skill` 制品。 + 面向 agent runtime 的可安装 skill。`skills/open-computer-use/SKILL.md` 是通用 Computer Use 入口，安装、MCP/CLI 使用、排障和 Record & Replay 细节拆到相邻 `references/` 文件里按需加载；`skills/open-computer-use-record-and-replay/SKILL.md` 是独立 Record & Replay thin skill，只规定 macOS 录制、停止、读取 session 产物和生成可复用 skill 的流程，便于后续迁到独立 repo。`scripts/scaffold-record-and-replay-skill-repo.py` 可把这个 thin skill 导出成 standalone repo 骨架，npm 包也暴露同等安装态命令 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `；生成物只依赖已安装的 `open-computer-use` runtime，不复制本仓库源码。骨架内自带 `scripts/package-skill.sh`、`scripts/verify-manifest.py`、`scripts/verify-source-baseline-summary.py`、`scripts/verify-readme-handoff.py`、`scripts/verify-runtime.py`、`scripts/verify-skill-workflow.py`、`scripts/wait-notify-contract-smoke.py`、`scripts/recording-to-skill-smoke.py`、组合门禁 `scripts/check.sh`、可选真实录制闭环检查 `scripts/recording-lifecycle-smoke.py` 和 pin 到 commit SHA 的 `.github/workflows/ci.yml`，并在 `record-and-replay-skill-repo.json` 里把默认 `checks`、OCU `extensionLayer` 与可选 `optionalChecks` 分开声明。默认自检不启动录制，会校验 skill frontmatter、打包 skill、用 `scripts/verify-manifest.py` 校验 `record-and-replay-skill-repo.json` 里的官方证据、source repo audit、`manifestContract`、OCU 扩展边界和 recording-to-skill 门禁没有漂移，用 `scripts/verify-source-baseline-summary.py` 审计随 scaffold 复制的 `evidence/source-baseline-summary.json`，确认源仓 `dist/record-and-replay-baseline-summary.json` 的脱敏投影仍显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`，official golden 状态要么保持当前 required gap 明确、要么在未来 official golden/equivalence ready 时完成态自洽，`preflightPipelines.checkedOfficialCapturePacketSetContractManifest=true`、`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff=true` 和 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff=true` 被保留，并且 `checkedSourceBaselineSummaryEvidence=true` / `checkedSourceBaselineSummaryOfficialGoldenState=true` / `checkedSourceBaselineSummaryCapturePacketSetContractManifest=true` / `checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true` / `checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true` / `checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` / `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true` 进入 source / npm staged evidence，用 `scripts/verify-readme-handoff.py` 校验 README 里的官方证据、baseline / strict audit handoff、official golden 缺口、recording-to-skill 命令和 wait/notify 边界没有漂移，握手验证 `open-computer-use event-stream mcp` 的官方三件套 surface 与 no-active status/stop response shape，验证 thin skill 保留官方 handoff 语义（start 后结束 turn、不轮询、用户完成后 stop、取消不再 stop）、OCU wait/notify 扩展边界（listener 不进入 MCP surface、notify 通过 stdin/env 回调），并通过等待不存在的 session 验证 `waitSessionMatched=false` 时 callback skipped 且不创建 session 文件，通过合成 completed session 验证 notify callback 能收到 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH`，且非零退出或超时都会让 CLI 非零，并分别返回 `reason=nonZeroExit` / `reason=timeout`，再用临时合成 recording 验证安装态 `validate --require-skill-draft` / events-only gate / `scaffold-skill` 可生成 skill 草稿，同时用无动作 recording 验证失败的 `scaffold-skill` 会返回非零退出码；需要本地确认真实 session 文件链路时，可显式运行 lifecycle smoke 验证重复 start 返回同一 active session、stop 幂等、停止后的 status 仍指向同一个 completed session，并用安装态 runtime validate 校验 recording 产物。`scripts/package-skill.sh` 会扫描 `skills/*/SKILL.md`，校验 frontmatter 名称和描述，并为每个 skill 打包 `.zip` / `.skill` 制品。 - `docs/` 逆向分析、执行计划、history 和项目约束。 @@ -47,6 +47,8 @@ ### 2. MCP 层 - 面向 MCP host 的外部 transport 仍是 `stdio`；macOS 终端 CLI 到 `.app` app agent 之间额外有一层本地 Unix domain socket 代理，用来保证真实 automation 运行在 app bundle 权限身份下。 +- `open-computer-use event-stream mcp` 启动 Record & Replay 兼容的独立 MCP surface，server name 为 `Record & Replay`，当前暴露 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三个无参数 tools；它复用同一套 stdio framing 和 app-agent 代理，但使用独立的 event-stream dispatcher。app-agent 内的 event-stream MCP server 按需创建，避免在 CLI `event-stream start` 的环境覆盖生效前提前初始化共享 recorder。start approval policy 设为 `mcp` / `elicitation` 时，server 只在 host 的 `initialize` 明确声明 `capabilities.elicitation` 后发送内部 `elicitation/create` 请求，并把 accept / decline / cancel 映射回 recorder approval；tools/list 和 tool input schema 不因此增加任何参数。 +- Codex 安装入口现在分成两个幂等脚本：`open-computer-use install-codex-mcp` 写入普通 9-tool Computer Use server，`open-computer-use install-codex-record-and-replay-mcp` 写入独立 `record_and_replay` server，启动参数为 `["event-stream","mcp"]`。这避免上层 skill/plugin 继续手写 TOML，也避免把 Record & Replay 扩展混入普通 MCP server。 - 当 `OPEN_COMPUTER_USE_VISUAL_CURSOR` 未被显式关闭时，`mcp` 命令会切到一个最小 AppKit runtime：主线程保留 event loop 承载 overlay UI，stdio server 仍在后台线程串行读取与响应。 - 请求 framing 采用一行一个 JSON-RPC message。 - 当前支持的 method： @@ -61,6 +63,9 @@ ### 3. Tool Service 层 - `ComputerUseService` 负责把 Computer Use tool 请求映射到本地能力，`ComputerUseToolDispatcher` 则把 9 个 tool 的参数解析与 service 方法分发收敛成 MCP server 和 `open-computer-use call` 共用的一层。 +- `EventStreamService` 负责 Record & Replay baseline 录制状态。它在长驻 app-agent / MCP server 进程内维护 active session，写入 `events.jsonl`、OCU 完整状态 `metadata.json`、官方风格最小 handoff `session.json`、`suppressed.jsonl`、`latest-session.json` 和录制中的 `active-session.json`，并记录 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.click`、`mouse.context_menu`、`mouse.drag`、`keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut`、`terminal.value_changed`、`selection.changed`、`debug.error`、`session.ended` 这类基础事件；主事件和 suppressed 事件都会双写官方观察到的 `kind` 字段与 OCU 既有兼容字段 `type`，源码 validator / summarizer / fixture import / readiness / compare 链路也接受 `kind`-only 官方样本；官方兼容 MCP start/status/stop response 会暴露 `sessionID`、`sessionDirectoryPath`，并让 `metadataPath` 指向 `session.json`，贴近 Codex-hosted completed session handoff；Swift/Python strict validator 在同目录存在 `metadata.json` 时仍以完整 OCU metadata 校验计数、路径和状态，同时接受 `session.json` 与 `metadata.json` 语义兼容但不字节相等。鼠标事件现在按 down / dragged / up 维护 pending pointer，只有 mouse up 后才决定写 `mouse.click` / `mouse.context_menu` 还是 `mouse.drag`，避免把拖拽误记成点击。`mouse.click` / `mouse.context_menu` 会记录 y-down screen-state `location`、原始 AppKit `appKitLocation`，并尽量通过 `AXUIElementCopyElementAtPosition` 补 `targetAccessibilityElement`；`mouse.drag` 会记录起止 y-down `startLocation` / `endLocation`、起止 AppKit location、距离、时长和同样的目标 AX 摘要。键盘事件会尽量补当前 `focusedAccessibilityElement`，帮助后续 skill 识别用户输入、提交和快捷键目标；如果 focused element 暴露 protected/secure/password 信号，则只写 `secureInput=true` 和文本长度，且 raw key event 不写明文 `characters`；如果 focused element 属于明显终端上下文，则键盘 raw characters 和 focused element value 也会脱敏，并在 AX value 变化时写轻量 `terminal.value_changed`，只保留 value hash / length。输入事件后如果当前 focused AX element 暴露非空 `selectedText` 且 selection signature 变化，会写轻量 `selection.changed` 事件；如果此前有选择而当前选择为空，会写 `selectionCleared=true` 的清空事件；secure input、terminal value、selection 字段与清空选择语义仍待 official golden recording 校准。官方字符串出现过的 `experimentalRawEvents` 现在作为 OCU baseline 默认开启，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS=0` 关闭；点击、拖拽、键盘事件会附带底层 NSEvent 摘要，scroll wheel 在未确认官方独立 scroll event 名称前只写 `kind=experimentalRawEvents` / `type=experimentalRawEvents` 的 raw-only 事件，不臆造 `mouse.scroll`。`window.changed` 会在录制开始和输入事件前 app/window context 变化时写入，并继续补 `AX.focusedWindowChanged` 作为 AX 上下文；`AX.focusedWindowChanged` 会尽量附带 `accessibilityInspectorPayload`，首个 payload 是 full tree，并双写官方观察到的 `fullTree` 与 OCU 既有 `treeLines`，同一 app/window 的后续 payload 会用轻量 `~` / `+` / `-` compact diff，diff baseline 已改为 LCS 风格行 diff 并归一化 AX tree leading element index，避免插入元素后把后续稳定行误判为变更；diff payload 会保留 previous diff 并附加 cumulative diff baseline。字段、初始帧语义、diff 算法和事件顺序仍待 official golden recording 校准。AX snapshot 失败、cumulative diff 过大、截图写入失败和 Input Monitoring monitor 全部安装失败等上下文采集降级会写低敏 `debug.error` 主事件，并继续把更完整细节写入 `suppressed.jsonl`；其中 input monitor 不可用会写 `subsystem=inputMonitoring` / `reason=inputMonitorsUnavailable`，避免独立集成方拿到无输入事件的空录制却没有诊断信号。截图上下文由 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never` 控制，默认 `auto`，只在 payload 标记 `screenshotNeededForContext` 且 snapshot 有截图数据时写入 session 的 `screenshots/` 并返回 `screenshotPath`。录制默认最多持续 30 分钟，超时会自动写入 `session.ended` 并以 `recording_time_limit_reached` 结束；该 endReason 是 OCU baseline，仍待 official golden recording 校准。测试和本地调试可用 `OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS` 覆盖时长。`event-stream start/status/stop/cancel/wait --json` 会通过 app-agent 访问同一份进程内状态；app-agent 对 `status/stop/cancel/wait` 不持有全局环境覆盖锁，避免上层 listener 正在 `wait` 时阻塞另一个 `stop` / `cancel`；`wait --json` 是 OCU 扩展层 listener，会在返回体增加 `waitTimedOut` 和 `waitSessionMatched` 标记，方便独立 plugin/skill 区分结束唤醒、超时返回和 session id 不匹配。`wait --session-id` 会优先读取目标 session 目录中的完成态 metadata，即使 latest session 已被后续录制覆盖也能返回指定历史 session；未知或不匹配的 session id 会快速返回 `waitTimedOut=true` / `waitSessionMatched=false` 并跳过 notify callback。未通过 app-agent 的短生命周期 CLI 只适合调试。如果新的 recorder 进程发现 `latest-session.json` 仍是 active 状态但进程内没有 active session，会把它持久化改写为 `state=stopped`、`endReason=recording_process_unavailable` 并清理 stale `active-session.json`，避免独立集成方一直看到幽灵录制。start 前已接 OCU baseline approval gate：已有 active session 时直接返回，新的 session 只有 approved 后才创建；denied / cancelled 使用官方已观察错误文案且不写 session 文件，`OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=auto|interactive|approve|deny|cancel|mcp|elicitation` 可控制策略，自动化验证显式使用 `approve`，MCP elicitation smoke 显式使用 `mcp`。交互式录制默认先展示 OCU 自有开始确认弹窗，开始后展示 Record & Replay 控制条，支持 Done、Discard、计时和拖动；自动化验证可用 `OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS=0` 关闭控制条。 +- EventStream 输入监听在生产路径优先安装 listen-only `CGEvent` tap，捕获 session 级鼠标、键盘和 scroll 输入；如果 event tap 不可用，再回退到原 `NSEvent.addGlobalMonitorForEvents` 路径。`make event-stream-action-smoke` 是 opt-in 真实输入链路验证：默认 `mixed-action-stop` 让 app-agent recorder 录制外部进程发出的低影响 CGEvent click，并继续发送 best-effort scroll wheel；release baseline 必需证据是 `mouse.click`、AX context、MCP transcript 和 skill handoff，捕获到的 scroll raw-only `experimentalRawEvents` 仍会进入 summary/scaffold，但不作为硬门槛。设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop` 时，同一 smoke 会分别采样 click 或 drag，用于生成对应 OCU candidate。完整 baseline audit 会同时消费默认 mixed action smoke 和 simple / drag candidate smoke，并在 `evidence.realInputActionSmoke` 中要求 `checkedSimpleActionStopCandidate` / `checkedDragStopCandidate` 为 true；这只是 OCU candidate 证据，仍不能替代 official successful recording golden。`keyboard-input-stop` 当前通过已有 recording / smoke JSON 导入和 readiness 校验覆盖，不走 synthetic keyboard `--run-action-smoke`，避免 macOS 过滤合成键盘事件造成真实输入 smoke 不稳定。summary/scaffold 会把捕获到的 `reason=scrollWheel` raw-only event 渲染成 Scroll replay step，同时确认草稿和 `recording-summary.json` 不泄漏本机绝对路径且 summary 仍可创建 skill 草稿。 +- OCU 的 `event-stream wait --notify-command ''` 是独立集成回调扩展，不属于官方兼容 MCP surface；它只在 `waitTimedOut=false` 时同步执行本地 callback，stdin 写入最终 status JSON，并通过 `OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON`、`OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID`、`OPEN_COMPUTER_USE_EVENT_STREAM_STATE`、`OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON`、`OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH`、`OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH`、`OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH` 和 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` 环境变量传递上下文；callback 默认 10 秒超时，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS` 覆盖，非零退出或超时会让 CLI 返回 error 并写入 `notification` 结果。若 `waitSessionMatched=false`，返回体同时会有 `waitTimedOut=true`，callback 会被跳过。status / metadata JSON 里的 `sessionPath` 是 OCU 便利字段，指向官方风格 `session.json` handoff 文件；官方兼容消费仍以 `metadataPath` / `eventsPath` 为主。 - `list_apps` 通过 Spotlight metadata query 拉取标准 application 目录里的 app bundle，并读取 `kMDItemUseCount` / `kMDItemLastUsedDate_Ranking` 这类系统元数据；再与 `NSWorkspace` 的运行态 app 合并，输出“当前运行中 + 近 14 天用过”的视图。 - `get_app_state` 优先走真实 AX / 窗口截图；真实 app 必须同时有未最小化的 `AXWindow` 和可匹配的 on-screen `CGWindow`。如果目标 app 只是隐藏或暂时没有 on-screen window，会先 best-effort unhide / activate / `open -b` / `AXRaise` 并短暂重试，以贴近官方 `computer-use` 会把 Lark / Electron 窗口拉回再采集的行为；恢复后仍无法匹配时返回官方风格的 `Apple event error -10005: cgWindowNotFound`，不再把 application 根节点或无截图窗口伪装成可操作状态。当目标是仓库内 fixture app 时，回退到 fixture 导出的合成状态。真实 AX tree 对 Electron/WebView 这类深层 UI 会放宽遍历深度，同时压缩空 `AXGroup` / `AXUnknown` wrapper、过滤 `AXScrollToVisible` 噪音和空字符串属性，避免 action-critical 的输入框被无语义容器挤出节点预算；对原生 open panel / Finder column view 这类把内容放在 `AXContents` / `AXVisibleChildren` 里的控件，也会把可见文件项纳入元素树。 - MCP `tools/list` 的 description / input schema 当前按官方 `computer-use` 的 9 个 tools 文案和参数面收敛，尽量减少 host 侧提示词和 tool surface 偏差。 @@ -127,7 +132,7 @@ - 开源版当前不复刻官方闭源实现里的 caller signing、私有 IPC、完整 overlay choreography 和 plugin 自安装逻辑。 - 因为官方 `SkyComputerUseClient` 带有宿主侧 launch constraints，普通 stdio MCP client 在本机上可能被系统直接杀掉；如果要探测官方 bundled `computer-use`，`scripts/computer-use-cli` 的 app-server 模式现在只适合做工具清单和协议面观察。官方 `1.0.755` 的真实 tool call 还会经过 service-side sender authorization / active IPC client 追踪，外部 raw helper 即使走已签名 Codex binary，也可能返回 `Sender process is not authenticated`；需要真实使用官方工具时应走正常 Codex agent/tool 调用链，开源版则继续提供可直连的 `open-computer-use` MCP server。 -- 当前权限引导已经具备可运行 app、深链、拖拽辅助，以及一版更接近官方的 accessory panel 入场动画和返回 affordance；点击链路也已经补上独立 visual cursor、官方 asset fallback 和相对目标 window 的排序逻辑，并且在 overlay 可见期间会持续重申“排在目标 window 之上”，避免用户手动激活目标 app 后 cursor 被目标窗口重新盖住；但整体还没有完全复刻官方那套嵌入式 choreography / host 集成 / session approval 体验。 +- 当前权限引导已经具备可运行 app、深链、拖拽辅助，以及一版更接近官方的 accessory panel 入场动画和返回 affordance；点击链路也已经补上独立 visual cursor、官方 asset fallback 和相对目标 window 的排序逻辑，并且在 overlay 可见期间会持续重申“排在目标 window 之上”，避免用户手动激活目标 app 后 cursor 被目标窗口重新盖住；Record & Replay 目前也已有官方兼容 MCP surface、session 文件、start approval gate、MCP elicitation approval baseline、基础输入事件记录、AX full / lightweight diff payload、按需截图上下文和 OCU 自有录制控制条。但整体还没有完全复刻官方那套嵌入式 choreography / host 集成细节，Record & Replay 也还缺官方算法级 AX compact diff、官方 Record & Replay 真实 elicitation business metadata 和官方 golden fixture 校准。 - screenshot 当前通过 `ScreenCaptureKit` 捕获目标窗口，并以 MCP `image` content block 的 base64 PNG 返回，不再把普通 app 截图落盘到仓库或临时目录；编码前会按最大尺寸和目标字节数自适应缩小，避免复杂页面的大 PNG 触发 host 侧 MCP result 降级，同时 coordinate tools 继续按实际返回的 screenshot pixel 尺寸映射坐标；单次 ScreenCaptureKit capture 会设置超时，超时后省略 image block 而不是卡住整个 `get_app_state`。 - 会话状态现在是进程内内存态，保存每个 app 最近一次 snapshot 和 element index 映射。 @@ -137,11 +142,49 @@ - standalone cursor 构建：`swift build --product StandaloneCursor` - cursor lab 构建：`swift build --product CursorMotion` - 端到端 smoke：`./scripts/run-tool-smoke-tests.sh`（标准 9-tool smoke + visual cursor idle smoke；脚本默认以 headless 模式启动内部 fixture，避免在用户桌面弹出测试窗口） +- Record & Replay event-stream smoke：`./scripts/run-event-stream-smoke-tests.sh` 或 `make event-stream-smoke`，通过真实 `event-stream mcp` stdio 路径验证官方 3 tools、重复 start 返回同一 active session、stop 幂等、session 文件、`session.json` alias 和 AX payload；脚本默认用 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=never` 避免在锁屏/登录窗口等不可截图环境触发 ScreenCaptureKit 噪音；生成录制产物后会调用 runtime CLI `open-computer-use event-stream validate --json --strict-ocu` 与源码脚本 `scripts/validate-event-stream-recording.py --strict-ocu` 统一校验 metadata/session alias、event counts、suppressed counts、`session.ended`、`endReason`、`screenshotPath`、strict OCU current segment path gate，以及 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` declared handoff path gate，并同时调用源码脚本 `scripts/summarize-event-stream-recording.py` 与 runtime CLI `open-computer-use event-stream summarize` 验证事件流可被 skill 生成前置分析读取；默认 lifecycle 还会按官方 skill handoff 方式从 `event_stream_stop` 返回的 text JSON 读取 `eventsPath` / `metadataPath` / `sessionPath`，并用 `sessionPath` alias 再跑 runtime validate/summarize；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS=1` 时额外验证截图上下文，并在当前 snapshot 有截图数据时验证截图落盘；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1` 时通过 MCP status/stop 验证从未 start 时返回官方 no-active `isRecording=false` / `maxDurationSeconds=1800` shape 且不会创建 session 文件；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT=1` 时通过 MCP start/status/stop 验证最长录制时长自动结束路径；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1` 时通过 app-agent CLI start/wait/cancel 验证 `waitTimedOut=true`；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1` 时只探测官方 initialize/tools-list，并与 OCU surface 对比，不启动官方录制，官方响应默认 10 秒超时，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT` 覆盖；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=deny|cancel` 时验证 start approval 拒绝路径不会创建 session；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1` 时验证 MCP elicitation approval；设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1` 时启动 app-agent socket，验证 proxied MCP `event_stream_start` 的环境覆盖、共享 recorder 状态、CLI `wait` 被另一个连接上的 `stop` 唤醒、`wait --notify-command` 收到最终 status JSON 和 session/state 环境变量，以及 session 已完成后的第二次 `wait` 会立即返回最终状态。 +- Record & Replay action smoke：`make event-stream-action-smoke`，opt-in 启动 app-agent recorder，通过 proxied `event-stream mcp` 三件套 start/status/stop 同一 session，默认外部 Swift 进程向 HID tap 发出一次低影响 click 和 best-effort scroll wheel，停止后要求 `mouse.click`、`AX.focusedWindowChanged` 和 `session.ended` 都进入 recording，并用该 recording 调用 runtime `event-stream scaffold-skill --json` 生成 `SKILL.md` 草稿；捕获到的 scroll raw-only `experimentalRawEvents` 会继续由 summary/scaffold 渲染为 Scroll replay step，但不作为 hard gate。设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop` 时，则分别要求 `mouse.click` 或 `mouse.drag` 并校验对应 Replay Steps。同时断言生成的草稿和 `references/recording-summary.json` 不含本机绝对路径、summary 带 `skillReadiness.canCreateSkillDraft=true`。输出 JSON 会包含同 session 的 `mcpTranscriptPath` 和 `actionScenario`，用于后续 `scripts/ingest-ocu-record-and-replay-candidate.py --smoke-json` 自动导入 response-shape evidence；如果要保存 stdout 后再导入，需要设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP=1` 或 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR=` 保留录制和 transcript 临时目录。它用于验证真实输入监听、MCP lifecycle evidence 与 recording-to-skill 链路，不进入默认 CI。 +- Record & Replay surface drift smoke：`make event-stream-surface-smoke` 或 `scripts/compare-event-stream-surface.py`，只调用 initialize / `tools/list`，用已保存的官方 1.0.857 non-recording fixture 校验 OCU `event-stream mcp` 的 protocol、server name、capabilities 和三件套 tool schema。`scripts/compare-event-stream-surface.py --use-default-official` 会额外探测本机默认官方 plugin cache 中的 `SkyComputerUseClient event-stream mcp` 并和同一 fixture 对比；这个官方检查不启动录制，且不进入默认 CI。 +- Record & Replay no-active drift smoke：`make event-stream-no-active-smoke` 或 `scripts/compare-event-stream-no-active.py`，只启动本地 OCU `event-stream mcp`，调用 no-active `event_stream_status` / `event_stream_stop`，用 Codex-hosted 官方 1.0.857 no-active fixture 校验 text JSON 为 `isRecording=false` / `maxDurationSeconds=1800`，并断言不会创建 session 文件；该检查不调用 `event_stream_start`。 +- Record & Replay raw recording probe：`make event-stream-probe` 或 `scripts/probe-event-stream-recording.py --target local --start-stop` 会用同一套 JSON-RPC transcript 跑 OCU start/repeat start/status/stop/repeat stop/final status，验证 probe 脚手架能产出 paths，并采集 one-active、idempotent stop 和 completed status 证据；`make event-stream-official-probe` 只探测官方 bundled client 的 initialize / `tools/list`；`make event-stream-official-start-probe` 显式尝试官方 raw start/repeat start/status/stop/repeat stop/final status 并记录 transcript，当前本机宿主外预期会在首个 start 超时，不进入默认 CI。`make event-stream-probe-fixture-smoke` 校验已入库的官方 raw start-timeout sanitized fixture，并用 fake MCP server 验证 probe 能处理 `elicitation/create`、按 accept `{}` / decline 或 cancel `null` 写回 content、以及把 request form schema 摘进 fixture，确保它不含本机路径且仍表达当前官方采样边界。 +- Record & Replay local probe smoke：`make event-stream-local-probe-smoke` 或 `scripts/test-event-stream-local-probe.py` 会构建本地 `OpenComputerUse`，通过 raw probe 采集 OCU initialize/tools-list/start/repeat start/status/stop/repeat stop/final status transcript，确认两次 start 指向同一个 active session，repeat stop 与 final status 仍指向同一个 completed session，start/status active response 带脱敏后的 `currentSegmentEventsPath` / `currentSegmentMetadataPath`，stop final response 不残留 current segment 字段，fixture 的 MCP text JSON 已转换成脱敏 `textJSON` 且不含本机绝对路径，并用源码 validator 与 runtime validator 校验最终 session 文件；该检查接入默认 CI，不依赖官方 bundle。 +- Record & Replay matrix smoke：`make event-stream-smoke-matrix` 顺序运行默认 lifecycle、no-active、maximum-duration timeout、wait timeout、approval denied/cancelled、MCP elicitation approval、app-agent wait/notify、fixture import 和 recording compare；截图上下文和官方 surface 对比分别用 `OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_SCREENSHOTS=1`、`OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_OFFICIAL=1` 显式加入，避免默认矩阵依赖当前桌面可截图状态或本机官方 Codex bundle。 +- Record & Replay wait notify 单测：`swift test` 覆盖 `wait --notify-command` 在 stop 后执行 callback、向 stdin/env 传最终 status、wait timeout 时跳过 callback、按 session id 返回非 latest 的历史 completed session，以及未知 session id 不阻塞且不触发 callback。 +- Record & Replay input monitor 单测：`swift test --filter testEventStreamServiceRecordsInputMonitorMouseAndKeyboardEvents` 通过测试安装器捕获 recorder 注册的 mouse/key handlers，并喂入 `NSEvent`，验证 `mouse.click` / `keyboard.text_input` 会经真实 handler 路径写入 `events.jsonl`；这条测试避免依赖 macOS 是否把程序合成输入回放给全局 monitor。 +- Record & Replay fixture import smoke：`make event-stream-fixture-smoke`，用临时 recording 覆盖 `scripts/import-event-stream-fixture.py` 的脱敏导入路径，确认输出 fixture 不含原始 session id、app attribution、window title、AX value / selectedText、AX `fullTree` 文本、MCP text JSON 原文或 screenshot filename，并且相对路径 metadata 能继续被 validator / summary 读取；AX `treeLines` / `cumulativeTreeLines` 会保留 `+` / `-` / `~` marker 以支持后续 diff 结构对比；官方 hosted 最小 `events.jsonl + session.json` 输入会被归一化为 fixture 侧 `metadata.json`、官方风格 `session.json` handoff 和空 `suppressed.jsonl`，并把 `session.json.id` 一并脱敏，保证后续 readiness 的 metadata count 与 handoff path gate 可直接消费；带 `--mcp-transcript` 时会写入脱敏 `mcp-transcript.json`；`--scenario ` 会把 official / OCU fixture 标记成可配对的机器场景，并把共享 `scenarioRecipe` 写入 `fixture-manifest.json`，用于后续集合级 official baseline gate 和采集合同追溯。 +- Record & Replay hosted official fixture ingest smoke：`make event-stream-official-fixture-ingest-smoke`，用合成 hosted `event_stream_stop` / JSON-RPC response 覆盖 `scripts/ingest-official-record-and-replay-fixture.py`；该脚本会从嵌套 text JSON 中递归提取 `metadataPath` / `sessionPath` / `eventsPath` / `sessionDirectoryPath`，也支持 `--status-json -` 从 stdin 读取 hosted JSON，并可用 `--status-json-base-dir` 固定 stdin 或转存 JSON 中相对 handoff path 的解析基准。若 hosted JSON 只提供 `sessionDirectoryPath`，导入器会把该目录作为 recording input，再交给 fixture import 读取其中的 `metadata.json` 或官方最小 `session.json`。`--inspect-only` 会只解析 recording input、handoff paths 和 MCP transcript 证据，不创建 fixture，适合刚复制出 hosted JSON 后先做导入前诊断；正式导入随后调用 fixture import 脱敏入库，再按共享 scenario catalog 跑 golden readiness。传入 `--mcp-transcript` 与 `--require-mcp-transcript-evidence` 时，会要求 response-shape evidence；`--mcp-transcript -` 可在 status JSON 来自文件时从 stdin 读取单独 transcript，两个输入都要求 stdin 时会明确失败；传入 `--use-status-json-as-transcript` 时，如果同一 status JSON 含 transcript / response-shape evidence，会临时复用它作为 transcript；传入 `--check-fixture-set` 时，会继续跑集合级 official fixture set gate；传入 `--check-coverage` 时会把导入后的 official scenario coverage + required readiness report 写入返回 JSON，传入 `--require-coverage` 时 required scenario 覆盖或 readiness 不足都会让命令失败。该入口用于正常 Codex hosted R&R 录制完成后把返回路径一键落成 official fixture，direct ingest smoke 也覆盖 `keyboard-input-stop` / `drag-stop` 的 catalog readiness，避免推荐场景只在集合级 gate 中被校验。 +- Record & Replay official fixture coverage smoke：`make event-stream-official-fixture-coverage-smoke`，覆盖 `scripts/check-event-stream-official-fixture-coverage.py` 的仓库覆盖率扫描。脚本默认 required / recommended scenario 清单来自共享 scenario catalog，默认要求 official recording fixtures 至少包含 `simple-action-stop`，并会在 coverage 层校验 `fixture-manifest.json` 里的 `scenarioRecipe` 与 catalog 一致，避免只带正确 scenario 名但采集合同漂移的 fixture 被算作有效覆盖；可用 `--check-readiness` / `--require-readiness` 在 required scenario 存在时继续跑 official fixture set readiness。同时会单独报告建议校准场景 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 的覆盖情况，recommended coverage 只作为后续 official golden 采集清单，不影响退出码。当前 required `simple-action-stop` official fixture 已入库并可通过 readiness；recommended 覆盖仍会报告 `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 等缺口。`make record-and-replay-official-golden-fixture-gate` 是同一 coverage/readiness 逻辑的快速 gate，导入或更新 official successful recording 后可先用它确认 fixture 层已满足 required golden，再跑完整 `record-and-replay-official-golden-gate`。 +- Record & Replay official golden capture preflight：`scripts/prepare-record-and-replay-official-golden-capture.py` 是采集 official successful recording 前的只读入口。它不会启动录制或写 fixture，只会检查本机 bundled official `record-and-replay` plugin cache、调用 official fixture coverage report，并输出下一条 `inspect-only`、正式导入、coverage/readiness、strict gate 和可用时的 OCU candidate 命令；显式传入 `--capture-packet-dir ` 时，会额外生成 capture packet 目录，包含 scenario recipe、placeholder hosted JSON、`verify-inputs.sh`、inspect/import/check shell wrappers 和 preflight JSON，方便录完官方样本后直接替换输入、先确认输入存在/合法/非占位，再运行脱敏导入；`scripts/finalize-record-and-replay-official-capture-packet.py` 可把 Codex-hosted `event_stream_start`、active `event_stream_status`、`event_stream_stop` 和 final `event_stream_status` JSON 规整成 packet inputs 与 `mcp-transcript.json`，并立即跑 `verify-workflow.sh` / `verify-inputs.sh`，因此 official successful recording gate 不需要额外 `repeatStartResponseShape`；再加 `--capture-packet-recommended-scenarios` 时，会为 required + recommended 场景各生成一个子目录、根级 `capture-packets.json` 以及根级 `verify-all.sh` / `inspect-all.sh` / `import-all.sh` / `check-all.sh` / `ingest-ocu-candidates.sh` 批量 wrapper。Makefile 也提供 `make record-and-replay-official-golden-capture-packet`、`make record-and-replay-official-golden-capture-packet-set` 和不启动官方录制的 `make record-and-replay-official-capture-finalizer-smoke`；前者默认 `RNR_SCENARIO=simple-action-stop`，packet targets 都可用 `RNR_PACKET_DIR=` 覆盖输出位置，也可用 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1` 在缺官方 plugin cache 时只生成命令，且同样不启动官方录制。当 coverage report 发现 recipe drift、重复 scenario 或 manifest 错误时，preflight 顶层会输出 `coverageErrors` 并把修复动作放进 `nextActions`，避免带坏 manifest 的样本继续进入配对。`make record-and-replay-official-golden-capture-preflight` 运行真实 preflight，`make record-and-replay-official-golden-capture-preflight-smoke` 覆盖这份 JSON 输出合同。 +- Record & Replay OCU candidate pairing preflight：`scripts/prepare-record-and-replay-ocu-candidate-pairing.py` 是 official fixture 入库后的只读入口。它不会启动 action smoke 或写 candidate fixture，会先调用 official fixture coverage report，再调用 official fixture set gate 检查 selected scenario 的 official readiness、可选 OCU candidate readiness/compare 状态，并输出下一条 candidate ingest 与 fixture-set compare 命令；当 official fixture manifest 有 recipe drift、重复 scenario 或其它 coverage error 时，preflight 会输出 `officialCoverageErrors` 并把修复动作放在 `nextActions` 首位。`make record-and-replay-ocu-candidate-pairing-preflight` 运行真实 preflight，`make record-and-replay-ocu-candidate-pairing-preflight-smoke` 覆盖输出合同。 +- Record & Replay OCU candidate ingest smoke：`make event-stream-ocu-candidate-ingest-smoke`，用合成 OCU recording 覆盖 `scripts/ingest-ocu-record-and-replay-candidate.py`；该脚本可直接导入 `--recording`，也可从 action smoke JSONL 的 `recordingsRoot` / `sessionId` 找到真实 OCU session 并导入为 `source=ocu` fixture，且会自动使用同一 JSON 里的 `mcpTranscriptPath`；导入后跑 strict OCU validation、共享 scenario catalog readiness，并可用 `--official-root --check-fixture-set` 立即进入 same-scenario official-vs-candidate gate。导入输出还会写入 `commands.pairingPreflightShell` 和 `commands.fixtureSetGateShell`，让后续 official fixture 入库后可以直接复跑 candidate pairing preflight 或集合级 gate。当前 candidate ingest 已对 `simple-action-stop`、`keyboard-input-stop`、`drag-stop` 三类 stopped action scenario 做场景级 validation/readiness，分别要求 `mouse.click`、`keyboard.text_input`、`mouse.drag`、完整 AX payload 和 completed MCP response-shape evidence；`--run-action-smoke` 是 opt-in 真实输入采样入口，会按 `--scenario simple-action-stop|drag-stop` 映射到同名 action smoke 场景，其它非 keyboard 场景回退到默认 `mixed-action-stop`，并用短 `/tmp/ocu-rnr-*` 目录避免 app-agent Unix socket 路径过长。`keyboard-input-stop` 因合成键盘事件在不同 macOS 环境下可能被 session event tap 过滤，当前要求通过 `--recording` 或保留的 `--smoke-json` 导入，不进入 `--run-action-smoke`。这些真实输入采样不进入默认 CI。 +- Record & Replay golden readiness smoke：`make event-stream-golden-readiness-smoke`，用临时 fixture 覆盖 `scripts/check-event-stream-golden-readiness.py`，确认可作为动作类 golden 的样本必须具备 fixture manifest/source/version、final state、metadata/session handoff 兼容性、metadata event/suppressed counts 与文件行数一致、metadata 声明的 handoff path 存在、`session.started` 恰好一次且为首事件、`session.ended` 恰好一次且为最终事件、动作事件和 AX payload；校准 stop / cancel / timeout 样本时可用 `--require-end-reason` 固定 expected `endReason`，需要证明 handoff 文件、count 或 path 字段完整时可用 `--require-session-alias` / `--require-metadata-counts` / `--require-handoff-paths`；需要官方 MCP 对外行为证据时可要求 `mcp-transcript.json` 和 `event_stream_start/status/stop` response；required MCP tool timeout 会明确失败，生命周期样本只有显式放宽 action / AX 要求后才通过。 +- Record & Replay official fixture set smoke：`make event-stream-official-fixture-set-smoke`，用临时 official / OCU fixtures 覆盖 `scripts/check-event-stream-official-fixture-set.py` 的集合级 gate；该 gate 要求 official fixture manifest 带 `scenario` 和与共享 catalog 一致的 `scenarioRecipe`，默认至少要求 `simple-action-stop` successful recording，并完全通过共享 scenario catalog 生成 strict readiness 参数，其中 `simple-action-stop` 必须包含 `mouse.click` 和 `recording_controls_stopped` endReason，`keyboard-input-stop` 必须包含 `keyboard.text_input` 和 `recording_controls_stopped` endReason，`drag-stop` 必须包含 `mouse.drag` 和 `recording_controls_stopped` endReason，`cancel` 场景必须包含 `recording_controls_cancelled` endReason，`timeout` 场景允许只有 `session.started` / `session.ended` 的生命周期证据且暂不固定 endReason 直到官方样本确认；传入 `--candidate-root` 后会按同 scenario 找 OCU candidate，先用 `source=ocu` 跑同 scenario readiness，再用 recording compare 固定 event sequence、schema、metadata keys/values、handoff paths、final session evidence、semantic fields、MCP response shape、AX diff evidence / marker 和 suppressed event sequence / schema。当前仓库还没有 official successful recording fixture，因此这条 smoke 只验证 gate 本身，不代表官方 successful baseline 已入库。 +- Record & Replay fixture compare smoke：`make event-stream-compare-smoke`，用临时 recordings 覆盖 `scripts/compare-event-stream-recordings.py` 的成功路径、metadata key 缺失失败路径、metadata stable value drift 失败路径、handoff path 缺失 / 不存在失败路径、start/end lifecycle evidence 失败路径、replay-critical semantic field 缺失失败路径、schema 漂移失败路径、event sequence 漂移失败路径和 AX diff/cumulative evidence gate；后续官方 golden fixtures 入库后用同一脚本对比 OCU candidate，并可用 `--require-same-metadata-keys` 守住 metadata/session 布局键，用 `--require-same-metadata-values` 守住 state / active / endReason / count 这类稳定状态值，用 `--require-handoff-paths` 守住 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 都存在且可解析，用 `--require-final-session-evidence` 守住 `session.started` 起点与最终 `session.ended` 闭环，用 `--require-semantic-fields` 守住 target AX、keyboard focused AX、drag 起止点、window attribution、AX `fullTree` 和 AX `treeLines` 等 replay 关键证据。 +- Record & Replay skill scaffold smoke：`make event-stream-skill-scaffold-smoke`，用临时 recording 同时覆盖源码 `scripts/scaffold-event-stream-skill.py` 和 runtime CLI `open-computer-use event-stream scaffold-skill --json` 的录制摘要到 `SKILL.md` 草稿生成路径；两条路径都会先跑 recording validator 的 skill draft gate，再确认 summary / 生成产物包含 `skillReadiness`、结构化 `runtimeInputs`、结构化 `safetySignals`、结构化 `summaryLimits`、完成态 `recordingIncomplete` gate、`session.started` 起点 / 去重 gate、`session.ended` 收尾 / 去重 gate、阻断性 `blockingDiagnostics` gate、先检查 connector/API/专用工具再使用 Computer Use 的 agent replay procedure、replay steps、Safety confirmation signals、Summary Limits 截断提示、verification checklist、`skill-creator` finalization 指引、脱敏后的 `references/recording-summary.json`，且不泄漏本机绝对路径；smoke 也覆盖 `events.jsonl` 输入、event count 不一致、外部 `screenshotPath` 引用、缺失 / 重复 `session.started`、`session.started` 非首事件、重复 `session.ended` 和 `session.ended` 后仍有事件时 scaffold 拒绝生成草稿。 +- Record & Replay standalone thin skill repo smoke：`make record-and-replay-skill-repo-smoke`，用临时目录覆盖 `scripts/scaffold-record-and-replay-skill-repo.py` 的独立 repo 骨架生成路径，确认生成物包含 thin skill、README、package script、`scripts/verify-package-artifact.py`、runtime contract manifest、`scripts/verify-runtime.py`、`scripts/verify-skill-workflow.py`、`scripts/wait-notify-contract-smoke.py`、`scripts/recording-to-skill-smoke.py`、`scripts/check.sh`、可选 `scripts/recording-lifecycle-smoke.py` 和 `.github/workflows/ci.yml`；smoke 会确认 manifest 的 `checks` 声明默认 package/package artifact/runtime/skill-workflow/wait-notify/recording-to-skill/self-check，`extensionLayer` 声明 wait/notify 扩展不进入官方 MCP surface，且 `callbackFailureMakesCliFail=true` / `callbackTimeoutMakesCliFail=true`，`optionalChecks` 只声明真实录制 lifecycle smoke。随后先验证生成态 package script 拒绝缺失 description 的 frontmatter，再打包并运行 `scripts/verify-package-artifact.py`，确认 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` 的 frontmatter 和 Record & Replay handoff 片段仍存在，并输出 `checkedPackageArtifact=true`；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费该 evidence。之后直接验证 runtime 和 skill workflow，运行生成 repo 的组合 `check.sh`，确认 `.skill` 可产出、不泄漏本机源码绝对路径，并用当前构建出的 `OpenComputerUse` binary 实际验证 `event-stream mcp` 的 protocol、server name、initialize capabilities、无 `instructions` initialize response、`event_stream_start/status/stop` 三件套 tool metadata（description、empty input schema、annotations）、官方 no-active status/stop response shape、unexpected / non-object arguments 拒绝、缺失或非 object params 拒绝、缺失或非 string tool name 拒绝，以及所有被拒 malformed request 都不创建 `latest-session.json` / `active-session.json`；生成态 manifest 也声明 `mcpServer.rejectedRequestsDoNotCreateSessionFiles=true`。生成态 `scripts/verify-skill-workflow.py` 不启动录制，只检查 thin skill 保留官方 handoff 约束、OCU validator/scaffold 前置门禁和 Independent Wait / Notify Integration（`waitTimedOut` / `waitSessionMatched`、notify stdin/env、listener 不放进 MCP surface）；生成态 `scripts/wait-notify-contract-smoke.py` 也不启动录制，只等待不存在的 session 并确认 callback skipped、`waitSessionMatched=false` 且不创建 session 文件，再用合成 completed session 确认失败 callback 非零退出会返回 `reason=nonZeroExit` 且 CLI 非零、callback 超时会返回 `reason=timeout` 且 CLI 非零；同一默认自检还会用临时合成 completed recording 验证安装态 `event-stream validate --strict-ocu --require-skill-draft`、events-only `validate --require-skill-draft` 和 `event-stream scaffold-skill --json` 草稿生成路径，并用无动作 recording 验证失败的 `scaffold-skill` JSON `ok=false` 会映射为非零 CLI exit；随后会运行生成 repo 的 optional lifecycle smoke，通过同一 `event-stream mcp` 进程 start/repeat start/status/stop/repeat stop/final status 一个最小 recording，验证 one-active、idempotent stop 和 completed status 语义，并用安装态 `event-stream validate --strict-ocu` 校验 `metadata.json`、`session.json` 和 `events.jsonl`；生成 workflow 也会被检查为使用 40 位 SHA pin 的 GitHub Actions 并调用 `./scripts/check.sh`。 +- Record & Replay official capture packet verifier：`scripts/prepare-record-and-replay-official-golden-capture.py` 生成的单场景 packet 和 recommended packet set 现在都包含 `verify-workflow.sh`。该 verifier 不依赖 hosted JSON，先校验 `postCaptureWorkflow`、input 文件、wrapper command、可执行 bit 和 transcript 开关一致；baseline summary / artifact audit 消费 `checkedOfficialCapturePacketWorkflowVerifier` 与 `checkedOfficialCapturePacketSetWorkflowVerifier`，standalone source summary 继承 `checkedSourceBaselineSummaryCapturePacketWorkflowVerifier` 与 `checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier`。 +- 生成态 `scripts/verify-runtime.py` 和可选 `scripts/recording-lifecycle-smoke.py` 在等待 `event-stream mcp` response 超时时，会把 runtime path、执行命令、`--version` 结果和 `OPEN_COMPUTER_USE_CLI` 修复提示写进错误；`OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS` 仅用于本地排障和测试加速。源码 standalone smoke 用 fake stale runtime 覆盖该负例，npm staged smoke 验证安装态 scaffold 生成物保留同样诊断字段。 +- baseline summary 和 artifact audit 会把 source / npm staged 的 `checkedRuntimeTimeoutDiagnostics` 作为 `usableBaseline` 必需 evidence 直接复审；也会消费 `checkedInitializeSurfaceContract`、`checkedToolMetadataContract` 和 `checkedToolInputSchemaNoArguments`，把 initialize surface、三件套完整 tool metadata 和 no-arg schema 从宽泛 runtime contract 中拆成可审计证据。缺失时最终摘要会报告对应的 `standaloneSkillRepo.*` / `npmStagedSkillRepo.*` 缺口。 +- Record & Replay npm launcher diagnostics evidence：npm staged standalone smoke 还会在调用安装态 scaffold 前验证 `PYTHON` 指向不存在的可执行文件和 Python 2 版本都会失败，并输出 command-specific `Python 3 is required to run` / `PYTHON=/path/to/python3` 诊断；baseline summary、runner 早期断言和 artifact audit 会把 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics` 作为 `usableBaseline` 必需 evidence，避免安装态 standalone repo 创建链路只覆盖成功路径。 +- Record & Replay generated README prerequisites evidence：standalone scaffold 生成的 README 会显式声明生成 repo 自检脚本要求 Python 3 可通过 `python3` 访问，并提示 npm launcher 用户在 shell 找不到 Python 3 时设置 `PYTHON=/path/to/python3`。源码 standalone smoke、npm staged smoke、baseline runner、summary 和 artifact audit 会把 `checkedGeneratedReadmePrerequisites` 作为 source / npm staged 必需 evidence，避免独立 repo handoff 只在 launcher 错误里暴露 Python 依赖。 +- Record & Replay standalone screenshot containment evidence：生成态 `scripts/recording-to-skill-smoke.py` 还会构造一个 `screenshotPath` 指向 session 目录外但文件存在的 recording，要求安装态 `event-stream validate --strict-ocu` 拒绝；生成态 `record-and-replay-skill-repo.json` 同步声明 `recordingToSkill.strictValidation.requiresScreenshotPathsInsideSession=true`，源码 standalone smoke、npm staged smoke 和 baseline smoke 都会要求 `checkedScreenshotPathContainment=true`。 +- Record & Replay npm standalone repo smoke：`make npm-record-and-replay-skill-repo-smoke`，用当前 `dist/` native artifacts staging `open-computer-use` npm 包，先通过 staged launcher 校验一份合成 recording，确认 strict validator 已输出 `declaredPaths` evidence，并用外部但存在的 `screenshotPath` 负例确认 strict validator 已拒绝 session 外截图引用，避免旧 native artifact 进入后续 scaffold 自检；随后通过 staged `bin/open-computer-use scaffold-record-and-replay-skill-repo --output-dir ` 生成 standalone repo，并把生成 repo 的 `scripts/check.sh` 指向同一个 staged launcher 运行；该路径会覆盖安装态 package、runtime contract（含 no-active status/stop shape）、skill workflow verifier、合成 recording-to-skill scaffold、`recordingToSkill` manifest contract 和 `skill-creator` handoff。该检查证明安装态命令与生成态默认自检可一起工作，但依赖本地已有 `dist/Open Computer Use.app`、Linux 和 Windows artifacts，因此作为 opt-in smoke，不进入默认 CI。 +- Record & Replay npm package artifact gate：`scripts/npm/build-packages.mjs` 打包 `open-computer-use` 时必须把 `dist/record-and-replay-baseline-summary.json` 复制进 npm package；安装态 scaffold 随后才能生成 `evidence/source-baseline-summary.json` 并通过 `scripts/verify-source-baseline-summary.py`。如果该 artifact 缺失，打包会以 `Missing Record & Replay baseline summary artifact` 失败并要求先运行 `make record-and-replay-baseline-audit`，避免 npm 制品静默漏带 source baseline summary evidence。 +- Record & Replay baseline smoke：`make record-and-replay-baseline-smoke` 转调 `scripts/run-record-and-replay-baseline-smoke.sh`，是本机 opt-in 候选验证入口，会顺序运行 baseline contract smoke、默认 `event-stream` matrix、截图上下文 smoke、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、official raw start-timeout fixture smoke、official fixture set gate smoke、official fixture coverage + readiness report、hosted official fixture ingest smoke、OCU candidate ingest smoke、official golden capture preflight smoke、OCU candidate pairing preflight smoke、standalone thin skill repo smoke 和 npm standalone repo smoke，并在成功时输出机器可读 JSON 摘要。最终摘要包含顶层 `status`，直接区分 `usableBaseline`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady`、`officialGoldenGatePassed`、默认模式下的 `officialGoldenRequirementSatisfied`、`requiresOfficialGoldenCapture`、缺失的 required / recommended official successful recording scenarios，以及已覆盖但未通过 readiness 的 `notReadyRequiredOfficialSuccessfulRecordingScenarios`；`standaloneRepoBaselineReady=true` 只表示 standalone scaffold baseline 与 npm staged scaffold baseline 已可交付，不表示全部官方 successful recording schema 已校准。如果 coverage report 发现 recipe drift、重复 scenario 或 manifest 错误，也会把原始错误带到 `status.officialFixtureCoverageErrors` 和 `evidence.officialFixtureSetGate.coverageErrors`，严格 gate 的 stderr 会输出 `coverageErrors=...`，方便判断是补采样本还是修 fixture manifest。`usableBaseline` 会从各 evidence 计算，必需 evidence 缺失时输出 `missingUsableBaselineEvidence` 并返回非零；`nextActions` 会按状态给出修 baseline、采 required/recommended official golden 和 scaffold standalone repo 的下一步命令，其中 official capture 分支优先指向 `make record-and-replay-official-golden-capture-packet` / `make record-and-replay-official-golden-capture-packet-set` 以及 packet 内 verify/inspect/import wrapper，recommended 分支也会显式列出根级 `ingest-ocu-candidates.sh`。脚本也支持 `--summary-json ` 把同一份最终 JSON 额外落盘，保持 stdout 输出和原退出码，便于 release / standalone audit 留存 evidence。脚本也支持 `--require-official-golden`，`make record-and-replay-official-golden-gate` 会复用同一套检查，但只有 required official successful recording fixture 覆盖和 required readiness 都通过时才会写入 `officialGoldenGatePassed=true` 且 `officialSuccessfulRecordingEquivalenceReady=true`。最终摘要包含 `evidence.baselineContract`，证明 shared `REQUIRED_BASELINE_CHECKS`、standalone / npm staged smoke required keys、summary evidence keys 和 standalone lifecycle summary rename mapping 已由同一轮 baseline 自检守住；也包含脱敏的 `evidence.eventStreamMatrix`、`evidence.screenshotContextSmoke` 和 `evidence.realInputActionSmoke`，证明生命周期矩阵、`OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 下的 `screenshotNeededForContext` 标记、可用时的 `screenshotPath` 校验、真实输入录制、同 session MCP transcript 捕获和 skill 草稿链路都跑过，并额外证明 `simple-action-stop` click candidate 与 `drag-stop` drag candidate 已通过 `checkedSimpleActionStopCandidate` / `checkedDragStopCandidate` 进入 usable baseline 证据；也包含 `evidence.officialSurfaceCompare` 与 `evidence.officialRawStartTimeout`，分别证明本地 OCU 与官方 bundled `record-and-replay/1.0.857` 的 non-recording surface 都匹配入库 fixture，以及官方宿主外 raw `event_stream_start/status/stop` 当前三次 `tools/call` 都超时、没有返回 `metadataPath` / `eventsPath` / `sessionPath` 这类 recording handoff paths；`evidence.officialFixtureSetGate` 会证明集合级 gate、`simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` scenario policy 和 candidate pairing smoke 已跑过，并通过 `scripts/check-event-stream-official-fixture-coverage.py --allow-missing --check-readiness` 暴露当前仓库真实 official recording fixture的 required 与 recommended scenario 覆盖以及 required readiness；required `simple-action-stop` ready 时会显示 required 覆盖/readiness 通过，`repoHasRecommendedOfficialScenarioCoverage=false` / `missingRepoRecommendedOfficialScenarios=[...]` 则表示后续 keyboard、drag、cancel、timeout 等校准样本仍未齐备；`evidence.fixtureIngestPipelines` 会证明 hosted official fixture inspect-only、hosted official fixture ingest、hosted `sessionDirectoryPath` handoff、post-ingest coverage readiness、OCU candidate ingest、candidate ingest 后续 pairing/gate handoff 命令、OCU smoke JSON import、official/candidate pairing、candidate redaction，以及 `simple-action-stop` / `keyboard-input-stop` / `drag-stop` 的 candidate readiness 都跑过；`evidence.preflightPipelines` 会证明 official capture packet、recommended capture packet set、缺 official fixture、缺/ready candidate、keyboard recording-required、coverage error 和 paired candidate 这些采样/配对前置合同都跑过；同时包含 standalone / npm staged skill repo evidence，其中 `checkedOfficialEvidenceScenarioManifest=true` 单独证明生成 repo的 required / recommended official successful recording 场景清单仍被守住，`checkedOfficialEvidenceAuditManifest=true` 证明生成 repo README/manifest 暴露 source repo 的 baseline audit 与 strict golden audit 入口，`checkedRequiresObjectParams`、`checkedRequiresStringToolName`、`checkedRequiresObjectArguments`、`checkedRejectsUnexpectedArguments`、`checkedRejectsNonObjectArguments` 和 `checkedRejectedRequestsDoNotCreateSessionFiles` 则证明安装态和源码生成态 malformed official-compatible MCP calls 不会启动或污染 recording session。默认入口用于发版或拆独立 repo 前确认“当前 baseline 能用”，严格入口用于 official golden 入库后的 release gate；两者都依赖桌面输入权限、官方 bundled cache 和当前 `dist/` artifacts，因此不进入默认 CI。 +- Record & Replay baseline preflight evidence：`evidence.preflightPipelines` 还会单独要求 capture packet Make targets、单场景 `verify-inputs.sh` 输入校验、单场景 `inspect-only.sh` / `import-fixture.sh` placeholder guard、MCP transcript placeholder guard、packet set 根级 `verify-all.sh` 输入校验、packet set 根级 `inspect-all.sh` / `import-all.sh` placeholder guard，以及根级批量 wrapper 在执行任何子 wrapper 前先全量检查所有场景输入的 preflight guard；这些检查确保 summary `nextActions` 指向的 packet 命令和批量 wrapper 不是只存在于文档里的手工步骤，也避免批量导入时前几个场景已写入、后续场景才因为占位 JSON 失败的半批次状态。单场景 packet 还会写 `capture-contract.json`，并由 `checkedOfficialCapturePacketInputSemanticGuard` 证明 `verify-inputs.sh` / `inspect-only.sh` / `import-fixture.sh` 会检查 hosted status JSON 含 handoff path evidence、transcript enabled 时含 MCP transcript evidence，避免合法 JSON 但不是 successful recording handoff 的输入进入导入链路。单场景 `capture-contract.json` 和返回 JSON 还会写 `postCaptureWorkflow`，由 `checkedOfficialCapturePacketPostCaptureWorkflow` 证明录制后替换 hosted JSON、verify、inspect、import、coverage/fixture-set、可选 OCU candidate ingest、strict gate 和 strict expected-failure audit 的顺序已经机器可读化。recommended packet set 的根级 `capture-packets.json` 还会内联 `captureContracts` / `captureContractPaths` / `postCaptureWorkflow`，由 `checkedOfficialCapturePacketSetContractManifest` 证明批量采集包保留每个 scenario 的 expected action event、expected `endReason`、handoff path evidence 和 transcript requirement，由 `checkedOfficialCapturePacketSetPostCaptureWorkflow` 证明根级 workflow 与各子 `capture-contract.json` 一致，避免独立 repo 或 skill 只能遍历子目录推断合同或录制后步骤。单场景 packet 还会把 `check-fixture-set.sh`、`strict-golden-gate.sh` 和可用时的 `ingest-ocu-candidate.sh` 暴露到 README 与 `capturePacket.*Shell` 字段，并由 `checkedOfficialCapturePacketHandoffScripts` 纳入 baseline evidence，避免采到官方样本后缺少 fixture-set / strict gate / same-scenario OCU candidate 的后续命令；`checkedOfficialCapturePacketStrictAuditHandoff` 进一步要求 `strict-golden-gate.sh` 运行 `make record-and-replay-official-golden-gate-audit`，确保 official fixture 入库后会刷新 strict summary artifact；`checkedOfficialCapturePacketOcuCandidateOutputDir` 进一步要求该 OCU candidate wrapper 显式写到当前 official fixture root 的 `ocu-candidates/` 子目录，确保自定义 fixture root 下的 official fixture 与 candidate 仍能同根配对；`checkedOfficialCapturePacketSetOcuCandidateHandoff` 要求 recommended packet set 根级生成 `ingest-ocu-candidates.sh`，用于批量运行 simple / drag 等可自动采样的 OCU candidate wrapper，并明确跳过 keyboard / cancel / timeout 这类不生成 synthetic wrapper 的场景。`preflightPipelines.checkedOfficialCapturePacketTranscriptManifest` 还会要求单场景 packet、根级 packet set 和 `capture-packets.json` 机器可读声明 `includeTranscript` / `requiresMcpTranscriptInput`，避免消费方只能靠 README 或文件存在性判断 transcript 输入合同；`checkedOfficialCapturePacketNoTranscript` 和 `checkedOfficialCapturePacketSetNoTranscript` 分别守住单场景与批量 `--no-include-transcript` packet 不会错误要求未生成的 transcript 输入。`preflightPipelines` 也消费 baseline audit Make target dry-run smoke，要求 baseline 默认/自定义 `--summary-json` 路径、strict 默认/自定义 `--summary-json` 路径、strict 与 baseline 默认 artifact 分离、baseline/strict 两个 summary 变量互不污染，以及 strict `--require-official-golden` target 都未漂移。 +- Record & Replay baseline summary audit：`scripts/run-record-and-replay-baseline-smoke.sh` 在生成最终 summary 后会立即调用 `scripts/check-record-and-replay-baseline-summary.py` 审计同一份摘要，audit 输出走 stderr，stdout 仍保留最终 summary JSON；`--summary-json ` 落盘的摘要也可以后续单独用 `scripts/check-record-and-replay-baseline-summary.py ` 审计，不重跑桌面 smoke。默认审计要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致、顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查、usable baseline、standalone repo baseline、official non-recording baseline、raw timeout boundary 都成立，并直接检查所有 usable baseline 子证据：baseline contract、event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence；也会检查 summary 派生状态是否一致，包括 `requiresOfficialGoldenCapture`、`officialSuccessfulRecordingEquivalenceReady`、`standaloneRepoBaselineReady` 和 required gap / coverage error 组合；还会检查 `nextActions` 里的 required/recommended official capture packet 命令、recommended 根级 `ingest-ocu-candidates.sh`、standalone source baseline audit、scaffold、generated check 和 lifecycle 命令，避免 release / standalone audit artifact 只保留正确状态却给出漂移的下一步 handoff。加 `--require-official-golden` 后才要求 golden gate、required scenario readiness、无 coverage errors 和 official equivalence ready；如果审计的是 strict gate 缺 official golden 时写出的失败摘要，使用 `--allow-strict-official-golden-missing`，它要求 summary 处于 strict mode、`usableBaseline=true`、baseline/standalone/官方 non-recording/raw timeout evidence 完整、无 fixture coverage errors，并且失败由 required official scenario gap 或 not-ready 解释。 +- Record & Replay baseline audit Make targets：`make record-and-replay-baseline-audit` 与 `make record-and-replay-official-golden-gate-audit` 分别包装默认 baseline 与 strict official golden gate 的 `--summary-json` 产物输出。默认 baseline artifact 写 `dist/record-and-replay-baseline-summary.json`，可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖；strict official golden gate artifact 默认写 `dist/record-and-replay-official-golden-gate-summary.json`，可用 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=` 覆盖，避免缺 official golden 时的预期失败摘要覆盖可用 baseline 证据。默认 CI 用 `scripts/test-record-and-replay-baseline-audit-make-targets.py` dry-run 检查这两个 Make target 的参数展开，不启动完整 baseline 或官方录制；完整 baseline smoke 同样运行该 dry-run smoke，并把结果作为 `usableBaseline` 必需 evidence。 +- Record & Replay baseline no-active evidence：`scripts/run-record-and-replay-baseline-smoke.sh` 会显式运行 `scripts/compare-event-stream-no-active.py`，并在最终 JSON 写入 `evidence.officialNoActiveResponse`，证明本地 OCU MCP no-active `event_stream_status` / `event_stream_stop` 匹配 Codex-hosted 官方 1.0.857 fixture 的 `isRecording=false` / `maxDurationSeconds=1800` 最小 text JSON，同时确认该路径不会创建 session 文件。这是 non-recording response 证据，不替代 official successful recording golden。 +- Record & Replay baseline raw timeout evidence：`scripts/run-record-and-replay-baseline-smoke.sh` 会显式运行 `scripts/test-event-stream-probe-fixtures.py`，并在最终 JSON 写入 `evidence.officialRawStartTimeout`，证明官方 bundled 1.0.857 在宿主外 raw start/status/stop probe 中三次 `tools/call` 都超时、不会返回 recording handoff paths，并且该 sanitized fixture 不含本机路径。这是 official hostless boundary 证据，不替代 successful recording golden。 +- Record & Replay standalone official evidence：`scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `record-and-replay-skill-repo.json` 包含 `officialEvidence`，声明 standalone repo contract 当前基于官方 `record-and-replay/1.0.857` 的 non-recording surface fixture、Codex-hosted no-active fixture、hostless raw start-timeout boundary fixture，以及从源仓 baseline summary 派生的 `hasSuccessfulRecordingGolden` 状态；同一 manifest 也写入 required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景，并通过 `officialEvidence.scenarioRecipes` 固化每个场景的 capture goal、预期 action event、预期 endReason、evidence 要求和 OCU candidate 来源。`officialEvidence.sourceRepoBaselineChecks` 声明 baseline contract smoke、hostless raw timeout、official fixture set gate same-scenario compare policy、official capture preflight 和 OCU candidate pairing preflight 由 OCU 源仓 baseline 验证，其中 `officialFixtureSetGate.sameScenarioComparePolicy` 固定 AX diff evidence / marker 与 suppressed event sequence / schema 强对比边界，`officialGoldenCapturePreflight.requiredEvidence` 固定 `checkedOfficialCapturePacketInputSemanticGuard`、`checkedOfficialCapturePacketSetContractManifest`、`checkedOfficialCapturePacketPostCaptureWorkflow`、`checkedOfficialCapturePacketSetPostCaptureWorkflow`、`checkedOfficialCapturePacketStrictAuditHandoff` 与 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff` 必须存在；`officialEvidence.sourceRepoBaselineAudit` 声明 source repo 的 `make record-and-replay-baseline-audit`、`make record-and-replay-official-golden-gate-audit`、`make record-and-replay-baseline-audit-targets-smoke`、默认 baseline artifact `dist/record-and-replay-baseline-summary.json`、复制进 standalone repo 的 `copiedBaselineSummaryEvidence=evidence/source-baseline-summary.json`、默认 strict artifact `dist/record-and-replay-official-golden-gate-summary.json`、覆盖变量 `RNR_BASELINE_SUMMARY_JSON` / `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`、strict 缺 official golden 时的 `strictOfficialGoldenExpectedFailureAudit` 命令，以及 `verifiesSummaryArtifactSeparation=true` / `verifiesSummaryEnvVarIsolation=true` handoff；`officialEvidence.standaloneRepoBoundary` 声明独立 repo 默认自检不启动官方录制、不复制 preflight 脚本、不复制 OCU runtime 源码。standalone / npm staged / baseline smoke 都会检查这份 manifest evidence，并输出 / 消费 `checkedOfficialFixtureSetComparePolicyManifest=true`、`checkedSourceBaselineSummaryEvidence=true`、`checkedSourceBaselineSummaryOfficialGoldenState=true`、`checkedSourceBaselineSummaryCapturePacketSetContractManifest=true`、`checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 与 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`。 +- Record & Replay standalone source summary evidence 还会保留 post-capture workflow 与 strict expected-failure handoff：`evidence/source-baseline-summary.json` 中的 `preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow=true` 和 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff=true` 会由生成态 `scripts/verify-source-baseline-summary.py` 校验，并输出 / 消费 `checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true` 与 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`，避免独立 repo 丢失采样包录制后的有序 handoff 或 `strict-expected-failure-audit.sh` 对应的 `--allow-strict-official-golden-missing` 审计链路。 +- Record & Replay standalone source summary evidence 也会保留 official hosted directory ingest handoff：`record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialFixtureIngest` 声明源仓 baseline 要求 `checkedOfficialSessionDirectoryPathHandoff`，`evidence/source-baseline-summary.json` 保留 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff=true`，生成态 `scripts/verify-source-baseline-summary.py` 输出 / 消费 `checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff=true`。source / npm staged scaffold smoke、baseline summary 和 artifact audit 都要求该字段，避免后续独立 repo 自检丢掉官方 completed response 仅提供 `sessionDirectoryPath` 时的导入证据。 - app 打包：`./scripts/build-open-computer-use-app.sh debug` - 权限 onboarding 端到端回归：`./scripts/run-permission-onboarding-e2e.sh`（需要当前 macOS 对被测 `open-computer-use` 已授予 Accessibility 与 Screen Recording；默认禁用 app-agent proxy 来测试当前 CLI 运行态，可用 `OPEN_COMPUTER_USE_E2E_CLI=/path/to/open-computer-use` 指定被测 CLI，或用 `OPEN_COMPUTER_USE_E2E_DISABLE_APP_AGENT_PROXY=0` 显式覆盖默认代理行为） - npm staging：`node ./scripts/npm/build-packages.mjs` - release tgz：`./scripts/release-package.sh` -- skill 打包：`npm run package:skill` +- skill 打包：`npm run package:skill`，产出 `open-computer-use` 和 `open-computer-use-record-and-replay` 两类 skill 制品，并写入 `dist/skills/package-manifest.json` +- Record & Replay MCP smoke：`open-computer-use event-stream mcp` 后通过 JSON-RPC 调用 `initialize`、`tools/list`、`event_stream_start`、重复 `event_stream_start`、`event_stream_status`、`event_stream_stop`、重复 `event_stream_stop`，检查 one-active / idempotent 行为、返回路径、`events.jsonl` / `metadata.json` / `session.json` / `suppressed.jsonl`，并用 `open-computer-use event-stream validate` 和 `scripts/validate-event-stream-recording.py` 复核文件一致性、用 `scripts/summarize-event-stream-recording.py` 和 `open-computer-use event-stream summarize` 复核事件流可读性；在 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 且 snapshot 有截图数据时检查 payload 的 `screenshotPath` 和 session `screenshots/` 产物；no-active 模式通过 MCP status/stop 验证官方 no-active response shape 和不写 session 文件；timeout 模式则等待短时长后用 `event_stream_status` 和 `event_stream_stop` 验证自动写入 `recording_time_limit_reached`；wait-timeout 模式通过 app-agent CLI start/wait/cancel 验证 OCU 扩展字段 `waitTimedOut=true`；official compare 模式只调用官方 initialize/tools-list，确认 `instructions` 缺失和三件套 tool schema 对齐；approval 模式验证 denied / cancelled 返回官方错误文案且不写 session 文件；app-agent wait 模式验证 OCU 扩展 CLI 在长驻 app-agent 中共享同一个 recorder，并且 `wait` 可以被并发 `stop` 唤醒。仓库脚本入口是 `./scripts/run-event-stream-smoke-tests.sh`。 - Windows runtime 单测：`(cd apps/OpenComputerUseWindows && go test ./...)` - Windows exe 构建：`./scripts/build-open-computer-use-windows.sh --arch arm64` - Linux runtime 单测：`(cd apps/OpenComputerUseLinux && go test ./...)` diff --git a/docs/CICD.md b/docs/CICD.md index a68bd591..18dbbdbe 100644 --- a/docs/CICD.md +++ b/docs/CICD.md @@ -26,6 +26,70 @@ 4. 技术栈和环境稳定后，再补具体的部署 job。 5. 即使交付方式变化，SBOM 和 provenance 这类供应链能力也建议保留。 +## 当前 CI 门禁 + +`scripts/ci.sh` 是本地和 GitHub Actions 共用的默认入口，当前会顺序执行： + +- `scripts/check-docs.sh` +- `scripts/check-repo-hygiene.sh` +- `scripts/check-action-pinning.sh` +- 所有 `scripts/**/*.sh` 的 `bash -n` +- 所有 `scripts/**/*.mjs` 的 `node --check` +- 所有 `scripts/**/*.py` 的 `python3 -m py_compile` +- `npm run package:skill` +- `scripts/compare-event-stream-surface.py` +- `scripts/compare-event-stream-no-active.py` +- `scripts/test-event-stream-probe-fixtures.py` +- `scripts/test-event-stream-recording-probe.py` +- `scripts/test-event-stream-local-probe.py` +- `scripts/test-event-stream-golden-readiness.py` +- `scripts/test-event-stream-official-fixture-coverage.py` +- `scripts/check-event-stream-official-fixture-coverage.py --allow-missing` +- `scripts/test-record-and-replay-baseline-summary.py` +- `scripts/test-record-and-replay-baseline-summary-audit.py` +- `scripts/test-record-and-replay-baseline-audit-make-targets.py` +- `scripts/test-record-and-replay-baseline-runner-summary-json.py` +- `scripts/test-record-and-replay-official-golden-capture-preflight.py` +- `scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py` +- `scripts/test-event-stream-official-fixture-set.py` +- `scripts/test-official-record-and-replay-fixture-ingest.py` +- `scripts/test-ocu-record-and-replay-candidate-ingest.py` +- `scripts/test-event-stream-skill-scaffold.py` +- `scripts/test-record-and-replay-skill-repo-scaffold.py` +- `swift test` +- `scripts/run-event-stream-smoke-matrix.sh` +- Windows / Linux Go runtime tests，前提是当前环境安装了 `go` + +`scripts/check-docs.sh` 不只检查通用文档骨架，也守住 Record & Replay baseline 的关键入口文档、官方 non-recording fixture 文件和 recommended packet set 的 `ingest-ocu-candidates.sh` handoff 文案，避免 R&R summary / artifact audit 合同只在脚本里更新、短入口和 CI 文档继续漂移。 + +Record & Replay surface drift check 默认只比对 OCU 自己的 `event-stream mcp` 与仓库内官方 1.0.857 non-recording fixture。探测本机官方 Codex bundle 的路径需要显式执行 `scripts/compare-event-stream-surface.py --use-default-official`，不进入默认 CI，避免 CI 依赖当前机器的 Codex plugin cache。 + +Record & Replay no-active drift check 默认只启动本地 OCU `event-stream mcp`，调用 `event_stream_status` / `event_stream_stop`，并和 Codex-hosted 官方 1.0.857 no-active fixture 比对 text JSON shape，同时断言不会创建 `latest-session.json`、`active-session.json` 或 session 目录。该检查不调用 `event_stream_start`，不启动真实录制。 + +Record & Replay raw probe fixture check 默认只校验仓库内已经脱敏入库的官方 raw start-timeout transcript shape，不会启动官方 bundled client，也不会调用官方 `event_stream_start`。真实官方 raw start/status/stop 探测需要显式执行 `make event-stream-official-start-probe`。 + +Record & Replay local probe smoke 会启动本地 OCU `event-stream mcp`，通过 `scripts/probe-event-stream-recording.py --target local --start-stop` 采集 initialize/tools-list/start/repeat start/status/stop/repeat stop/final status transcript，断言重复 start 返回同一个 active session、repeat stop 和 final status 返回同一个 completed session，生成的 fixture 里 MCP text JSON 已转换为脱敏 `textJSON`、不含本机绝对路径，并用源码 validator 与 runtime validator 校验最终 session 文件。该检查只使用本地 OCU，不依赖官方 bundle。 + +Record & Replay 默认 CI 只跑稳定本地矩阵：lifecycle、no-active、timeout、wait timeout、approval denied/cancelled、MCP elicitation approval、app-agent wait/notify、fixture import、golden readiness 和 recording compare；另外单独跑 skill scaffold smoke，覆盖录制摘要生成带 `skillReadiness` 的 `SKILL.md` 草稿和脱敏 summary 的 handoff，并跑 standalone thin skill repo scaffold smoke，确认导出的独立 repo 骨架可打包、只声明 `open-computer-use event-stream mcp` runtime contract，且生成态 `scripts/package-skill.sh` 会拒绝缺失 description 的 frontmatter，生成态 `scripts/verify-manifest.py` 会校验 `record-and-replay-skill-repo.json` 的 `manifestContract`、官方 evidence、source repo audit、OCU 扩展边界和 recording-to-skill handoff，生成态 `scripts/verify-source-baseline-summary.py` 会审计 `evidence/source-baseline-summary.json` 这份来自 `dist/record-and-replay-baseline-summary.json` 的脱敏投影，确认 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、official golden 状态自洽（当前 required gap 明确，或未来 official golden/equivalence ready 完成态成立）且 source / npm staged standalone evidence 完整，生成态 `scripts/verify-runtime.py` 可以用当前构建出的 `OpenComputerUse` binary 实际握手验证官方三件套 surface，包括 initialize capabilities、tool description、empty input schema 和 MCP annotations；同一 smoke 也会运行生成态 `scripts/check.sh`，固定独立 repo 发布/安装前的组合自检入口。生成态 manifest 会把 `scripts/package-skill.sh`、`scripts/verify-manifest.py`、`scripts/verify-source-baseline-summary.py`、`scripts/verify-runtime.py`、`scripts/wait-notify-contract-smoke.py`、`scripts/recording-to-skill-smoke.py` 和 `scripts/check.sh` 放在默认 `checks`，把会启动真实录制的 `scripts/recording-lifecycle-smoke.py` 放在 `optionalChecks`。默认自检会运行生成态 `scripts/wait-notify-contract-smoke.py`，用 missing session 验证 callback skipped / 不创建 session 文件，并用合成 completed session 验证 notify callback 能收到 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH`、非零退出会让 CLI 非零且返回 `reason=nonZeroExit`、callback 超时会让 CLI 非零且返回 `reason=timeout`；也会运行生成态 `scripts/recording-to-skill-smoke.py`，用临时合成 completed recording 验证安装态 validator 的 strict skill-draft gate、events-only gate 和 runtime `scaffold-skill` 草稿生成路径，不启动真实桌面录制；smoke 还会显式运行生成态 `scripts/recording-lifecycle-smoke.py`，通过当前构建出的 runtime 创建最小 recording，覆盖重复 start 返回同一 active session、stop 幂等和停止后的 status，再用安装态 validator 校验 session 文件；生成态 `.github/workflows/ci.yml` 仍只调用不启动真实录制的 `check.sh`，并会被校验使用 SHA-pinned actions 且会在安装发布版 runtime 后调用同一自检。截图上下文和官方 Codex bundle surface 对比仍然保持 opt-in，分别通过 `OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_SCREENSHOTS=1` 和 `OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_OFFICIAL=1` 打开，避免默认 CI 依赖当前桌面截图状态或本机官方 bundle。 + +Record & Replay baseline artifact check 默认用 stub evidence 跑 `scripts/run-record-and-replay-baseline-smoke.sh --summary-json ` 的 runner 级测试，确认落盘 JSON 与 stdout 的最终摘要一致；同一测试也会确认 runner 在生成摘要后调用 summary audit，且 audit 输出只走 stderr，不改变 stdout / 文件中的最终 summary 合同。测试还会打开 `--require-official-golden`，确认缺 required official successful recording fixture 时仍写出失败摘要并保留非零退出码。默认 CI 还会运行 `scripts/test-record-and-replay-baseline-summary-audit.py`，用合成摘要覆盖 `scripts/check-record-and-replay-baseline-summary.py` 的默认 / 严格 / 预期 strict missing 审计口径：默认要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致，顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查且不带未知或重复 check，并在失败时通过 `declaredChecks.missingRequired`、`declaredChecks.unknown` 和 `declaredChecks.duplicates` 报告具体名称；允许 usable standalone baseline 缺 official golden，但必须保持 `officialSuccessfulRecordingEquivalenceReady=false`；`--require-official-golden` 必须要求 official golden gate 和 equivalence ready；`--allow-strict-official-golden-missing` 只允许 strict summary 因 required official scenario missing / not-ready 失败，同时继续要求 baseline evidence 完整、无 coverage errors，并输出 `checkedStrictOfficialGoldenMissingFailure` 与 `checkedStrictMissingGoldenHasRequiredGap` evidence。审计会直接检查 event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence；source / npm staged standalone evidence 也必须包含 `checkedManifestContract=true`、`checkedSourceBaselineSummaryEvidence=true`、`checkedSourceBaselineSummaryOfficialGoldenState=true`、`checkedOfficialEvidenceAuditManifest=true`、`checkedOfficialFixtureSetComparePolicyManifest=true` 和 `checkedNotifySuppressedEventsPathEnv=true`，分别证明生成 repo 的 `scripts/verify-manifest.py` / `manifestContract` 会自校验 manifest、生成 repo README/manifest 保留 `make record-and-replay-baseline-audit`、`make record-and-replay-official-golden-gate-audit`、`make record-and-replay-baseline-audit-targets-smoke`、默认 baseline artifact、默认 strict artifact、`RNR_BASELINE_SUMMARY_JSON` / `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 变量名、`strictOfficialGoldenExpectedFailureAudit` 命令、`copiedBaselineSummaryEvidence=evidence/source-baseline-summary.json` 的 official golden 状态自洽检查、`officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy` 的 AX diff / suppressed stream 强对比策略，以及 `verifiesSummaryArtifactSeparation=true` / `verifiesSummaryEnvVarIsolation=true` 隔离声明，并证明 notify callback 能拿到 suppressed stream handoff path。同一 audit smoke 也会断言 builder 产出的顶层 `checks` 与 `scripts/record_and_replay_baseline_contract.py` 的 shared contract 顺序完全一致，并篡改顶层 `checks`、派生状态，确认缺失、未知、重复 check，或 `requiresOfficialGoldenCapture`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingEquivalenceReady`、golden complete 与 required gaps / coverage errors 组合不一致时会失败；还会篡改 `nextActions` commands，确认 required capture packet 的 Make/verify/inspect/import/`record-and-replay-official-golden-gate-audit` strict audit step、recommended capture packet set 的 Make/verify/inspect/import/`ingest-ocu-candidates.sh`、standalone source baseline audit、scaffold/check/lifecycle 命令缺失时会失败；其中 standalone source baseline audit 对应 `checkedStandaloneNextActionBaselineAuditCommand`。`scripts/test-record-and-replay-baseline-audit-make-targets.py` 还会用 `make -n` 守住 `record-and-replay-baseline-audit` / `record-and-replay-official-golden-gate-audit` 两个 Make 包装入口确实传递 `--summary-json`，且 strict 入口传递 `--require-official-golden`；默认 CI 运行该 smoke 时会额外设置外层 `RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-summary.json` 和 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-golden-summary.json`，确认脚本内部默认路径检查不会被调用方环境污染，同时 dry-run smoke 也会通过命令行覆盖反向变量，确认 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 不影响 baseline audit、`RNR_BASELINE_SUMMARY_JSON` 不影响 strict audit；`scripts/test-record-and-replay-official-golden-capture-preflight.py` 会守住 capture packet 的 `capture-contract.json`、`verify-inputs.sh` semantic guard、packet set 的 `verify-all.sh`、根级 `ingest-ocu-candidates.sh`、inspect/import placeholder guard、`strict-golden-gate.sh` 指向 audit target 和批量 preflight-all 行为，其中 `checkedCapturePacketInputSemanticGuard` / `checkedOfficialCapturePacketInputSemanticGuard` 证明 status JSON 必须带 handoff path evidence、transcript enabled 时必须带 MCP transcript evidence，`checkedOfficialCapturePacketStrictAuditHandoff` 证明采样包不会退回只跑不落盘的 strict gate；完整 baseline runner 也会消费这些 smoke 输出，在最终摘要的 `evidence.preflightPipelines` 写入 baseline audit target、baseline 默认/自定义 summary path、baseline 忽略 strict summary 变量、strict 默认/自定义 summary path、strict 忽略 baseline summary 变量、strict 独立 summary path、official capture packet、capture packet strict audit handoff、capture packet input semantic guard、recommended packet set OCU candidate handoff 和 OCU pairing evidence，缺失时 `usableBaseline=false`，且落盘 summary 的 artifact audit 会再次直接检查这些 preflight evidence、其它 usable baseline 子证据、派生状态、顶层 `checks` 和 `nextActions` command handoff。这样 release / standalone audit 依赖的 `--summary-json`、standalone/npm evidence、official capture packet wrapper、状态语义、检查声明、strict 预期失败摘要和下一步命令不是只靠完整 opt-in baseline 手工覆盖。 + +Recommended official capture packet set 还必须在根级 `capture-packets.json` 写入 `captureContracts` / `captureContractPaths`，并通过 `checkedOfficialCapturePacketSetContractManifest` 进入 `evidence.preflightPipelines`。这条 CI 证据证明批量采样包保留每个 scenario 的 expected action event、expected `endReason`、handoff path evidence 和 transcript requirement，后续独立 repo 或 skill 可以直接消费集合级合同，不需要靠 README 或遍历子目录推断。 + +Official capture packet / packet set 还必须暴露机器可读 `postCaptureWorkflow`。默认 CI 的 preflight smoke 会输出 `checkedCapturePacketPostCaptureWorkflow=true`、`checkedCapturePacketWorkflowVerifier=true`、`checkedCapturePacketSetPostCaptureWorkflow=true` 和 `checkedCapturePacketSetWorkflowVerifier=true`，baseline summary 投影为 `checkedOfficialCapturePacketPostCaptureWorkflow`、`checkedOfficialCapturePacketWorkflowVerifier`、`checkedOfficialCapturePacketSetPostCaptureWorkflow` 与 `checkedOfficialCapturePacketSetWorkflowVerifier`，artifact audit 通过对应 evidence 守住录制后替换 hosted JSON、verify、inspect、import、coverage / fixture-set gate、可选 OCU candidate ingest、strict audit 和 strict expected-failure audit 的有序 handoff，也守住 `verify-workflow.sh` 对 input / command / wrapper 文件一致性的校验。 + +Codex-hosted official capture 的 transcript gate 按官方 skill 流程要求 `event_stream_start`、active `event_stream_status`、`event_stream_stop` 和 final `event_stream_status` response shape；不要求额外 `repeatStartResponseShape`。`make record-and-replay-official-capture-finalizer-smoke` 运行 `scripts/test-record-and-replay-official-capture-finalizer.py`，用合成 hosted JSON 验证 `scripts/finalize-record-and-replay-official-capture-packet.py` 可以把 start/status/stop/final-status JSON 写入 packet inputs、生成 `mcp-transcript.json`，并通过 `verify-workflow.sh` / `verify-inputs.sh`；这条 smoke 不启动官方录制。 + +Official capture packet 还必须暴露 `strict-expected-failure-audit.sh`。默认 CI 的 preflight smoke 会输出 `checkedCapturePacketStrictExpectedFailureAuditHandoff=true`，baseline summary 投影为 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，artifact audit 通过 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence` 守住 `--allow-strict-official-golden-missing` 审计入口，避免 required official golden 尚缺时只能人工判断 strict 失败摘要是否符合预期。 + +Standalone source summary verifier 也会继承这条证据。生成 repo 的 `evidence/source-baseline-summary.json` 会保留 `preflightPipelines.checkedOfficialCapturePacketSetContractManifest`、`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow`、`preflightPipelines.checkedOfficialCapturePacketWorkflowVerifier`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow`、`preflightPipelines.checkedOfficialCapturePacketSetWorkflowVerifier`、`preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff` 和 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，`scripts/verify-source-baseline-summary.py` 会输出 `checkedSourceBaselineSummaryOfficialGoldenState=true`、`checkedSourceBaselineSummaryCapturePacketSetContractManifest=true`、`checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketWorkflowVerifier=true`、`checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier=true`、`checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 与 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`；source scaffold smoke、npm staged scaffold smoke、baseline summary 和 artifact audit 都消费这些字段，避免独立 repo 自检只证明 source summary 存在，却漏掉 official golden 状态自洽检查、recommended packet set 的集合级 contract manifest、录制后的 post-capture workflow、workflow verifier、strict audit summary artifact 刷新 handoff，或 strict 缺样本 expected-failure 审计 handoff。 + +Record & Replay npm package build 也守住同一个 source baseline summary artifact。`node ./scripts/npm/build-packages.mjs` 打包 `open-computer-use` 时必须复制 `dist/record-and-replay-baseline-summary.json`，因为安装态 `scaffold-record-and-replay-skill-repo` 会把它投影成生成 repo 内的 `evidence/source-baseline-summary.json`；如果该 artifact 缺失，build 会失败并提示 `Missing Record & Replay baseline summary artifact`，同时要求先运行 `make record-and-replay-baseline-audit` 再构建 npm 包。`scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 会临时移走该 artifact 验证负例，避免 npm 制品静默漏带 baseline evidence，等到独立 repo 自检才失败。 + +Skill 打包也在默认 CI 中运行，确保 `skills/*/SKILL.md` 的 frontmatter、zip 根目录、`.skill` alias 和 `dist/skills/package-manifest.json` 的多 skill manifest 结构持续可用。Record & Replay standalone generated repo 还会在默认 `scripts/check.sh` 里运行 `scripts/verify-package-artifact.py`，打开生成的 `.skill` archive，确认 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` frontmatter 和 Record & Replay handoff 片段仍存在，并把 `checkedPackageArtifact=true` 写入 source / npm staged scaffold smoke、baseline summary 和 artifact audit，避免独立 repo 只证明文件存在却没有证明可安装制品内容。 + +`.github/workflows/ci.yml` 只调用 `scripts/ci.sh`，避免 GitHub Actions 和本地验证入口分叉。`docs-check`、`repo-hygiene` 和 `supply-chain-security` workflow 是轻量补充门禁；所有 `uses:` 都必须继续 pin 到 40 位 commit SHA。 + ## 默认 release 产物 当前 release 流水线会产出： diff --git a/docs/QUALITY_SCORE.md b/docs/QUALITY_SCORE.md index f5b3a47f..793efdf1 100644 --- a/docs/QUALITY_SCORE.md +++ b/docs/QUALITY_SCORE.md @@ -11,10 +11,166 @@ | 区域 | 评分 | 原因 | 下一步 | | --- | --- | --- | --- | -| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导，以及一轮按官方 surface / result 行为收敛过的 9 个 tools。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI 和更清晰的用户错误提示。 | +| 产品面 | B | 已经有 Swift 本地 `computer-use` MCP server、默认 app 模式权限引导、一轮按官方 surface / result 行为收敛过的 9 个 tools，以及 Record & Replay 兼容的 baseline `event-stream mcp` / session 文件 / CGEvent tap 优先的基础输入事件记录 / 点击、右键、拖拽、文本输入、提交、快捷键、terminal value changed、selection changed、window changed、debug error、input monitor unavailable debug signal、secure input redaction / 目标 AX 上下文 / input 前 focused window 检测 / experimental raw events / AX previous+cumulative diff payload / 官方兼容 `fullTree` full AX payload / LCS 风格 AX compact diff baseline / 按需截图上下文 / start approval gate / MCP elicitation approval baseline / OCU 自有录制控制条 / wait listener / notify callback / 30 分钟自动结束；`open-computer-use install-codex-record-and-replay-mcp` 已提供 Codex 侧独立 R&R MCP 安装入口；`open-computer-use` skill 已补 Record & Replay reference，`open-computer-use-record-and-replay` thin skill 也已单独提供录制到 skill 生成流程，runtime CLI `open-computer-use event-stream scaffold-skill --json` 和源码脚本 `scripts/scaffold-event-stream-skill.py` 都可在前置 recording 结构校验通过后，把录制摘要生成带 workflow readiness、结构化 runtime inputs、结构化 safety signals、结构化 summary limits、recording completion gate、阻断性 blocking diagnostics gate、先检查 connector/API/专用工具再使用 Computer Use 的 agent replay procedure、replay steps、Safety confirmation signals、Summary Limits 截断提示、verification checklist 和 `skill-creator` finalization 指引的第一版 skill 草稿；`scripts/scaffold-record-and-replay-skill-repo.py` 还能导出只依赖 `open-computer-use` runtime 的 standalone thin skill repo 骨架，并在生成 repo 内提供 package frontmatter gate、`scripts/verify-runtime.py`、`scripts/recording-to-skill-smoke.py`、组合 `scripts/check.sh`、可选 `scripts/recording-lifecycle-smoke.py` 和 GitHub Actions CI workflow；生成态 manifest 已把默认 package/runtime/recording-to-skill/self-check 放入 `checks`，把真实录制 lifecycle smoke 放入 `optionalChecks`，用于验证实际 runtime contract（含 initialize capabilities、tool description、empty input schema、MCP annotations 和 no-active response shape）、用合成 completed recording 验证安装态 recording-to-skill handoff、产出 `.skill`，并在本地显式验证最小 recording lifecycle 的 one-active、idempotent stop 和 completed status 语义，作为后续真实独立 repo 的集成蓝本。 | 继续收敛复杂 AX 场景下的 state rendering 细节、权限 UI、Record & Replay 官方算法级 AX compact diff、official golden fixtures、官方 Record & Replay 真实 elicitation business metadata 和更清晰的用户错误提示；standalone repo 目前仍是可生成骨架，真实发布/安装链路和 skill scaffold 的业务语义仍需人工补齐。 | | Windows runtime | C | 已新增独立 Go `.exe`，通过 Windows UI Automation + Win32 window message 暴露同样 9 个 tools、MCP server 和 `call --calls`；默认不再自动启动 app、执行 `SetFocus`，或让 `type_text` 走可能抢前台的 UIA text fallback，并已接入 npm bundled artifact 分发，但仍是功能性第一版。 | 补交互式桌面 smoke、Windows fixture、installer/signing，以及更原生的 Go UIA 实现或更稳定的 bridge。 | | Linux runtime | C | 已新增独立 Go binary，通过 Python GI / AT-SPI2 暴露同样 9 个 tools、MCP server 和 `call --calls`；Ubuntu GNOME VM 已跑通 `list_apps`、MCP tools list 和 Text Editor 8-tool sequence，并已接入 npm bundled artifact 分发，但截图在 GNOME Wayland 下仍只能 best-effort，coordinate input 也不是通用后台模型。 | 补 Linux fixture、可重复 smoke runner、portal/compositor screenshot 路径，以及更原生的 Go D-Bus/libatspi bridge。 | | 架构文档 | B | 顶层结构、fixture bridge、app 模式和验证路径已经落文档。 | 后续补 release artifact、code signing / notarization 和 host 集成方式。 | -| 测试 | B | `swift test` + smoke suite 已覆盖 9 个 tools 的回归，并新增了针对“前台焦点是否被抢占”的手工对比样本沉淀。 | 增加更多普通 app 的录制回归，减少只依赖 fixture 和一次性手工检查。 | +| 测试 | B | `swift test` + smoke suite 已覆盖 9 个 tools 的回归，并新增了针对“前台焦点是否被抢占”的手工对比样本沉淀；Record & Replay baseline 已覆盖 tool surface、initialize 无 instructions 对齐、官方 1.0.857 non-recording surface fixture 对齐、no-active status/stop 官方 response shape、session 文件生命周期、cancel endReason、wait/stop 并发收敛、wait after completed、wait session id 匹配 / 历史 completed session / unknown session 不通知、CLI wait listener、wait timeout signal、wait notify callback 成功 / cancel / timeout 跳过 / 非零退出 / 超时失败、stale active session 恢复、30 分钟自动结束、控制条环境开关、控制条 Done/Discard 按钮回调、轻量 AX previous/cumulative diff、AX payload 服务级 full/diff/cumulative 写盘、AX screenshot context 服务级写盘、AX compact diff changed/added/removed/over-budget fallback、截图策略开关、input monitor handler 级 mouse/key 写盘、opt-in CGEvent tap action smoke、点击坐标转换、AX context key、window.changed payload、键盘/右键/拖拽事件分类、terminal value hash/脱敏 helper、debug.error 低敏 payload、Input Monitoring monitor 安装失败 debug signal、selection signature 与 selection clear payload、secure input 识别、raw event 开关和 scroll/raw redaction summary，并新增可复跑的 `run-event-stream-smoke-tests.sh` 覆盖默认、截图上下文、no-active status/stop、MCP 级 timeout、CLI wait timeout、approval 拒绝、MCP elicitation accept/decline/unsupported、官方 surface 对比、app-agent wait listener、app-agent wait notify callback 和 app-agent completed wait 路径；`make event-stream-action-smoke` 已验证 app-agent recorder 能从外部 HID click 和 scroll wheel 录到 `mouse.click` 与 scroll raw-only `experimentalRawEvents`，并能用该真实录制产物生成路径脱敏且 `skillReadiness.canCreateSkillDraft=true` 的 `SKILL.md` 草稿 / `recording-summary.json`；显式 `keyboard-input-stop` / `drag-stop` action smoke 场景分别验证 `keyboard.text_input` 和 `mouse.drag`。`make event-stream-surface-smoke` / `scripts/compare-event-stream-surface.py` 已把 OCU vs 官方 1.0.857 non-recording fixture 的 surface drift 检查独立出来，并支持 opt-in 复核本机官方 bundle；2026-06-27 复跑 `scripts/compare-event-stream-surface.py --use-default-official`，确认 OCU 和官方默认 cache 的 initialize / `tools/list` 仍同时匹配该 fixture；`scripts/probe-event-stream-recording.py` 已把 local/official raw MCP transcript、elicitation response 和短 start/status/stop probe 固化成脚手架，local start/status/stop probe 可用，官方宿主外 start/status/stop 当前超时；官方 raw start-timeout transcript shape 已脱敏入库并由 `scripts/test-event-stream-probe-fixtures.py` 默认校验；`make event-stream-smoke-matrix` 已把稳定本地 Record & Replay baseline 验证收敛成一条命令，截图和官方 surface 对比保持 opt-in；`make codex-record-and-replay-installer-smoke` 覆盖 Codex `record_and_replay` MCP 安装器的 TOML 写入和幂等行为；`scripts/ci.sh` 现在默认运行 shell / Node / Python 脚本语法检查、`npm run package:skill`、surface drift smoke、probe fixture smoke、skill scaffold smoke、standalone thin skill repo scaffold smoke、Codex R&R installer smoke、`swift test` 和稳定 event-stream matrix，让 baseline 不只停留在手工 smoke；runtime CLI `open-computer-use event-stream validate --json` 与 `scripts/validate-event-stream-recording.py` 已作为 OCU smoke 和后续官方 golden recording 的录制产物结构校验入口，并新增 `--require-skill-draft` gate 来拒绝无动作、未完成、已取消、`session.started` 缺失 / 重复 / 非首事件、`session.ended` 重复 / 非最终事件或存在阻断性诊断的 recording，也支持非 strict `events.jsonl` 输入；runtime CLI `open-computer-use event-stream summarize --json` 与 `scripts/summarize-event-stream-recording.py` 已作为录制后 skill 生成前置摘要入口并接入默认 smoke；runtime CLI `open-computer-use event-stream scaffold-skill --json` 已有 Swift parser/helper 单测覆盖，`make event-stream-skill-scaffold-smoke` 会同时跑源码 Python scaffold 和真实 runtime CLI scaffold，覆盖 scaffold 前置结构校验拒绝路径、events-only 输入路径，以及录制摘要生成带 workflow readiness、runtime inputs、safety signals、summary limits、recording completion gate、session.started first-event/dedupe gate、session.ended final-event/dedupe gate、blocking diagnostics gate、先检查 connector/API/专用工具再使用 Computer Use 的 agent replay procedure、confirmation signals、Summary Limits 截断提示、verification checklist 和 `skill-creator` finalization 指引的 `SKILL.md` 草稿和脱敏 summary 的 handoff；`make record-and-replay-skill-repo-smoke` 会验证 standalone repo scaffold 可生成、可打包 `.skill`、不泄漏本机源码路径，并验证生成态 package script 会拒绝缺失 description 的 frontmatter；随后运行生成态 `scripts/verify-runtime.py`、`scripts/recording-to-skill-smoke.py` 和 `scripts/check.sh` 实际检查 `open-computer-use event-stream mcp` 的 protocol、server name、initialize capabilities、无 `instructions` initialize response 和三件套 tool metadata（description、empty input schema、annotations）和 no-active response shape，也会用合成 completed recording 验证安装态 strict/events-only validator 与 runtime scaffold，并显式运行生成态 `scripts/recording-lifecycle-smoke.py`，通过当前构建出的 runtime 创建最小 recording，验证重复 start 返回同一 active session、stop 幂等、停止后的 status 指向同一个 completed session，并校验 session 文件；同一 smoke 还会校验生成态 CI workflow 使用 SHA-pinned actions 且调用不启动录制的 `check.sh` 自检入口；`scripts/import-event-stream-fixture.py` 与 `make event-stream-fixture-smoke` 已覆盖官方/OCU recording 脱敏导入 fixture 的路径改写、MCP transcript 脱敏、AX diff marker 保留和敏感内容过滤；`scripts/check-event-stream-golden-readiness.py` 与 `make event-stream-golden-readiness-smoke` 已覆盖 fixture 是否足够作为动作类 golden baseline 的 source/version、final state、action、AX payload、MCP start/status/stop response 和 required tool timeout gate；`scripts/compare-event-stream-recordings.py` 与 `make event-stream-compare-smoke` 已覆盖后续官方 fixture vs OCU candidate 的事件序列、metadata key drift、stable metadata value drift、final session evidence、replay-critical semantic field、schema drift、AX diff/cumulative evidence 和 marker 缺失 gate。 | 增加更多普通 app 的录制回归，通过正常 Codex 宿主流程沉淀官方 Record & Replay golden fixtures，减少只依赖 fixture 和一次性手工检查。 | | 可观测性 | C | 已有 `doctor`、`snapshot`、smoke 输出，以及一组仓库内留档的官方 `computer-use` / 本仓库实现对比样本。 | 补统一日志级别、失败上下文和 release artifact 里的诊断信息，把一次性样本收敛成可重复采集流程。 | -| 安全 | B | 已明确本地-only、权限边界和 fixture test bridge 的作用域，并将内置 denylist 收缩到密码管理器。 | 增加 session approval 和更清楚的敏感 app policy，避免策略长期硬编码在仓库里。 | +| 安全 | B | 已明确本地-only、权限边界和 fixture test bridge 的作用域，并将内置 denylist 收缩到密码管理器；Record & Replay baseline 已对 secure/protected focused text input 做明文脱敏，也避免把明显终端上下文的 focused value / raw characters 明文写入事件；start approval gate 已接入，denied / cancelled 不创建 session，交互式 runtime 有 OCU 自有开始确认弹窗，支持 MCP elicitation 的 host 可通过 `elicitation/create` 承载 approval，且 request shape 已按 Codex.app host form schema 使用显式 `mode: "form"`。 | 继续校准官方 Record & Replay 真实 elicitation metadata / response shape，并补更清楚的敏感 app policy，避免策略长期硬编码在仓库里。 | + +## Record & Replay 校准补充 + +- 2026-07-01：standalone scaffold 的 `officialEvidence.hasSuccessfulRecordingGolden` 不再硬编码为 false，而是从 `dist/record-and-replay-baseline-summary.json` 的 `status.officialSuccessfulRecordingGoldenComplete` 派生；生成态 `scripts/verify-manifest.py` 会把该字段和 copied `evidence/source-baseline-summary.json` 对齐检查，避免 required official fixture 已 ready 后独立 repo 继续传播旧状态。Record & Replay Python helper 也补齐 `from __future__ import annotations`，让 macOS system Python 3.9 可以直接运行 capture packet / ingest / readiness / scaffold 相关脚本。 +- 2026-07-01：真实输入 action smoke 的默认 `mixed-action-stop` hard gate 收敛为 click + AX context + MCP transcript + recording-to-skill handoff。外部 Swift 进程仍发送 best-effort scroll wheel，捕获到的 `experimentalRawEvents` / `reason=scrollWheel` 仍会被 summary/scaffold 渲染成 Scroll replay step，但不再作为 release baseline 必需项，避免 macOS 合成 scroll 被过滤时让 official golden gate 抖动；显式 simple / drag candidate smoke 仍分别要求 click / drag。 +- 2026-07-01：required `simple-action-stop` official successful recording fixture 已脱敏入库到 `docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/`，并通过 `scripts/check-event-stream-golden-readiness.py`、`scripts/check-event-stream-official-fixture-coverage.py --check-readiness --require-readiness` 和 `scripts/check-event-stream-official-fixture-set.py --require-scenario simple-action-stop`。readiness / compare 现在识别官方事件顶层 `ax.mode=fullTree` 作为 full AX payload evidence。capture packet 生成的 OCU candidate wrapper 改为只导入同场景 candidate 并输出后续 `fixtureSetGateShell` / `pairingPreflightShell`；same-scenario 强对比继续用于校准和差距分析，不再被误写成 required official golden strict gate 的硬前置。 +- 2026-07-01：official hosted fixture ingest 现在把 `sessionDirectoryPath` 作为一等 handoff evidence 解析。`scripts/ingest-official-record-and-replay-fixture.py` 若只从 hosted JSON 看到 session 目录，会直接把该目录交给 fixture import，后者可继续读取官方最小 `events.jsonl + session.json` 并归一化为 fixture metadata/session/suppressed 产物；`scripts/test-official-record-and-replay-fixture-ingest.py` 输出 `checkedOfficialSessionDirectoryPathHandoff=true` 固定这条路径，`scripts/run-record-and-replay-baseline-smoke.sh`、summary builder 和 artifact audit 也把 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff` 作为 `usableBaseline` 必需 evidence。standalone thin skill repo 的 manifest 现在也声明 `officialFixtureIngest` source baseline check，`evidence/source-baseline-summary.json` 会保留该 evidence，生成态 `scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff=true` 并有缺失负例；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费该字段，避免拆独立 repo 时丢掉 official completed directory handoff 证据。这不放宽 official capture packet 的 strict 语义合同，packet 仍要求 `eventsPath` 加 `metadataPath` 或 `sessionPath`，只是让手工/后续集成导入更兼容官方 completed response 里的目录字段。 +- 2026-06-28：official capture packet 新增 `capture-contract.json` 和 `checkedCapturePacketInputSemanticGuard` / `checkedOfficialCapturePacketInputSemanticGuard` evidence。`verify-inputs.sh` 现在除 JSON 合法、非 `_placeholder=true` 外，还要求 hosted status JSON 暴露 `eventsPath`，并且暴露 `metadataPath` 或 `sessionPath` 这类 handoff 文件 evidence；`suppressedEventsPath` / `sessionDirectoryPath` 只作为 optional evidence 记录，避免官方 hosted 最小包被误拒。transcript enabled 时还要求 `inputs/mcp-transcript.json` 看起来像 MCP transcript evidence。`inspect-only.sh` / `import-fixture.sh` 会先跑同一语义校验，baseline summary 与 artifact audit 都把这项作为 `usableBaseline` 必需 evidence，避免后续把合法但明显不是 official successful recording handoff 的 JSON 导入 fixture。 +- 2026-06-28：recommended official capture packet set 的根级 `capture-packets.json` 现在内联 `captureContracts` / `captureContractPaths`，把每个 scenario 的 expected action event、expected `endReason`、handoff path evidence 和 transcript requirement 留在集合级 manifest。preflight smoke 输出 `checkedCapturePacketSetContractManifest=true`，baseline runner、summary builder 和 artifact audit 通过 `checkedOfficialCapturePacketSetContractManifest` 消费该 evidence；缺失时 `usableBaseline=false`，防止后续独立 repo 或 skill 只能靠 README 或遍历子目录推断采样合同。 +- 2026-06-28：official capture packet / packet set 新增机器可读 `postCaptureWorkflow`。单场景 `capture-contract.json` 与返回 JSON 会列出替换 hosted status、按需替换 transcript、verify、inspect、import、coverage / fixture-set gate、可选 OCU candidate ingest、strict audit 和 strict expected-failure audit 的顺序；根级 `capture-packets.json` 会按 scenario 内联同一 workflow。preflight smoke 输出 `checkedCapturePacketPostCaptureWorkflow=true` / `checkedCapturePacketSetPostCaptureWorkflow=true`，summary builder 和 artifact audit 通过 `checkedOfficialCapturePacketPostCaptureWorkflow` / `checkedOfficialCapturePacketSetPostCaptureWorkflow` 与对应 evidence gate 消费该证据，避免录完官方样本后仍要从 README 或聊天上下文拼执行顺序。 +- 2026-06-28：baseline summary 的 missing official golden `nextActions` 现在把最后一步指向 `make record-and-replay-official-golden-gate-audit`，而不是只跑普通 strict gate。artifact audit 新增 `checkedMissingGoldenNextActionStrictAuditStep`，要求缺 required official fixture 时的下一步命令能直接刷新 `dist/record-and-replay-official-golden-gate-summary.json`，避免采到 official golden 后忘记留 release / standalone strict summary evidence。 +- 2026-06-28：official capture packet 的 strict handoff 也进入 baseline artifact evidence。`scripts/test-record-and-replay-official-golden-capture-preflight.py` 输出 `checkedCapturePacketStrictAuditHandoff=true`，summary builder 投影为 `preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff`，artifact audit 通过 `checkedOfficialCapturePacketStrictAuditHandoffEvidence` 守住 `strict-golden-gate.sh` 实际运行 `make record-and-replay-official-golden-gate-audit`，避免采样包只跑不落盘的 strict gate。 +- 2026-06-28：official capture packet 和 packet set 新增 `strict-expected-failure-audit.sh`。该 wrapper 审计 `dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing`，preflight smoke 输出 `checkedCapturePacketStrictExpectedFailureAuditHandoff=true`，summary builder / artifact audit 消费 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff` / `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence`，避免 required official golden 尚缺时 strict 失败摘要只能靠人工判断。 +- 2026-06-28：npm package build 现在把 `dist/record-and-replay-baseline-summary.json` 作为 Record & Replay standalone scaffold 的硬前置。`scripts/npm/build-packages.mjs` 会把该 artifact 复制进 `open-computer-use` npm 包，供安装态 scaffold 生成 `evidence/source-baseline-summary.json`；缺失时以 `Missing Record & Replay baseline summary artifact` 失败并提示先运行 `make record-and-replay-baseline-audit`。npm staged scaffold smoke 新增临时移走 artifact 的负例，避免发布出缺 source baseline summary evidence 的 npm 包。 +- 2026-06-28：standalone / npm staged thin skill repo 的 runtime contract evidence 继续拆细。生成态 `scripts/verify-runtime.py` 现在在 `checkedRuntimeContract` 之外输出 `checkedInitializeSurfaceContract`、`checkedToolMetadataContract` 和 `checkedToolInputSchemaNoArguments`；shared baseline contract、source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费这三项字段，直接证明 initialize capabilities / 无 instructions、三件套 tool description/schema/annotations 和 no-arg input schema 被实际验证。 +- 2026-06-28：完整 `make record-and-replay-baseline-audit` 现在不只消费默认 `mixed-action-stop` 真实输入 action smoke，还会显式运行 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop`。baseline summary / artifact audit 会把 `realInputActionSmoke.checkedSimpleActionStopCandidate` 和 `checkedDragStopCandidate` 作为 `usableBaseline` 必需 evidence，证明 OCU click / drag 同场景 candidate 采样、MCP transcript 捕获和 recording-to-skill handoff 已经进入 release / standalone audit 证据；runner summary-json smoke 也按场景拆分 action fixture，确认 runner 确实传递 scenario env，而不是靠聚合 JSONL 偶然通过。这仍是 OCU candidate baseline，不替代 required `simple-action-stop` official successful recording golden。 +- 2026-06-28：standalone thin skill repo 生成物新增 `scripts/verify-manifest.py`，并把 `checks.manifestContract=scripts/verify-manifest.py` 写入 `record-and-replay-skill-repo.json`。默认 `scripts/check.sh` 会先校验 manifest 里的官方 evidence、required / recommended scenario、`sourceRepoBaselineAudit`、`strictOfficialGoldenExpectedFailureAudit`、OCU `extensionLayer`、三件套 no-arg MCP surface 和 recording-to-skill handoff 没有漂移；source scaffold smoke、npm staged scaffold smoke 和 baseline shared contract 都新增 / 消费 `checkedManifestContract=true`，避免后续独立 repo 只跑 runtime smoke 却遗漏 manifest 自身的机器合同。 +- 2026-06-28：standalone thin skill repo 生成物新增 `scripts/verify-readme-handoff.py`，并把 `checks.readmeHandoffContract=scripts/verify-readme-handoff.py` 写入 `record-and-replay-skill-repo.json`。默认 `scripts/check.sh` 会校验 README 继续暴露官方 evidence、baseline / strict audit handoff、strict 缺 official golden 的 expected-failure 审计命令、required / recommended successful recording 场景、recording-to-skill 命令和 wait/notify 边界；source scaffold smoke、npm staged scaffold smoke、baseline shared contract 和 summary builder 都消费 `checkedReadmeHandoffContract=true`、`checkedReadmeOfficialEvidenceHandoff=true`、`checkedReadmeOfficialGoldenGap=true` 与 `checkedReadmeWaitNotifyBoundary=true`，避免独立 repo README 从机器合同里滑出去。 +- 2026-06-28：standalone thin skill repo manifest 的 `officialEvidence.sourceRepoBaselineAudit` 新增 `strictOfficialGoldenExpectedFailureAudit`，把 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing` 作为机器可读 handoff 暴露给未来独立 repo。生成态 README、source scaffold smoke、npm staged scaffold smoke、架构 / handoff / replication / CI 文档和 `scripts/check-docs.sh` 同步该字段，避免 strict gate 缺 official golden 时的可解释失败审计只停留在源仓上下文里。 +- 2026-06-28：复跑 `make record-and-replay-official-golden-gate-audit`，命令按预期非零退出并刷新 `dist/record-and-replay-official-golden-gate-summary.json`。strict artifact 显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`strictOfficialGoldenRequired=true`、`officialGoldenRequirementSatisfied=false`、`officialGoldenGatePassed=false`、`officialSuccessfulRecordingEquivalenceReady=false`，且失败只由 required `simple-action-stop` official successful recording 缺失解释，`officialFixtureCoverageErrors=[]`。`scripts/check-record-and-replay-baseline-summary.py` 新增 `--allow-strict-official-golden-missing`，已用该参数审计 strict artifact 通过，避免后续只能人工判断 `ok=false` 是预期 golden 缺口还是 baseline 回归。 +- 2026-06-28：复跑 `make record-and-replay-baseline-audit` 通过，并刷新 `dist/record-and-replay-baseline-summary.json`。新 artifact 的 `checks` 以 `baseline-contract-smoke` 开头，包含 `evidence.baselineContract.ok=true`，并由 `scripts/check-record-and-replay-baseline-summary.py` 审计通过；状态仍明确区分 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`officialNonRecordingBaselineVerified=true`、`officialRawStartTimeoutBoundaryVerified=true` 和 `officialSuccessfulRecordingEquivalenceReady=false`，同时继续报告缺少 required `simple-action-stop` 与 recommended `cancel` / `drag-stop` / `keyboard-input-stop` / `simple-action-stop` / `timeout` official successful recording fixtures。 +- 2026-06-28：strict official golden gate audit 默认 artifact 已从 baseline artifact 拆开。`make record-and-replay-baseline-audit` 仍默认写 `dist/record-and-replay-baseline-summary.json` 并支持 `RNR_BASELINE_SUMMARY_JSON=`；`make record-and-replay-official-golden-gate-audit` 默认写 `dist/record-and-replay-official-golden-gate-summary.json` 并支持 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=`，避免缺 required official fixture 时的预期失败摘要覆盖可用 baseline 证据。dry-run smoke、baseline summary、artifact audit、CI 外层环境污染回归和 standalone manifest 都已同步这两个路径。 +- 2026-06-28：audit target dry-run smoke 继续补强变量隔离：传 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 给 baseline audit 不会改变 baseline 默认 artifact，传 `RNR_BASELINE_SUMMARY_JSON` 给 strict audit 不会改变 strict 默认 artifact。baseline runner、summary builder 和 artifact audit 新增 `checkedBaselineAuditIgnoresStrictSummaryVar` 与 `checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar` evidence，防止后续 Make target 又把两个路径变量误接回一起。 +- 2026-06-28：standalone thin skill repo manifest 的 `officialEvidence.sourceRepoBaselineAudit` 继续补齐 audit target dry-run handoff：新增 `auditTargetDryRunSmoke=make record-and-replay-baseline-audit-targets-smoke`、`baselineSummaryEnvVar=RNR_BASELINE_SUMMARY_JSON`、`strictOfficialGoldenSummaryEnvVar=RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`、`verifiesSummaryArtifactSeparation=true` 和 `verifiesSummaryEnvVarIsolation=true`。source scaffold smoke、npm staged scaffold smoke、架构 / handoff / replication / CI 文档和 history 同步这份机器合同。 +- 2026-06-28：在 standalone audit isolation manifest 合同补齐后，复跑 `make record-and-replay-baseline-audit` 通过并刷新 `dist/record-and-replay-baseline-summary.json`，随后用 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-baseline-summary.json` 复审通过。artifact 显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`checkedBaselineAuditIgnoresStrictSummaryVar=true`、`checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar=true`，并继续保持 `officialSuccessfulRecordingGoldenComplete=false` / `officialSuccessfulRecordingEquivalenceReady=false` 和 required `simple-action-stop` official successful recording 缺口。 +- 2026-06-28：`scripts/run-record-and-replay-baseline-smoke.sh` 现在会先运行 baseline contract smoke，并把结果写入最终摘要的 `evidence.baselineContract`；`scripts/build-record-and-replay-baseline-summary.py` 将 shared checks、standalone/npm key 集和 lifecycle rename mapping 纳入 `usableBaseline` 必需 evidence，`scripts/check-record-and-replay-baseline-summary.py` 也会审计该 evidence。这样 release / standalone audit artifact 自己就能证明 shared baseline contract 在同一轮候选验证中跑过，而不只依赖默认 CI。 +- 2026-06-28：standalone thin skill repo manifest 的 `officialEvidence.sourceRepoBaselineChecks` 新增 `baselineContract=baseline-contract-smoke`，README 也说明源仓会通过 baseline contract smoke 守住 standalone contract；source scaffold smoke 和 npm staged scaffold smoke 的 exact manifest 断言同步覆盖该字段，避免后续拆独立 repo 时只看到 source audit 命令，却不知道 audit artifact 还要求 shared contract 自证。 +- 2026-06-27：新增 `scripts/test-record-and-replay-baseline-contract.py` / `make record-and-replay-baseline-contract-smoke` 并接入默认 CI，直接校验 `scripts/record_and_replay_baseline_contract.py` 中 baseline checks、standalone / npm staged smoke required keys 和 summary evidence keys 非空、无重复，并固定 standalone lifecycle evidence 在 smoke 与 summary 之间的三处重命名关系。这样 shared contract 自身也有回归门禁，避免 runner、summary builder、artifact audit 都消费同一文件时仍悄悄引入错误映射。 +- 2026-06-27：baseline runner 现在在 source / npm staged standalone 子 smoke 返回后先执行完整 early evidence gate，再进入 summary builder；该 gate 覆盖 generated README scenario list、`sourceRepoBaselineAudit` manifest、malformed MCP request 副作用、notify suppressed path 以及 callback failure/timeout evidence。`scripts/test-record-and-replay-baseline-runner-summary-json.py` 新增篡改 standalone evidence 的负例，确认缺关键字段时不会继续写最终 summary artifact。 +- 2026-06-27：standalone / npm staged skill repo evidence key 已收敛进 `scripts/record_and_replay_baseline_contract.py`，runner early gate、summary evidence key 集合和 artifact audit 共同消费这份 contract；summary smoke 会断言最终 `standaloneSkillRepo` / `npmStagedSkillRepo` evidence keys 与 shared contract 完全一致，避免 shell runner、summary builder 和 audit 三处列表继续漂移。 +- 2026-06-27：`event-stream wait --notify-command` 现在会把 `suppressedEventsPath` 作为 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` 传给成功 notify callback；standalone wait-notify contract smoke 用合成 completed session 验证该环境变量，source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费 `checkedNotifySuppressedEventsPathEnv=true`，避免独立监听方结束后只能读取主事件流而拿不到 suppressed diagnostics handoff。 +- 2026-06-27：baseline summary 的 standalone `nextActions` 现在会先给出 `make record-and-replay-baseline-audit`，再给出 standalone repo scaffold、生成 repo `scripts/check.sh` 和 opt-in lifecycle smoke。artifact audit 新增 `checkedStandaloneNextActionBaselineAuditCommand`，summary smoke 和 audit smoke 都覆盖该命令缺失负例，避免拆独立 repo 前忘记先落 source baseline summary artifact。 +- 2026-06-27：standalone thin skill repo 的 `officialEvidence` 新增 `sourceRepoBaselineAudit`，在生成态 README / manifest 中显式暴露 `make record-and-replay-baseline-audit`、`make record-and-replay-official-golden-gate-audit` 和 source repo audit artifact handoff。源码 scaffold smoke、npm staged scaffold smoke、baseline summary 和 artifact audit 都消费 `checkedOfficialEvidenceAuditManifest=true`，避免后续拆独立 repo 时只保留 preflight/scenario 清单，却丢掉 release / standalone audit 的源仓证据入口。 +- 2026-06-27：Record & Replay official capture preflight、OCU candidate pairing preflight、official hosted ingest、OCU candidate ingest、official fixture coverage report 和 official fixture set gate 的默认 required scenario / help 文案已统一从 `scripts/record_and_replay_scenarios.py` 派生；对应脚本测试覆盖无 `--scenario` 时使用 `DEFAULT_REQUIRED_SCENARIOS[0]`，避免后续 required 场景清单变化时 CLI 默认值、采集包和导入链路各自漂移。 +- 2026-06-27：standalone thin skill repo 的 README official scenario 清单现在从 `scripts/record_and_replay_scenarios.py` 生成，不再手写 required / recommended 场景列表；source scaffold smoke 和 npm staged scaffold smoke 都输出 `checkedGeneratedReadmeScenarioList=true`，baseline summary 与 artifact audit 也把该 evidence 纳入 `standaloneSkillRepo` / `npmStagedSkillRepo` 必需项，避免 manifest 场景合同正确但 README handoff 漂移。 +- 2026-06-27：raw probe fixture 脱敏补强；`scripts/probe-event-stream-recording.py --fixture-output` 会把 MCP text content 中可解析的 JSON 转成脱敏 `textJSON`，新增 `scripts/test-event-stream-local-probe.py` / `make event-stream-local-probe-smoke` 用真实本地 start/status/stop probe 固定 active current segment response、final cleanup response、路径脱敏和最终 session validator 路径，并接入默认 CI。 +- 2026-06-27：raw probe start/stop 采样扩展为 start/repeat start/status/stop；首次 start 成功后 probe 会立即再调用一次 `event_stream_start`，local probe smoke 断言两次 start 指向同一个 active session，用于后续 official golden transcript 采集 one-active 证据。 +- 2026-06-27：raw probe completed lifecycle 继续补齐 repeat stop 和 final status；首次 stop 成功后 probe 会再调用一次 `event_stream_stop` 和一次 `event_stream_status`，local probe smoke 断言 repeat stop / final status 仍指向同一个 completed session，用于后续 official golden transcript 采集 idempotent stop 与 completed status 证据。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 新增 strict OCU current segment path gate；`state=recording` 必须带存在的 `currentSegmentEventsPath` / `currentSegmentMetadataPath`，`state=stopped|cancelled` 不能残留这两个字段。该 gate 用来固定 OCU 自身 active/final session 文件一致性，官方 successful recording 的最终字段语义仍待 golden fixture 校准。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 新增 strict OCU declared handoff path gate；输出 `declaredPaths`，并在 `--strict-ocu` 下要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个字段都存在且解析后文件存在。该 gate 固定安装态 skill/plugin 能直接消费完整 OCU session，官方 successful recording 的路径字段布局仍待 golden fixture 校准。 +- 2026-06-27：新增 `scripts/compare-event-stream-no-active.py` / `make event-stream-no-active-smoke`，把 Codex-hosted 官方 no-active `event_stream_status` / `event_stream_stop` fixture 接入默认 CI；脚本只调用本地 OCU status/stop，不调用 `event_stream_start`，并断言不会创建 session 文件。 +- 2026-06-27：golden readiness gate 已把动作类样本的 start/end boundary 纳入硬要求，默认拒绝缺失、重复或非首事件的 `session.started`，也拒绝缺失、重复或非最终事件的 `session.ended`；fixture compare 的 `--require-final-session-evidence` 保留兼容 flag 名，但实际已比较 start/end lifecycle evidence，覆盖 `session.started` 计数 / 首事件位置和最终 `session.ended` 闭环。 +- 2026-06-27：golden readiness gate 新增 metadata/session alias evidence；默认拒绝同时存在但内容不同的 `metadata.json` / `session.json`，并提供 `--require-session-alias` 要求两份 alias 文件都存在，让官方 handoff fixture 同时覆盖 metadataPath 和 sessionPath。 +- 2026-06-27：golden readiness gate 新增 metadata count evidence；默认拒绝已声明但与文件行数不一致的 `eventCount` / `suppressedEventCount`，并提供 `--require-metadata-counts` 要求两项 count 字段都存在，同时校验 fixture manifest `suppressedEventCount`。 +- 2026-06-27：golden readiness gate 新增 declared handoff path evidence；输出 `declaredPaths`，默认拒绝已声明但不存在的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath`，并提供 `--require-handoff-paths` 要求四个字段都存在且可解析，避免官方样本入库后 skill handoff 路径不可消费。 +- 2026-06-27：fixture import smoke 现在串起导入、golden readiness response-shape gate 和 recording compare MCP response shape/schema gate；导入后的 sanitized `mcp-transcript.json` 会被要求通过 `repeatStartResponseShape` / `repeatStopResponseShape` / `finalStatusResponseShape` readiness 检查，并用 `--require-mcp-response-shapes --require-same-mcp-response-schema` 自比，避免官方样本脱敏后只停留在文件层验证。 +- 2026-06-27：recording compare 新增 suppressed stream gate；`--require-same-suppressed-event-sequence` 和 `--require-same-suppressed-schema` 会比较 `suppressed.jsonl` 里的降级/压缩/错误证据顺序与 schema，避免官方样本中的 AX / screenshot fallback 证据只被 `suppressedEventCount` 间接覆盖。 +- 2026-06-27：recording compare 新增 declared handoff path evidence；输出 baseline / candidate 的 handoff path 解析状态，并提供 `--require-handoff-paths` 要求双方都声明 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 且解析后文件存在，避免 OCU candidate 在官方对比中保留 metadata key 却丢掉可消费路径。 +- 2026-06-27：golden readiness gate 新增 suppressed evidence gate；fixture import smoke 同步覆盖 suppressed event 脱敏、事件 / suppressed event 通用 `path` / 未知 `*Path` redaction，并要求导入 fixture 通过 `--require-suppressed-events` / `--require-suppressed-event-type`。 +- 2026-06-27：golden readiness gate 新增 `--require-end-reason `；校准 stop / cancel / timeout 官方 fixture 时可以直接要求 `recording_controls_stopped`、`recording_controls_cancelled` 或后续观测到的官方 timeout reason，避免完成原因漂移只靠人工审查。 +- 2026-06-27：`make event-stream-action-smoke` 新增真实 CG scroll wheel 覆盖；在不臆造官方 `mouse.scroll` 事件名的前提下，录到的 `type=experimentalRawEvents` / `reason=scrollWheel` / raw `eventType=scrollWheel` 会继续进入 summary/scaffold。2026-07-01 后该 scroll raw-only 路径降为 best-effort 证据，不再作为默认 release hard gate。 +- 2026-06-27：Swift/Python summary 与 scaffold 会把 `reason=scrollWheel` 的 raw-only event 纳入 `actionSequence` 并渲染为 Scroll replay step；新增 Swift 单测、源码 scaffold smoke、runtime scaffold smoke 和 action smoke 断言，避免滚动示范录到了但生成 skill 草稿时丢步骤。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/verify-skill-workflow.py`，并接入默认 `scripts/check.sh` / manifest `checks`；它不启动录制，只静态守住官方 handoff 语义和 OCU validate/scaffold 前置门禁，避免独立 repo 后续漂移到轮询、扩展 MCP surface 或取消后再 stop。 +- 2026-06-27：standalone thin skill repo scaffold 的生成态 manifest 新增 `extensionLayer`，并让 `scripts/verify-skill-workflow.py` 同步守住 Independent Wait / Notify Integration；默认 smoke 现在会检查 `wait --notify-command`、`waitTimedOut` / `waitSessionMatched`、notify stdin/env 和 listener 不进入官方 MCP surface，避免独立 repo 丢失 OCU 自有结束回调边界。 +- 2026-06-27：standalone wait/notify contract smoke 现在把 notify callback 失败语义提升为机器证据：合成 completed session 下的失败 callback 非零退出必须让 CLI 非零并返回 `reason=nonZeroExit`，callback 超时必须让 CLI 非零并返回 `reason=timeout`。生成态 manifest 声明 `extensionLayer.waitNotify.callbackFailureMakesCliFail=true` 和 `callbackTimeoutMakesCliFail=true`，standalone / npm staged / baseline summary 都要求 `checkedNotifyCallbackFailureExit`、`checkedNotifyCallbackFailureReason`、`checkedNotifyCallbackTimeoutFailureExit` 和 `checkedNotifyCallbackTimeoutReason`。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/wait-notify-contract-smoke.py`，并接入默认 `scripts/check.sh` / manifest `checks`；它不启动录制，只用 missing session 验证安装态 CLI 返回 `waitTimedOut=true` / `waitSessionMatched=false`、跳过 notify callback 且不创建 session 文件，让独立 repo 默认自检覆盖 OCU 回调扩展的实际行为。 +- 2026-06-27：standalone thin skill repo 生成态 `scripts/verify-runtime.py` 和可选 `scripts/recording-lifecycle-smoke.py` 的 MCP response timeout 现在会输出 runtime path、`event-stream mcp` command、`open-computer-use --version` 结果和 `OPEN_COMPUTER_USE_CLI` 修复提示，并支持 `OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS` 缩短本地排障等待。源码 scaffold smoke 用 fake 旧 runtime 覆盖该失败路径，npm staged scaffold smoke 静态验证生成态诊断字段，避免全局旧版 runtime 只报裸 timeout。 +- 2026-06-27：baseline summary 和 artifact audit 已把 source / npm staged 的 `checkedRuntimeTimeoutDiagnostics` 纳入 `usableBaseline` 必需 evidence；缺失时会分别报告 `standaloneSkillRepo.checkedRuntimeTimeoutDiagnostics` 和 `npmStagedSkillRepo.checkedRuntimeTimeoutDiagnostics`，避免独立 repo release / audit 只证明 runtime 成功路径而漏掉旧 runtime 排障路径。 +- 2026-06-27：npm staged standalone smoke 现在输出 `checkedNpmPythonLauncherDiagnostics=true`，证明安装态 `open-computer-use scaffold-record-and-replay-skill-repo` 在 `PYTHON` 指向不存在文件或 Python 2 时会失败并提示 `Python 3 is required to run` / `PYTHON=/path/to/python3`；baseline runner、summary 和 artifact audit 已把该字段纳入 `usableBaseline` 必需 evidence，缺失时会报告 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics`。 +- 2026-06-27：standalone thin skill repo 生成态 README 现在显式声明自检脚本需要 Python 3 可通过 `python3` 访问，并提示 npm launcher 用户可用 `PYTHON=/path/to/python3` 修复 shell 找不到 Python 3 的环境；source scaffold smoke、npm staged scaffold smoke、baseline runner、summary 和 artifact audit 都消费 `checkedGeneratedReadmePrerequisites`，缺失时会报告 `standaloneSkillRepo.checkedGeneratedReadmePrerequisites` / `npmStagedSkillRepo.checkedGeneratedReadmePrerequisites`。 +- 2026-06-27：复跑 `make record-and-replay-baseline-audit` 通过并写出 `dist/record-and-replay-baseline-summary.json`；artifact 复审通过，摘要显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、source / npm staged 的 `checkedRuntimeTimeoutDiagnostics=true`，同时仍保持 `officialSuccessfulRecordingGoldenComplete=false`、`officialSuccessfulRecordingEquivalenceReady=false` 和 required `simple-action-stop` 缺口。 +- 2026-06-27：补入 npm Python launcher diagnostics gate 后再次复跑 `make record-and-replay-baseline-audit` 通过；新的 `dist/record-and-replay-baseline-summary.json` 显示 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics=true`、`usableBaseline=true`、`standaloneRepoBaselineReady=true`，并继续保持 `officialSuccessfulRecordingGoldenComplete=false` / `officialSuccessfulRecordingEquivalenceReady=false` 和 required `simple-action-stop` official golden 缺口。 +- 2026-06-27：补入 generated README prerequisites gate 后再次复跑 `make record-and-replay-baseline-audit` 通过，并用 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-baseline-summary.json` 单独复审通过；新的 artifact 显示 source / npm staged 的 `checkedGeneratedReadmePrerequisites=true`、`usableBaseline=true`、`standaloneRepoBaselineReady=true`，并继续保持 `officialSuccessfulRecordingGoldenComplete=false` / `officialSuccessfulRecordingEquivalenceReady=false` 和 required `simple-action-stop` official golden 缺口。 +- 2026-06-27：复跑 `make record-and-replay-official-golden-gate-audit` 按预期非零退出并写出 strict summary artifact；落盘摘要显示 `strictOfficialGoldenRequired=true`、`usableBaseline=true`、`standaloneRepoBaselineReady=true`、source / npm staged 的 `checkedRuntimeTimeoutDiagnostics=true`，失败只来自 `officialGoldenRequirementSatisfied=false` / `officialGoldenGatePassed=false` / `officialSuccessfulRecordingEquivalenceReady=false` 和 required `simple-action-stop` official successful recording fixture 缺失，`officialFixtureCoverageErrors=[]` 且没有 not-ready required 场景。这确认新增 runtime timeout diagnostics gate 没有引入额外 release gate 失败。 +- 2026-06-27：npm package staging 已带 `scaffold-record-and-replay-skill-repo.py` 和 thin skill 源，launcher 新增 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `；安装态可直接生成 standalone R&R skill repo 骨架，减少对源码 checkout 的依赖。 +- 2026-06-27：standalone thin skill repo 的默认 recording-to-skill smoke 现在检查 strict validator 输出的 `declaredPaths`，要求四个 handoff path 都可解析，避免独立 repo 自检只证明 recording 能生成草稿、却没有证明返回路径可消费。 +- 2026-06-27：standalone recording-to-skill smoke 新增无动作 recording 负例，验证失败的 `event-stream scaffold-skill --json` 会同时输出 `ok=false` 并返回非零退出码，避免后续独立 repo CI 只因命令打印了 JSON 就把不能生成 skill 的 recording 视为通过。 +- 2026-06-27：standalone recording-to-skill smoke 新增 cancelled recording 负例，验证 strict / events-only validation 和 `event-stream scaffold-skill --json` 都会拒绝 `recording_controls_cancelled` 录制；生成态 `recordingToSkill.rejectsCancelledRecordings=true` 让独立 repo contract 机器可读地固定该边界。 +- 2026-06-27：OCU 自有 Record & Replay 控制条定位改为可测试的 visible-frame clamp，并补常规屏幕、负坐标窄屏和高度不足屏幕的单测；这降低了多显示器 / 窄屏环境中 Done / Discard 控制初始出界的风险。 +- 2026-06-27：Record & Replay MCP 三件套的运行时参数校验已对齐空 object schema，并补强 `tools/call.params` / `tools/call.params.name` 校验；`event_stream_start/status/stop` 收到缺失 / 非 object params、缺失 / 非 string tool name、非 object 或非空 object `arguments` 会返回 tool error，并有单测确认不会因此创建 session 索引文件。standalone thin skill repo 的 manifest 与生成态 `verify-runtime.py` 也同步验证该 no-arg runtime contract。 +- 2026-06-27：standalone thin skill repo manifest 新增 `recordingToSkill` contract，机器可读声明 strict/events-only/scaffold-skill handoff 语义，避免独立 repo 只在 README 中描述 validation 边界。 +- 2026-06-27：新增 opt-in `make npm-record-and-replay-skill-repo-smoke`，用本地 `dist/` artifacts stage npm 包后从 staged launcher 生成 standalone repo，并让生成态默认 `scripts/check.sh` 使用同一个 staged launcher 验证 package、runtime contract、workflow verifier、合成 recording-to-skill scaffold 和 `skill-creator` handoff；该检查覆盖安装态命令与生成 repo 默认自检，不进入默认 CI。 +- 2026-06-27：npm standalone repo smoke 现在会先用 staged launcher 校验合成 recording，确认 strict validator 已输出 `declaredPaths`，再读取安装态生成的 `record-and-replay-skill-repo.json`，断言 `recordingToSkill` contract 中 strict validation、events-only validation、scaffold-skill 和 cancelled recording 拒绝边界没有从 npm scaffold 路径丢失。 +- 2026-06-27：当前 worktree 复跑默认 `make event-stream-smoke-matrix`、`make event-stream-action-smoke` 和 `scripts/compare-event-stream-surface.py --use-default-official` 均通过，证明本地 baseline 在生命周期矩阵、真实输入录制到 skill 草稿、以及官方 1.0.857 non-recording surface 对齐三条路径上仍可复跑。 +- 2026-06-27：新增 opt-in `make record-and-replay-baseline-smoke`，把默认 matrix、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、standalone thin skill repo smoke 和 npm standalone repo smoke 串成一条本机 baseline 候选验证入口，便于发版或拆独立 repo 前复核“当前 baseline 能用”；该组合入口已在本机复跑通过。 +- 2026-06-27：`make record-and-replay-baseline-smoke` 现在转调 `scripts/run-record-and-replay-baseline-smoke.sh`，baseline 候选验证顺序和成功 JSON 摘要有了独立脚本入口，后续独立 repo 或 release gate 可直接复用，不需要复制 Makefile 命令串。 +- 2026-06-27：`scripts/run-record-and-replay-baseline-smoke.sh` 现在包含 official fixture set gate smoke，baseline 候选验证会同时检查官方 successful recording 入库后的集合级校准门禁没有漂移。 +- 2026-06-27：新的脚本化 `make record-and-replay-baseline-smoke` 已复跑通过，覆盖默认 matrix、真实输入 action smoke、官方 1.0.857 surface compare、official fixture set gate smoke、standalone repo smoke 和 npm staged repo smoke，并输出 `baseline=record-and-replay` 的成功摘要；摘要会携带脱敏后的 `evidence.eventStreamMatrix`、`evidence.realInputActionSmoke` 和 `evidence.officialSurfaceCompare`，固定生命周期矩阵、真实 CGEvent 输入录制到 skill 草稿链路，以及本地 OCU / 官方 `record-and-replay/1.0.857` 都匹配 `Record & Replay` / `2025-11-25` / 三件套 tools。 +- 2026-06-27：baseline smoke 的成功摘要现在会携带 `evidence.standaloneSkillRepo` 和 `evidence.npmStagedSkillRepo`，并在脚本内强制解析子 smoke JSON，要求 standalone / npm 路径都证明 runtime contract、wait/notify、recording-to-skill handoff、declared handoff paths、scaffold failure exit、cancelled recording contract / rejection、lifecycle smoke、安装态 manifest contract、official evidence 场景清单、npm 生成 repo 默认 `check.sh` evidence 和 `skill-creator` handoff；standalone lifecycle evidence 也会进一步要求 one-active、idempotent stop 和 final status 三个具体语义。本机已复跑 `make record-and-replay-baseline-smoke` 通过。 +- 2026-06-27：baseline smoke 的最终摘要新增顶层 `status`，直接输出 `usableBaseline`、`officialNonRecordingBaselineVerified`、`standaloneRepoScaffoldBaselineVerified`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady`、`requiresOfficialGoldenCapture` 和缺失的 required / recommended official successful recording scenarios；`officialSuccessfulRecordingGoldenComplete` 现在要求 required scenario 覆盖和 required readiness 同时通过，`officialSuccessfulRecordingEquivalenceReady` 还要求 usable baseline 同时成立，避免只靠 evidence 细节或空壳 manifest 人工判断当前可用性与 golden 缺口。 +- 2026-06-27：baseline smoke 和 official fixture coverage report 进一步区分 `missingOfficialScenarios` 与 `notReadyOfficialScenarios`；最终摘要同步输出 `status.notReadyRequiredOfficialSuccessfulRecordingScenarios` 与 `evidence.officialFixtureSetGate.notReadyRepoOfficialScenarios`，严格 golden gate 失败时会把 `missing=...` 和 `notReady=...` 分开写到 stderr，方便后续判断是补采官方样本还是修复已有 fixture。 +- 2026-06-27：baseline summary 现在也保留 official fixture coverage report 的 `errors`，输出到 `status.officialFixtureCoverageErrors` 和 `evidence.officialFixtureSetGate.coverageErrors`；严格 golden gate 失败时会把 recipe drift、重复 scenario 或 manifest 错误作为 `coverageErrors=...` 打到 stderr，避免这类 fixture 问题被折叠成模糊的 readiness 失败。 +- 2026-06-27：baseline smoke 新增 `--require-official-golden` 严格模式，Makefile 暴露为 `make record-and-replay-official-golden-gate`；它复用完整 baseline 证据收集，但在 required official successful recording fixture 缺失或 required readiness 未通过时返回非零并输出 `strictOfficialGoldenRequired=true` / `officialGoldenRequirementSatisfied=false` / `officialGoldenGatePassed=false`，让后续 release 或拆 standalone repo 时可以用一条命令证明 official golden 已真正入库且可用。默认模式允许缺 official golden 时只会让 `officialGoldenRequirementSatisfied=true`，不会把 `officialGoldenGatePassed` 误标为 true。 +- 2026-06-27：新增 `make record-and-replay-official-golden-fixture-gate` 作为快速 fixture-only official golden gate，直接复用 `scripts/check-event-stream-official-fixture-coverage.py --require-readiness`；它不跑完整 baseline，适合导入 official successful recording 后立即确认 required scenario 覆盖和 readiness。 +- 2026-06-27：baseline 最终摘要生成逻辑已抽成 `scripts/build-record-and-replay-baseline-summary.py`，新增 `scripts/test-record-and-replay-baseline-summary.py` / `make record-and-replay-baseline-summary-smoke` 并接入默认 CI；快测用合成证据固定默认模式允许缺 official golden、严格模式缺 required official fixture 失败、严格模式覆盖 required official fixture 后通过，以及任一必需 baseline evidence 缺失时默认模式也会失败并输出 `status.missingUsableBaselineEvidence`，避免只靠完整 opt-in baseline 手工发现严格 gate 或 usable baseline 证据聚合回归。 +- 2026-06-27：baseline summary smoke 现在会实际执行默认缺 official golden 时 `nextActions` 输出的 packet Make 命令，用临时目录和 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1` 覆盖验证 required 单场景 packet 与 recommended packet set 都能生成对应 manifest、recipe 和 verify/inspect/import/ingest-ocu-candidates wrapper，避免 summary 只保留不可执行的字符串。 +- 2026-06-27：baseline summary artifact audit 现在也会离线检查 `nextActions` 的命令内容；如果 required official capture action 丢失 packet Make、`verify-inputs.sh`、inspect/import 或 strict gate，recommended packet set 丢失 `verify-all.sh`、`inspect-all.sh`、`import-all.sh` 或 `ingest-ocu-candidates.sh` 等批量 wrapper，或 standalone action 丢失生成 repo `check.sh`，`scripts/test-record-and-replay-baseline-summary-audit.py` 会用篡改摘要负例失败，避免 release / standalone audit 跟着漂移的下一步命令推进。 +- 2026-06-27：baseline summary artifact audit 现在还会检查派生状态不变量；`requiresOfficialGoldenCapture` 必须跟 official golden 完成状态一致，`officialSuccessfulRecordingEquivalenceReady` 必须等于 usable baseline + official golden complete，`standaloneRepoBaselineReady` 不能在 `usableBaseline=false` 时为 true，official golden complete 也不能同时带 required scenario gaps 或 coverage errors。summary audit smoke 已用篡改摘要覆盖这些失败路径。 +- 2026-06-27：baseline summary artifact audit 现在还会检查顶层 `checks` 声明是否包含所有必需 baseline smoke / compare / preflight / standalone 检查，并拒绝未知或重复 check 名称；audit JSON 会在 `declaredChecks.missingRequired`、`declaredChecks.unknown` 和 `declaredChecks.duplicates` 输出具体名称；summary audit smoke 会删除、追加和重复 check 名称并要求审计失败，避免 release / standalone audit 消费检查清单漂移或被污染的摘要。 +- 2026-06-27：baseline summary 顶层 `checks` 的 canonical list 已抽到 `scripts/record_and_replay_baseline_contract.py`；builder 按该 tuple 顺序输出，audit 按同一合同复审，summary audit smoke 仍断言输出顺序与 shared contract 完全一致，避免后续新增或删除 baseline 检查时只更新生成器或审计器一侧。 +- 2026-06-27：baseline smoke 的成功摘要现在也会携带 `evidence.officialFixtureSetGate`，证明集合级 official fixture gate、`simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` scenario policy、candidate pairing 和导入 scenario manifest 自检已跑过；同一 evidence 会扫描仓库真实 official recording fixtures，当前输出 `repoHasRequiredOfficialSuccessfulFixture=false` / `missingRepoOfficialScenarios=["simple-action-stop"]` 时，明确表示还没有官方 successful recording 入库，不把 gate smoke 误读成 golden 已完成。 +- 2026-06-27：baseline smoke 现在显式运行截图上下文 smoke，并在最终摘要写入 `evidence.screenshotContextSmoke`；该 evidence 要求 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 下出现 `screenshotNeededForContext`，并在当前环境实际返回截图数据时要求 `screenshotPath` 存在，避免 opt-in baseline 只证明事件流而没有覆盖截图上下文字段。 +- 2026-06-27：baseline smoke 现在显式运行官方 no-active response compare，并在最终摘要写入 `evidence.officialNoActiveResponse`；该 evidence 固定 Codex-hosted `event_stream_status` / `event_stream_stop` no-active text JSON 为 `isRecording=false` / `maxDurationSeconds=1800`，同时确认该对比路径不会创建 session 文件。 +- 2026-06-27：standalone thin skill repo manifest 新增 `officialEvidence` contract，声明当前独立 repo 骨架基于官方 `record-and-replay/1.0.857` 的 surface fixture、Codex-hosted no-active fixture 和 hostless raw timeout fixture，并显式标记还没有 successful recording golden；manifest 同时写入 required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景清单，并通过 `officialEvidence.scenarioRecipes` 固化每个场景的 capture goal、预期 action event、预期 endReason、evidence 要求和 OCU candidate 来源。源码 standalone、npm staged 和 baseline smoke 都会输出 `checkedOfficialEvidenceManifest=true`，并用 `checkedOfficialEvidenceScenarioManifest=true` 单独暴露这份场景合同检查。 +- 2026-06-27：official fixture coverage report 现在把 required 与 recommended 覆盖分开输出。required 仍只用 `simple-action-stop` 作为最小阻断门槛；recommended 默认报告 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`，并在 baseline 摘要里写入 `repoHasRecommendedOfficialScenarioCoverage` / `missingRepoRecommendedOfficialScenarios`，用于指引后续官方 golden 采样，不影响当前 `--allow-missing` 的通过语义。`--check-readiness` / `--require-readiness` 会在 required scenario 存在时继续跑集合级 readiness；baseline strict golden gate 现在要求 required 覆盖和 readiness 同时通过。 +- 2026-06-27：OCU thin Record & Replay skill 与 standalone repo workflow verifier 继续对齐官方 skill handoff：`event_stream_status` 只能在用户询问状态或录制后返回时使用，不能作为等待完成的轮询机制；MCP server 不直接暴露 event-stream 内容，agent 必须读取返回的 `eventsPath` / `metadataPath` / `sessionPath`。standalone repo smoke 现在带负例验证，删除 status 非轮询边界或删除 MCP 不直接暴露内容说明都会让 workflow verifier 失败；源码 standalone 与 npm staged 路径都会在 baseline evidence 中输出 `checkedStatusNotUsedAsWaitLoopGuard` / `checkedMcpNoDirectEventContentsGuard`。`make record-and-replay-skill-repo-smoke` 与 `make npm-record-and-replay-skill-repo-smoke` 已复跑通过。 +- 2026-06-27：standalone thin skill repo runtime contract 现在把 malformed official-compatible MCP calls 的副作用边界升为机器证据。生成态 manifest 声明 `mcpServer.rejectedRequestsDoNotCreateSessionFiles=true`；`verify-runtime.py`、源码 standalone smoke、npm staged smoke 和 baseline summary 都会要求 unexpected arguments、non-object arguments、缺失 / 非 object params、缺失 / 非 string tool name 被拒绝，并确认这些 rejected request 不创建 `latest-session.json` 或 `active-session.json`。源码与 npm staged baseline 现在也都显式消费 `checkedRequiresObjectParams`、`checkedRequiresStringToolName` 和 `checkedRequiresObjectArguments`，避免安装态 scaffold 只证明部分 request shape。 +- 2026-06-27：baseline summary 现在显式消费官方 raw start/status/stop 宿主外 timeout fixture。`scripts/test-event-stream-probe-fixtures.py` 输出 `checkedOfficialRawStartTimeoutBoundary`、`checkedOfficialRawStartStatusStopTimeout`、`checkedOfficialRawStartDoesNotReturnRecordingPaths`、`checkedOfficialRawSurface` 和 `checkedRawProbeFixtureRedaction`；`make record-and-replay-baseline-smoke` 会写入 `evidence.officialRawStartTimeout`，并把这些字段纳入 `usableBaseline` 必需 evidence，避免 standalone manifest 中的 hostless timeout 边界只停留在文档声明。 +- 2026-06-27：baseline summary 现在也显式消费 official golden capture preflight 与 OCU candidate pairing preflight。两个 preflight smoke 输出机器可读 JSON；`make record-and-replay-baseline-smoke` 会写入 `evidence.preflightPipelines`，并要求 capture packet Make targets、capture packet、single-scenario no-transcript packet、recommended packet set、no-transcript recommended packet set、单场景 verify-inputs、单场景 inspect/import placeholder guard、packet set 根级 verify-all、packet set 根级 OCU candidate handoff、packet set 根级 placeholder guard、packet set 根级 preflight-all placeholder guard、缺 official 样本、缺官方插件、coverage error、keyboard recording-required、缺 candidate、paired candidate 和 compare 成功路径都被覆盖，避免后续 official golden 采样与 OCU candidate 配对入口只在默认 CI 中存在、没有进入完整 baseline 可用性判断。 +- 2026-06-27：baseline summary 的 `evidence.preflightPipelines` 现在也要求 official capture packet placeholder guard；如果单场景 `inspect-only.sh` / `import-fixture.sh` 或 packet set 根级 `inspect-all.sh` / `import-all.sh` 不再拒绝 status response 或 MCP transcript 中的 `_placeholder=true` 输入，默认 baseline summary smoke 会失败；其中 `checkedOfficialCapturePacketImportPlaceholderGuard` 专门守住正式导入 wrapper，`checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard` 专门守住批量 wrapper 在执行任何子 wrapper 前先检查所有场景输入，避免半批次导入。 +- 2026-06-27：standalone thin skill repo manifest 现在把 preflight 边界也写入 `officialEvidence`。`sourceRepoBaselineChecks` 声明 hostless raw timeout、official capture preflight 和 OCU candidate pairing preflight 由 OCU 源仓 baseline 验证；`standaloneRepoBoundary` 声明独立 repo 默认自检不启动官方录制、不复制 preflight 脚本、不复制 OCU runtime 源码。源码 scaffold、npm staged scaffold 和 baseline summary 都新增 `checkedOfficialEvidencePreflightManifest=true`，避免 standalone repo 合同只保留场景清单却丢掉采样/配对前置门禁。 +- 2026-06-27：baseline summary 新增 `standaloneRepoBaselineReady` 与 `officialSuccessfulRecordingEquivalenceReady` 两个派生状态。当前缺 official successful recording golden 时，完整 baseline 可同时输出 `standaloneRepoBaselineReady=true` 和 `officialSuccessfulRecordingEquivalenceReady=false`，明确“可拆 standalone baseline”和“可宣称官方 successful recording 等价”不是同一件事。 +- 2026-06-27：baseline summary 新增 `nextActions`，把当前状态对应的下一步命令机器可读化：baseline evidence 缺失时先修 `make record-and-replay-baseline-smoke`，缺 required golden 时先跑 `make record-and-replay-official-golden-capture-packet RNR_SCENARIO= RNR_PACKET_DIR=`，再用 packet 内 `verify-inputs.sh`、`inspect-only.sh` / `import-fixture.sh` 和 strict gate 推进；recommended 未齐时跑 `make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR=` 并用 `verify-all.sh`、`inspect-all.sh` / `import-all.sh` / `ingest-ocu-candidates.sh` 推进，standalone baseline ready 时给出 scaffold repo 与生成 repo self-check 命令。 +- 2026-06-27：`scripts/run-record-and-replay-baseline-smoke.sh` 新增 `--summary-json `，可以把最终 baseline JSON 摘要写入文件作为 release / standalone audit evidence；脚本仍向 stdout 打印同一 JSON，并保持默认或严格 official golden gate 的原退出码。 +- 2026-06-27：Makefile 新增 `record-and-replay-baseline-audit` 与 `record-and-replay-official-golden-gate-audit`，分别包装默认 baseline 和 strict official golden gate 的 `--summary-json` 路径；baseline 默认落盘到 `dist/record-and-replay-baseline-summary.json`，可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖；strict 默认落盘到 `dist/record-and-replay-official-golden-gate-summary.json`，可用 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=` 覆盖，方便 release / standalone audit 留存 evidence 且避免 strict 失败摘要覆盖 baseline artifact。 +- 2026-06-27：新增 `scripts/test-record-and-replay-baseline-audit-make-targets.py` / `make record-and-replay-baseline-audit-targets-smoke`，通过 `make -n` 验证两个 audit Make target 的命令展开，确认 baseline 默认/自定义 summary 路径、strict 默认/自定义 summary 路径、strict 独立 artifact、baseline/strict summary 变量互不污染和 strict `--require-official-golden` 都未漂移；该检查已接入默认 CI，且默认 CI 会带外层 `RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-summary.json` 和 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-golden-summary.json` 运行它，防止脚本内部默认路径检查重新被调用方环境污染；该检查不启动完整 baseline 或官方录制。完整 baseline runner 也会运行它，并把对应 evidence 写入 `evidence.preflightPipelines`，缺失时 `usableBaseline=false`。 +- 2026-06-27：新增 `scripts/test-record-and-replay-baseline-runner-summary-json.py`，用临时 fake repo 和 stub evidence 跑真实 `run-record-and-replay-baseline-smoke.sh --summary-json` runner 路径，确认落盘 JSON 与 stdout 最终摘要一致；同一测试也覆盖 `--require-official-golden` 缺 required official fixture 时仍写出失败摘要并返回非零，且已接入默认 CI。 +- 2026-06-27：新增 `scripts/check-record-and-replay-baseline-summary.py` 和 `scripts/test-record-and-replay-baseline-summary-audit.py`，并接入默认 CI 与 `scripts/run-record-and-replay-baseline-smoke.sh` 最终 summary 生成路径。该审计入口只消费已落盘的 baseline summary，不重跑桌面 smoke；默认模式要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致、usable baseline、standalone repo baseline、official non-recording baseline 和 raw timeout boundary 成立，并直接检查 event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence，允许缺 official golden 但要求 `officialSuccessfulRecordingEquivalenceReady=false`；严格模式要求 official golden gate、required scenario gap、coverage error 和 equivalence ready 都满足。runner audit 输出走 stderr，避免破坏 stdout / `--summary-json` 的最终 summary 合同。 +- 2026-06-27：Makefile 新增 `record-and-replay-official-golden-capture-packet` 和 `record-and-replay-official-golden-capture-packet-set`。这两个入口只包装已有 official capture preflight，不启动官方录制；默认生成 required `simple-action-stop` 单场景 packet 或 required+recommended packet set，并可通过 `RNR_SCENARIO` / `RNR_PACKET_DIR` 覆盖；`RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`、`RNR_FIXTURE_ROOT` 和 `RNR_OFFICIAL_PLUGIN_ROOT` 让干净环境也能验证 packet 入口本身。official capture preflight smoke 已覆盖这两个 Make target。 +- 2026-06-27：新增 `scripts/ingest-official-record-and-replay-fixture.py` 和 `make event-stream-official-fixture-ingest-smoke`，把正常 Codex hosted R&R 录制结束后的 `event_stream_stop/status` JSON 转成脱敏 official fixture 的路径脚本化；脚本会递归解析 `content[].text` 中的 handoff path，也支持 `--status-json -` 从 stdin 读取刚复制出的 hosted JSON，并可用 `--status-json-base-dir` 为相对 handoff path 指定解析基准，调用 import 脚本，再按 scenario 跑 golden readiness，并可在提供 MCP transcript、通过 `--mcp-transcript -` 从 stdin 读取单独 transcript，或用 `--use-status-json-as-transcript` 复用同一 status JSON 时要求 response-shape evidence 与集合级 gate。该入口还支持 `--check-coverage` / `--require-coverage`，让导入后直接输出 official scenario 覆盖 + required readiness 报告，或在 required scenario 未齐 / readiness 未过时失败。该 smoke 已接入默认 CI，并覆盖 stdin 相对路径导入、transcript stdin、双 stdin 拒绝、post-ingest coverage/readiness report 和 require-coverage 失败路径。 +- 2026-06-27：official hosted ingest 入口新增 `--inspect-only`，可以在不创建 fixture 的情况下解析 hosted status JSON、recording input、handoff path 和 MCP transcript evidence；`make event-stream-official-fixture-ingest-smoke` 覆盖 inspect-only 成功、stdin status JSON 复用 transcript 成功、缺 transcript evidence 失败，baseline summary 同步要求 `checkedOfficialFixtureInspectOnly=true`，降低后续人工采集 official golden 时写入半成品 fixture 的风险。 +- 2026-06-27：新增 `scripts/ingest-ocu-record-and-replay-candidate.py` 和 `make event-stream-ocu-candidate-ingest-smoke`，把 OCU recording 或 action smoke JSONL 转成脱敏 `source=ocu` candidate fixture，并可在 official fixture 入库后通过 `--official-root --check-fixture-set` 直接跑 same-scenario 强对比；该 smoke 覆盖 direct recording、smoke JSONL、redaction 和 official candidate pairing，并接入默认 CI。candidate ingest 现在也会对 `keyboard-input-stop` / `drag-stop` 跑场景级 validation/readiness，要求对应的 `keyboard.text_input` / `mouse.drag` 事件、AX payload 和 completed MCP response-shape evidence，避免后续导入 keyboard/drag OCU candidate 时只通过通用结构检查。 +- 2026-06-27：baseline smoke 现在把 hosted official fixture ingest smoke 与 OCU candidate ingest smoke 纳入同一条 opt-in 聚合门禁，并在最终摘要写入 `evidence.fixtureIngestPipelines`；该 evidence 要求 official ingest、official `sessionDirectoryPath` handoff、candidate ingest、smoke JSON import、official/candidate pairing、candidate redaction、keyboard readiness 和 drag readiness 都为 true，确保后续采到官方 successful recording 后有现成入库/配对/候选校准链路。 +- 2026-06-27：`make event-stream-action-smoke` 现在通过 `event-stream mcp` start/repeat start/status/stop/repeat stop/final status 录制真实输入，并在 JSON 摘要中输出同 session `mcpTranscriptPath`；OCU candidate ingest 会从 smoke JSONL 自动消费该 transcript，`--run-action-smoke --require-mcp-transcript-evidence` 已实测可生成带 response-shape evidence 的 candidate fixture。 +- 2026-06-27：`make event-stream-action-smoke` 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop`，分别覆盖真实 click 和 drag 采样；默认 mixed action smoke 覆盖 click 和 scroll raw-only。`scripts/ingest-ocu-record-and-replay-candidate.py --run-action-smoke --scenario ` 会为 simple / drag 生成同名 OCU candidate；keyboard candidate 仍通过已有 recording / smoke JSON 导入和 readiness gate 覆盖，暂不走 synthetic keyboard action smoke。该能力仍属于 OCU candidate baseline，不能替代官方 successful recording golden。 +- 2026-06-27：新增 `scripts/prepare-record-and-replay-official-golden-capture.py` 作为 official successful recording 采集前的只读 preflight，报告 official plugin cache、当前 required/recommended scenario coverage/readiness，并生成 inspect-only、正式导入、strict gate 和 OCU candidate 命令；新增 `make record-and-replay-official-golden-capture-preflight-smoke` 并接入默认 CI，覆盖缺 required scenario、`--require-ready`、缺 official plugin、允许缺 plugin 和 keyboard 场景不走 synthetic action smoke 的输出合同。 +- 2026-06-27：official golden capture preflight 新增显式 `--capture-packet-dir `，在不启动官方录制、不写 fixture 的前提下生成 capture packet 目录，包含 `scenario-recipe.json`、placeholder hosted status/transcript JSON、`verify-inputs.sh`、`inspect-only.sh`、`import-fixture.sh` 和 coverage/check wrappers，方便录完官方样本后按同一合同替换输入、先验证输入存在/合法/非占位，再 inspect-only、再脱敏导入；消费 hosted JSON 的 wrapper 会拒绝 status response 或 MCP transcript 中仍带 `_placeholder=true` 的输入文件，避免误把占位 JSON 送入导入链路；`--capture-packet-recommended-scenarios` 会为 required + recommended 场景生成 per-scenario 子目录、根级 `capture-packets.json` 和根级 `verify-all.sh` / `inspect-all.sh` / `import-all.sh` / `check-all.sh` / `ingest-ocu-candidates.sh` 批量 wrapper；`verify-all.sh` 会逐个运行子 packet 输入校验，`inspect-all.sh` / `import-all.sh` 会先全量检查所有场景输入，再执行任何子 wrapper，`ingest-ocu-candidates.sh` 会批量运行可自动生成 OCU candidate 的子场景；单场景 `capturePacket`、根级 `capturePacketSet` 和 `capture-packets.json` 会用 `includeTranscript` / `requiresMcpTranscriptInput` 机器可读声明是否要求 transcript；使用 `--no-include-transcript` 生成单场景 packet 或批量 packet set 时，wrapper 只检查 status response，不要求未生成的 transcript 输入，manifest 字段也为 `false`；preflight smoke 覆盖单场景与批量 packet 的文件结构、wrapper 输入路径、verify-inputs / verify-all、两类 placeholder guard、manifest transcript requirement、根级批量 wrapper 对 placeholder 失败的传播、single-scenario no-transcript packet、no-transcript packet set，以及半批次导入防护。baseline summary 也把 `preflightPipelines.checkedOfficialCapturePacketNoTranscript` 和 `preflightPipelines.checkedOfficialCapturePacketTranscriptManifest` 纳入 `usableBaseline` 必需 evidence，并带缺失负例。 +- 2026-06-27：official capture packet 继续补齐采样后 handoff scripts：单场景 packet 的 README 和返回 JSON 现在显式暴露 `check-fixture-set.sh`、`strict-golden-gate.sh` 和可用时的 `ingest-ocu-candidate.sh`，并提供 `checkFixtureSetShell`、`strictGoldenGateShell`、`ingestOcuCandidateShell` 字段。preflight smoke 输出 `checkedCapturePacketHandoffScripts=true`，baseline summary、runner 早期断言和 artifact audit 通过 `checkedOfficialCapturePacketHandoffScripts` 消费该 evidence，避免后续采到官方样本后缺少 fixture-set gate、strict gate 或 same-scenario OCU candidate 导入命令。 +- 2026-06-27：official capture packet 的 OCU candidate wrapper 现在显式传入 `--output-dir /ocu-candidates`，让自定义 fixture root 下的 official fixture 与 same-scenario OCU candidate 保持同根配对。preflight smoke 输出 `checkedCapturePacketOcuCandidateOutputDir=true`，baseline runner、summary 和 artifact audit 通过 `checkedOfficialCapturePacketOcuCandidateOutputDir` 消费该 evidence，并有 summary / audit 篡改负例防止 handoff 命令漂移。 +- 2026-06-27：recommended official capture packet set 新增根级 `ingest-ocu-candidates.sh`，用于批量运行 simple / drag 等存在 `ingest-ocu-candidate.sh` 的子场景，并明确跳过 keyboard / cancel / timeout 这类不生成 synthetic wrapper 的场景。preflight smoke 输出 `checkedCapturePacketSetOcuCandidateHandoff=true`，baseline runner、summary 和 artifact audit 通过 `checkedOfficialCapturePacketSetOcuCandidateHandoff` 消费该 evidence；baseline summary 的 recommended `nextActions` 也会显式输出 `cd && ./ingest-ocu-candidates.sh`，artifact audit 通过 `checkedRecommendedGoldenNextActionIngestOcuCandidatesStep` 防止该 handoff 命令漂移，避免批量采样包导入后还要人工逐个查找可用 candidate wrapper。 +- 2026-06-27：OCU candidate ingest 输出新增后续 pairing/gate handoff commands；成功导入后会返回 `commands.pairingPreflightShell` 和 `commands.fixtureSetGateShell`。`make event-stream-ocu-candidate-ingest-smoke` 输出 `checkedCandidateIngestHandoffCommands=true`，baseline summary、runner 早期断言和 artifact audit 通过 `checkedOcuCandidateIngestHandoffCommands` 消费该 evidence，summary smoke 也新增 `checkedCandidateIngestHandoffCommandsEvidence=true` 与缺失负例，避免 same-scenario candidate 采样后仍要人工拼配对 preflight 或 fixture-set gate 命令。 +- 2026-06-27：新增 `scripts/prepare-record-and-replay-ocu-candidate-pairing.py` 作为 official fixture 入库后的只读 pairing preflight，报告 same-scenario OCU candidate 是否存在、是否 readiness/compare 通过，并生成 candidate ingest 与 fixture-set compare 命令；新增 `make record-and-replay-ocu-candidate-pairing-preflight-smoke` 并接入默认 CI，覆盖缺 official、official ready 但缺 candidate、candidate paired 通过和 keyboard 场景不走 synthetic action smoke 的输出合同。 +- 2026-06-27：official golden capture preflight 和 OCU candidate pairing preflight 都会显式暴露 coverage report errors；前者输出顶层 `coverageErrors`，后者输出 `officialCoverageErrors`，并把 recipe drift、重复 scenario 或 manifest 错误放进 `nextActions`，避免 official fixture manifest 已坏时继续按普通缺样本或缺 candidate 路径推进。 +- 2026-06-27：新增共享 `scripts/record_and_replay_scenarios.py`，让 official golden capture preflight 和 OCU candidate pairing preflight 都输出 `scenarioRecipe`；recipe 机器可读地记录 simple click、keyboard input、drag、cancel 和 timeout 场景的采集动作、预期 action event、预期 endReason、OCU candidate 来源方式和注意事项，避免官方采样步骤继续依赖聊天上下文。 +- 2026-06-27：`scripts/import-event-stream-fixture.py` 现在会把 `scenarioRecipe` 写入 `fixture-manifest.json`，`scripts/check-event-stream-official-fixture-set.py` 会拒绝 recipe 与 scenario catalog 不一致的 fixture；`make event-stream-fixture-smoke` / `make event-stream-official-fixture-set-smoke` 覆盖导入写入和 recipe drift 负例，避免官方样本只有 scenario 名但采集合同已经漂移。 +- 2026-06-27：scenario catalog 进一步成为 readiness policy 的单一来源；official fixture coverage 的 required / recommended 清单、official fixture set gate 的 scenario-specific readiness、official hosted ingest 和 OCU candidate ingest 的 direct readiness 都从 `scripts/record_and_replay_scenarios.py` 生成参数。coverage report 也会在不跑 readiness 时校验 `scenarioRecipe` drift，避免只有 scenario 名正确但采集合同漂移的 fixture 被算作官方覆盖。`make event-stream-official-fixture-set-smoke`、`make event-stream-official-fixture-coverage-smoke`、`make event-stream-official-fixture-ingest-smoke` 和 `make event-stream-ocu-candidate-ingest-smoke` 覆盖这条路径，其中 official ingest smoke 现在也直接覆盖 `keyboard-input-stop` / `drag-stop`，避免推荐场景只在集合级 gate 中被校验。 +- 2026-06-27：复跑 `make record-and-replay-baseline-smoke` 通过；最终摘要显示 `status.usableBaseline=true`、`status.officialSuccessfulRecordingGoldenComplete=false`、`status.requiresOfficialGoldenCapture=true`，并继续报告缺少 required `simple-action-stop` 和 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 官方 successful recording fixture。该次验证固定了 default action smoke 更新后的 baseline 聚合口径：真实输入 evidence 要求 click、AX context、MCP transcript 和 skill 草稿，不再把 synthetic keyboard event 作为默认必需项；2026-07-01 后 scroll raw-only 改为 best-effort evidence。 +- 2026-06-27：新增 `scripts/check-event-stream-official-fixture-set.py` 和 `make event-stream-official-fixture-set-smoke`，把官方 successful recording 入库后的 scenario 覆盖、official readiness 和 OCU candidate 强对比收敛成集合级 gate；`scripts/import-event-stream-fixture.py --scenario` 会把样本场景写入 manifest，避免后续靠目录名人工配对。 +- 2026-06-27：official fixture set gate 现在会在 compare 前对 OCU candidate 跑同 scenario readiness，并新增 candidate alias 漂移负例；这让后续手动导入的 candidate fixture 必须先证明自身 metadata/session/count/path/MCP evidence 完整，不能只靠事件序列 compare。 +- 2026-06-27：新增 `scripts/check-event-stream-official-fixture-coverage.py` 和 `make event-stream-official-fixture-coverage-smoke`，把“仓库真实 official successful recording fixture 是否覆盖 required scenario”从 baseline smoke 内部扫描抽成独立 report；默认缺 `simple-action-stop` 时失败，默认 CI 用 `--allow-missing` 只记录当前缺口，避免 synthetic gate 通过被误读成 official golden 已完成。该 report 也会输出 recommended scenario 覆盖，默认清单是 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`；打开 `--check-readiness` / `--require-readiness` 时会继续证明 required scenario 不是空壳 manifest，而是通过 official fixture set readiness。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 新增 `screenshotPath` session 边界 gate；事件 payload 中的截图路径必须解析到当前 session 目录内且文件存在，避免 recording-to-skill 或 fixture 校准链路消费引用到录制包外部的截图。Swift 单测与 `make event-stream-skill-scaffold-smoke` 覆盖外部但存在的 screenshot 文件负例。 +- 2026-06-27：standalone thin skill repo scaffold 已把 `screenshotPath` session 边界 gate 下沉到导出 repo 的默认自检；生成态 `scripts/recording-to-skill-smoke.py` 会构造外部截图路径负例，manifest 声明 `recordingToSkill.strictValidation.requiresScreenshotPathsInsideSession=true`，baseline / npm staged smoke 都要求 `checkedScreenshotPathContainment=true` evidence。 +- 2026-06-27：npm staged standalone smoke 已把 `screenshotPath` session 边界 gate 做成安装态 scaffold 前置 preflight；如果本地 `dist/Open Computer Use.app` 还是旧 native artifact，会直接提示重建 `dist`，而不是在生成 repo 的 `scripts/check.sh` 里晚失败。 +- 2026-06-27：baseline summary 现在消费 source standalone 与 npm staged recording-to-skill 的具体语义 evidence：strict validation、events-only validation、scaffold-skill 正路径和 skill-creator handoff 任一缺失都会让 `usableBaseline=false`。这把独立 repo 默认自检从“有 recording-to-skill smoke”提升为“源码与安装态关键 handoff 子路径都可证明”。 +- 2026-06-27：baseline summary 现在也消费真实输入 action smoke 的 concrete evidence：MCP response shapes、`skillReadiness.canCreateSkillDraft`、`skill-creator` finalization handoff 和生成草稿路径脱敏都成为 `realInputActionSmoke` 必需项，避免只凭 `skillPath` / `mcpTranscriptPath` 存在就误判 action smoke 录制到 skill 链路完整。 +- 2026-06-27：official fixture set gate 现在带 scenario-specific readiness：`simple-action-stop` 要求 `mouse.click` 和 `recording_controls_stopped` endReason，`keyboard-input-stop` 要求 `keyboard.text_input` 和 `recording_controls_stopped` endReason，`drag-stop` 要求 `mouse.drag` 和 `recording_controls_stopped` endReason，`cancel` 要求 `recording_controls_cancelled` endReason，`timeout` 允许只有生命周期事件且在官方 endReason 未观测前保持不固定；smoke 已覆盖 timeout no-action / no-AX fixture 通过。 +- 2026-06-28：official fixture set gate 的 same-scenario compare 默认提升到 AX / suppressed 强策略：`scripts/check-event-stream-official-fixture-set.py` 现在固定传入 `--require-ax-diff-evidence`、`--require-same-ax-diff-markers`、`--require-same-suppressed-event-sequence` 和 `--require-same-suppressed-schema`。`make event-stream-official-fixture-set-smoke` 覆盖 candidate 缺 AX diff / cumulative diff 的失败路径；baseline summary 和 artifact audit 消费 `checkedAxDiffComparisonPolicy`、`checkedSuppressedStreamComparisonPolicy` 与 `checkedAxDiffComparisonFailure`，避免 official golden 入库后 compact diff 或 suppressed fallback 缺失仍被误判为可用。 +- 2026-06-28：standalone thin skill repo manifest 现在把 official fixture set gate 的强对比策略下沉到 `officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy`，机器声明 AX diff evidence、AX diff marker、suppressed event sequence 和 suppressed schema 都属于官方 golden 入库后的 same-scenario compare 边界。生成态 manifest verifier、source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 消费 `checkedOfficialFixtureSetComparePolicyManifest`，缺失时 `usableBaseline=false`。 +- 2026-06-28：standalone thin skill repo 生成物新增 `scripts/verify-source-baseline-summary.py` 和 `evidence/source-baseline-summary.json`。scaffold 会从源仓 `dist/record-and-replay-baseline-summary.json` 复制脱敏投影，默认 `scripts/check.sh` 会审计 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、official golden 状态自洽（当前 required `simple-action-stop` gap 明确，或未来 official golden/equivalence ready 完成态成立），以及 source / npm staged standalone evidence 保留 fixture-set compare policy；source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 消费 `checkedSourceBaselineSummaryEvidence` 与 `checkedSourceBaselineSummaryOfficialGoldenState`，缺失时 `usableBaseline=false`。 +- 2026-06-28：standalone source summary projection 继续补齐 official capture packet set 合同证据。生成态 `evidence/source-baseline-summary.json` 现在保留 `evidence.preflightPipelines.checkedOfficialCapturePacketSetContractManifest`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketSetContractManifest=true` 并有缺失负例；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费该字段，避免独立 repo 自检只证明 source summary 存在，却不能证明 recommended packet set 的集合级 `captureContracts` 合同仍被源仓 baseline 守住。 +- 2026-06-28：standalone source summary projection 继续继承 official capture packet post-capture workflow 证据。生成态 `evidence/source-baseline-summary.json` 现在保留 `evidence.preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow` 与 `evidence.preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true` 和 `checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true` 并有缺失负例；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费这些字段，避免独立 repo 自检漏掉官方采样包录制后的机器可读 handoff 顺序。 +- 2026-06-28：standalone source summary projection 也继承 official capture packet strict audit handoff 证据。生成态 `evidence/source-baseline-summary.json` 现在保留 `evidence.preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 并有缺失负例；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费该字段，避免独立 repo 自检漏掉 official fixture 入库后刷新 strict summary artifact 的 handoff。 +- 2026-06-28：standalone source summary projection 继续继承 official capture packet strict expected-failure handoff 证据。生成态 `evidence/source-baseline-summary.json` 现在保留 `evidence.preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true` 并有缺失负例；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费该字段，避免独立 repo 自检漏掉 required official golden 缺失时的 `--allow-strict-official-golden-missing` 审计 handoff。 +- 2026-06-28：standalone generated repo 新增 package artifact verifier。默认 `scripts/check.sh` 现在在 `scripts/package-skill.sh` 后运行 `scripts/verify-package-artifact.py`，校验 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` frontmatter 和 Record & Replay handoff 片段仍存在；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费 `checkedPackageArtifact=true`，避免独立 repo 只证明 `.skill` 文件存在却没有证明制品内容可安装。 +- 2026-06-28：official capture packet 新增 workflow verifier。单场景 packet 和 recommended packet set 都会生成 `verify-workflow.sh`，在没有 hosted JSON 时先校验 `postCaptureWorkflow`、input 文件、wrapper command、可执行 bit 和 transcript 开关一致；preflight smoke 输出 `checkedCapturePacketWorkflowVerifier=true` / `checkedCapturePacketSetWorkflowVerifier=true`，baseline summary / artifact audit 消费 `checkedOfficialCapturePacketWorkflowVerifier` / `checkedOfficialCapturePacketSetWorkflowVerifier`，standalone source summary 也继承为 `checkedSourceBaselineSummaryCapturePacketWorkflowVerifier` / `checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier`。 +- 2026-06-28：official hosted capture transcript gate 对齐官方 skill 流程，不再把额外 `repeatStartResponseShape` 当成 successful recording 必需证据；required action scenario 现在要求 `startResponseShape`、`statusResponseShape`、`stopResponseShape` 和 `finalStatusResponseShape`。新增 `scripts/finalize-record-and-replay-official-capture-packet.py` 与 `make record-and-replay-official-capture-finalizer-smoke`，把 hosted start/status/stop/final-status JSON 规整成 packet 的 `inputs/event_stream_stop-response.json` 和 `inputs/mcp-transcript.json`，并立即跑 `verify-workflow.sh` / `verify-inputs.sh`。 +- 2026-06-28：fixture import 现在覆盖 official hosted 最小 session 包。`scripts/import-event-stream-fixture.py` 可直接导入只有 `events.jsonl + session.json` 的官方录制目录，脱敏 `session.json.id`，为 fixture 生成可验证的 `metadata.json`、官方风格 `session.json` 和空 `suppressed.jsonl`，并通过 `make event-stream-fixture-smoke` 固定 `--require-session-alias` / `--require-metadata-counts` / `--require-handoff-paths` readiness。这样后续 Codex-hosted official golden 不需要先手工补 OCU metadata 或 suppressed 文件。 +- 2026-06-28：README / README.zh-CN 的 Record & Replay 快速命令对齐 artifact-first baseline 流程。用户入口现在显式给出 `make record-and-replay-baseline-audit`、`make record-and-replay-official-golden-capture-packet` / `record-and-replay-official-golden-capture-packet-set`、`make record-and-replay-official-golden-gate-audit` 和 `--allow-strict-official-golden-missing` 审计命令；`scripts/check-docs.sh` 增加 README 针脚，避免公开入口退回只跑不落盘的 smoke / strict gate。 +- 2026-06-27：复跑 `make record-and-replay-baseline-smoke` 通过，最终摘要覆盖默认 matrix、真实输入 action smoke、官方 1.0.857 non-recording surface compare、official fixture set gate、official fixture coverage report、standalone skill repo smoke 和 npm staged skill repo smoke；`evidence.officialFixtureSetGate.checkedTimeoutScenarioPolicy=true`，同时仍报告 `repoHasRequiredOfficialSuccessfulFixture=false` / `missingRepoOfficialScenarios=["simple-action-stop"]`，说明 baseline 可用性和 official golden 缺口被同一份摘要区分开。 +- 2026-06-27：复跑新增 recommended coverage evidence 后的 `make record-and-replay-baseline-smoke` 通过，最终摘要里 `recommendedOfficialScenarios=["simple-action-stop","keyboard-input-stop","drag-stop","cancel","timeout"]`，`repoHasRecommendedOfficialScenarioCoverage=false`，`missingRepoRecommendedOfficialScenarios=["cancel","drag-stop","keyboard-input-stop","simple-action-stop","timeout"]`；这说明当前 baseline 仍可用，但官方 successful recording 的 required 与 recommended golden 样本都尚未入库。 +- 2026-06-27：`make event-stream-official-fixture-set-smoke`、`make event-stream-fixture-smoke`、`make event-stream-golden-readiness-smoke` 和 `make event-stream-compare-smoke` 已复跑通过，确认 official fixture set gate 与既有导入/readiness/compare 工具链兼容。 +- 2026-06-27：`make event-stream-official-start-probe` 再次确认官方 1.0.857 bundled raw MCP client 在宿主外只能返回 initialize / tools-list，`event_stream_start/status/stop` 均 10 秒超时且不返回 recording paths；official successful recording 仍需从正常 Codex 宿主流程采集。 +- 2026-06-27：刷新官方 raw start-timeout fixture，入库 transcript 现在显式包含 `startResponseShape`、`statusResponseShape` 和 `stopResponseShape` 三个 10 秒 timeout；`scripts/test-event-stream-probe-fixtures.py` 同步要求三次 `tools/call` timeout，避免官方宿主外边界证据退回旧的 start/stop-only 形态。 +- 2026-06-27：默认 `make ci` 已复跑通过，覆盖文档/仓库卫生、脚本语法、skill 打包、Record & Replay 默认 smoke、Swift 单测、默认 event-stream matrix 和 Windows / Linux Go test；真实输入 action 与 npm staging baseline 继续由 opt-in baseline smoke 覆盖。 +- 2026-06-27：再次复跑完整 opt-in `make record-and-replay-baseline-smoke` 通过；最终摘要显示 `status.usableBaseline=true`、`status.standaloneRepoBaselineReady=true`、`status.officialNonRecordingBaselineVerified=true`、`status.officialRawStartTimeoutBoundaryVerified=true`，同时保持 `status.officialSuccessfulRecordingGoldenComplete=false`、`status.officialSuccessfulRecordingEquivalenceReady=false`。这证明当前 OCU / standalone baseline 可用，但 official successful recording schema、AX diff 和 screenshot 触发策略仍必须等待 Codex-hosted official golden fixture 校准。 +- 2026-06-27：strict official golden gate audit 暴露并修复一个 dry-run smoke 环境污染问题：当外层设置 `RNR_BASELINE_SUMMARY_JSON` 时，`scripts/test-record-and-replay-baseline-audit-make-targets.py` 的默认路径检查也会继承该值。脚本现在会在内部 `make -n` 前移除继承环境，后续也同步移除继承的 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`；默认 `scripts/ci.sh` 会同时用两个外层变量覆盖运行该 smoke。重跑 `record-and-replay-official-golden-gate-audit` 后只因缺 required `simple-action-stop` official golden 按预期非零退出，并写出 strict summary artifact。 diff --git a/docs/design-docs/index.md b/docs/design-docs/index.md index 20ff742c..0875efd0 100644 --- a/docs/design-docs/index.md +++ b/docs/design-docs/index.md @@ -8,6 +8,22 @@ - 每份文档写清当前状态和简短摘要。 - 关联引入它的 execution plan 或 spec。 -## 初始文档 +## 文档 - `core-beliefs.md` +- `record-and-replay-replication.md` + - Record & Replay 官方复刻和 OCU 独立扩展的设计入口，包含官方兼容层、OCU 扩展层、推进顺序和当前缺口。 +- `record-and-replay-handoff.md` + - Record & Replay 后续推进的短入口，汇总当前决策、官方上下文、OCU baseline、待解决问题、推进协议、下一轮推进顺序和常用验证命令。后续处理 R&R 时优先从这里读起。 +- `record-and-replay-official-golden-capture.md` + - Record & Replay 官方 successful recording golden fixture 的采集、inspect-only、正式导入、OCU candidate 对比和验收 gate 操作清单。 + +## Record & Replay 阅读顺序 + +后续继续推进 Record & Replay 时，默认按下面顺序读： + +1. `record-and-replay-handoff.md`：先确认当前口径、边界、待解决问题和下一步路径。 +2. `record-and-replay-replication.md`：再看完整方案，区分官方兼容层、OCU 扩展层、校准合同和独立 repo 合同。 +3. `record-and-replay-official-golden-capture.md`：需要采集或导入官方 successful recording fixture 时读这份。 +4. `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`：需要追溯官方插件、Codex.app asar、二进制字符串、MCP surface 和 raw probe 证据时读这份。 +5. `../exec-plans/active/20260626-record-and-replay-event-stream.md`：需要确认当前里程碑、验证命令和执行拆解时读这份。 diff --git a/docs/design-docs/record-and-replay-handoff.md b/docs/design-docs/record-and-replay-handoff.md new file mode 100644 index 00000000..8d7e5aab --- /dev/null +++ b/docs/design-docs/record-and-replay-handoff.md @@ -0,0 +1,272 @@ +# Record & Replay Handoff + +## 状态 + +本文是 Record & Replay 后续推进的短入口。完整设计看 `record-and-replay-replication.md`，官方逆向证据看 `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`，执行拆解看 `../exec-plans/active/20260626-record-and-replay-event-stream.md`。 + +当前共识是：先直接复刻官方 Record & Replay 的可观察行为，再补 OCU 为独立运行必须拥有的控制条、等待和通知能力。官方兼容层和 OCU 扩展层必须保持边界清楚。 + +## 落库地图 + +后续推进不应再依赖聊天上下文。当前方案和上下文已经按用途拆到这些位置： + +- `record-and-replay-handoff.md`：短入口，记录当前结论、边界、推进协议、待解决问题和下一轮顺序。 +- `record-and-replay-replication.md`：完整设计入口，记录官方兼容层、OCU 扩展层、校准合同、独立 repo 合同、事件/AX/screenshot/control bar 方案和验收门槛。 +- `record-and-replay-official-golden-capture.md`：官方 successful recording golden fixture 的短操作入口，记录采集前检查、hosted JSON inspect-only、正式导入、OCU candidate 对比和严格验收 gate。 +- `../exec-plans/active/20260626-record-and-replay-event-stream.md`：执行计划，记录里程碑、已完成 baseline、P0/P1/P1.5/P2 推进拆解和具体命令。 +- `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`：官方上下文和证据入口，记录 plugin 包装、Codex.app asar 观察、Computer Use 二进制字符串、MCP surface、official fixture 和 raw probe 边界。 +- `../../skills/open-computer-use-record-and-replay/SKILL.md`：OCU thin skill baseline，约束独立 repo 中 agent 如何调用三件套、停止录制、读取产物和生成 skill。 +- `../../skills/open-computer-use/references/record-and-replay.md`：通用 OCU skill 的 R&R 参考说明。 +- `../../scripts/prepare-record-and-replay-official-golden-capture.py`：官方 successful recording golden 采集前的只读 preflight，报告当前 official scenario 覆盖、官方 plugin cache、scenario recipe、下一条 inspect/import/OCU candidate 命令。 +- `../../scripts/prepare-record-and-replay-ocu-candidate-pairing.py`：官方 fixture 入库后的只读 pairing preflight，报告 same-scenario OCU candidate 是否存在/ready，输出 scenario recipe、candidate ingest 与 fixture-set compare 命令。 +- `../../scripts/scaffold-record-and-replay-skill-repo.py`：源码 checkout 的 standalone repo scaffold 入口。 +- `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `：npm 安装态的 standalone repo scaffold 入口；用于后续真正创建独立 repo。 + +后续如果方案、证据或接口合同变化，优先更新本 handoff、完整设计、execution plan 和逆向参考，再按需同步 skill 与 README。 + +## 落库完成判定 + +后续处理 Record & Replay 时，不应再把聊天记录当作唯一上下文。至少要能从仓库内回答这些问题，才算方案已经落库完整： + +- 为什么选择直接复刻官方可观察行为，而不是重新设计一套 OCU 协议。 +- 哪些行为属于官方兼容层，必须保持 `event_stream_start/status/stop` 三件套无参数。 +- 哪些能力属于 OCU 独立扩展层，例如开始确认 UI、录制控制条、`wait`、notify callback、validation、summary、skill scaffold 和 standalone repo self-check。 +- 哪些字段、算法或触发策略仍只是 OCU baseline，必须等待 official golden recording 校准后才能宣称官方等价。 +- 后续如果要继续复刻官方，应该先采集什么样本、用哪些脚本脱敏导入、如何比较 OCU candidate。 +- 后续如果要拆真实 standalone repo，应该以哪个 scaffold 为蓝本、默认 CI 跑哪些不启动录制的检查、哪些 lifecycle smoke 只能 opt-in。 + +当前这些问题分别落在本 handoff、`record-and-replay-replication.md`、active execution plan、逆向参考文档、thin skill 和 standalone repo scaffold contract 中。推进新改动时，如果发现某个回答只能从聊天上下文里推断，就先补文档再继续实现。 + +## 本轮结论 + +这次推进的核心结论是：OCU 后续不要另设计一套 Record & Replay 协议，而是以官方当前可观察行为为默认目标。官方兼容路径继续围绕 `event_stream_start/status/stop` 三个 no-arg MCP tools、官方 session 文件、事件流、AX payload、截图上下文和录制到 skill 的 handoff 收敛；任何暂时没有 official golden recording 证明的实现都只能标成 OCU baseline 或待校准。 + +独立运行能力另放在 OCU 扩展层。因为 OCU 不能依赖 Codex.app 替它绘制录制 bar、接收 Done / Discard 或转发结束回调，所以开始确认 UI、录制中控制条、`cancel`、`wait --session-id`、`wait --notify-command`、validate / summarize / scaffold 和 standalone thin skill repo 都归 OCU 扩展层。上层使用方可以监听 `wait` / notify 结果继续生成 skill，但官方兼容 MCP surface 不因此增加工具、参数或 callback 字段。 + +后续拆独立 repo 时，repo 只包装 thin skill 和 runtime contract：要求安装 `open-computer-use`，启动 `open-computer-use event-stream mcp`，用 OCU 自有控制条完成录制，再读取 `metadataPath` / `eventsPath` 走 validation、summary 和 skill scaffold。独立 repo 不复制 OCU runtime 源码，也不假设 Codex.app 私有 UI、feature gate 或回调一定存在。 + +## Baseline 实施基准 + +后续第一波 baseline 的目标是“能用且不误报官方等价”，不是一次性完成所有官方算法级校准。实现和验收时按下面口径推进： + +- 官方兼容 baseline：`open-computer-use event-stream mcp` 只暴露 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三件套 no-arg tools；server name、initialize / tools-list、no-active status/stop response、session 文件布局、`session.started` / `session.ended` 生命周期、已确认动作事件、AX payload 和截图上下文都向官方样本收敛。 +- OCU 独立 baseline：OCU 自己提供开始确认 UI、录制中控制条、Done / Discard、`cancel`、`wait --session-id`、`wait --notify-command`、validate / summarize / scaffold 和 standalone repo self-check；这些能力可以交给独立 plugin / skill repo 使用，但不能进入官方兼容 MCP surface。 +- 可交付判定：默认 baseline summary 应能给出 `usableBaseline=true` 和 `standaloneRepoBaselineReady=true`。当 required official successful recording golden 缺失时，必须保持 `officialSuccessfulRecordingGoldenComplete=false`、`officialSuccessfulRecordingEquivalenceReady=false` 和 `requiresOfficialGoldenCapture=true`；当 required `simple-action-stop` 已入库且 readiness 通过时，可以把最小 required golden 标为完成，但 recommended 场景和算法级 schema 未齐前仍不能宣称全部 official successful recording schema、AX compact diff 或 screenshot 策略已经等价。 +- official fixture set gate 是 official golden 入库后的强对比入口。同 scenario OCU candidate 进入 compare 前必须先通过自身 readiness；compare 默认要求 AX diff evidence、AX diff marker 一致、suppressed event sequence 一致和 suppressed schema 一致。当前 baseline summary 会消费 `checkedAxDiffComparisonPolicy`、`checkedSuppressedStreamComparisonPolicy` 和 `checkedAxDiffComparisonFailure`，避免后续拿到官方 compact diff 或 suppressed fallback 后，candidate 缺字段仍被误判通过。 +- 暂不升级为官方等价的内容：AX compact diff 的 element identity / line budget / fallback 条件、截图触发阈值、timeout endReason、raw scroll event schema、terminal / selection / secure input / debug 字段，以及 MCP elicitation 的真实业务 message / `_meta`。这些都必须等 official golden recording 或新的官方 fixture 进入后再校准。 +- 后续真实 standalone repo 只消费 OCU runtime 和 thin skill。默认 CI 跑不启动录制的 package、runtime contract、skill workflow、wait/notify 和 synthetic recording-to-skill checks；真实桌面 lifecycle smoke 只能作为 opt-in。 +- standalone repo 默认自检还必须跑生成态 `scripts/verify-manifest.py`，也就是 `checks.manifestContract=scripts/verify-manifest.py`。这条检查只验证 `record-and-replay-skill-repo.json` 的机器合同：官方 evidence、required / recommended scenario、`sourceRepoBaselineAudit`、`strictOfficialGoldenExpectedFailureAudit`、`officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy`、OCU `extensionLayer`、官方三件套 no-arg surface 和 recording-to-skill handoff 是否漂移，不启动录制，也不把任何 OCU 扩展加进官方 MCP surface。source / npm staged smoke 与 baseline summary 都要输出 / 消费 `checkedManifestContract=true` 和 `checkedOfficialFixtureSetComparePolicyManifest=true`。 +- standalone repo 默认自检还必须跑生成态 `scripts/verify-source-baseline-summary.py`，也就是 `checks.sourceBaselineSummaryEvidence=scripts/verify-source-baseline-summary.py`。scaffold 会把源仓 `dist/record-and-replay-baseline-summary.json` 的脱敏投影复制到 `evidence/source-baseline-summary.json`，该 verifier 校验 `usableBaseline=true`、`standaloneRepoBaselineReady=true`，并要求 official golden 状态自洽：当前可以是 required `simple-action-stop` gap 明确，也可以是未来 official golden gate / successful recording equivalence 已完成；如果仍缺 required scenario 却声明 equivalence ready 会失败。它还会确认 source / npm staged standalone evidence 都保留 `checkedOfficialFixtureSetComparePolicyManifest=true`，并确认 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff=true`、`preflightPipelines.checkedOfficialCapturePacketSetContractManifest=true`、`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketWorkflowVerifier=true`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketSetWorkflowVerifier=true`、`preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff=true` 和 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff=true` 没有在投影里丢失。source / npm staged smoke 与 baseline summary 都要输出 / 消费 `checkedSourceBaselineSummaryEvidence=true`、`checkedSourceBaselineSummaryOfficialGoldenState=true`、`checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff=true`、`checkedSourceBaselineSummaryCapturePacketSetContractManifest=true`、`checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketWorkflowVerifier=true`、`checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier=true`、`checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 和 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`。源仓 baseline 还会单独消费 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff=true`、`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketWorkflowVerifier=true`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow=true` 与 `preflightPipelines.checkedOfficialCapturePacketSetWorkflowVerifier=true`，证明 official completed directory handoff 与 capture packet / packet set 的机器合同不只声明文件和 wrapper，还声明录制后的有序 handoff 以及 `verify-workflow.sh` 执行计划校验。 +- standalone repo 默认自检还必须跑生成态 `scripts/verify-readme-handoff.py`，也就是 `checks.readmeHandoffContract=scripts/verify-readme-handoff.py`。这条检查只验证 README 是否继续暴露官方 evidence 文件、baseline / strict audit handoff、strict 缺 official golden 的 expected-failure 审计命令、required / recommended successful recording 场景、recording-to-skill 命令和 wait/notify 边界；不启动录制，也不改变官方兼容 MCP surface。source / npm staged smoke 与 baseline summary 都要输出 / 消费 `checkedReadmeHandoffContract=true`、`checkedReadmeOfficialEvidenceHandoff=true`、`checkedReadmeOfficialGoldenGap=true` 和 `checkedReadmeWaitNotifyBoundary=true`。 + +## 快速上下文结论 + +这些结论是后续推进时最容易在聊天上下文里丢失的部分，默认以这里为准： + +- 官方 `record-and-replay` plugin 只是包装入口；它通过 `.mcp.json` 启动同一个 Computer Use runtime 的 `event-stream mcp` 模式。普通 Computer Use 和 Record & Replay 共用 runtime，但暴露的 MCP surface 不同。 +- 当前对 Codex.app asar 的观察只支持把 Codex.app 视为宿主侧 feature gate、插件可见性和通用 MCP elicitation / approval UI 的提供方；没有证据表明 Codex.app 自己实现 `event_stream_*` tools、写 recording files、绘制录制控制条或保存事件流。 +- OCU 因此不能耦合 Codex.app。开始确认 UI、录制中 bar、Done / Discard、结束状态、`wait` listener 和 notify callback 都必须由 OCU runtime 自己承载；使用方只通过 OCU 暴露的 CLI / MCP / generated standalone repo contract 集成。 +- 官方兼容 MCP surface 只允许 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三个无参数 tools。`cancel`、`wait`、notify callback、validation、summary、skill scaffold 和 standalone repo self-check 是 OCU 扩展层，不进入官方三件套 schema。 +- 截图不是“每个事件默认截图”。官方证据只确认 `screenshotNeededForContext` / `accessibilityInspectorPayload` 这类上下文线索；OCU 默认应最小化截图，只在 payload 判断需要视觉上下文且当前 snapshot 可用时写 `screenshots/` 并给出 `screenshotPath`，触发阈值仍等 official golden recording 校准。 +- Compact AX diff 指的是同一 app/window 的后续 AX payload 不再重复输出整棵 accessibility tree，而是用官方风格的 `~` / `+` / `-` 表示 changed / added / removed，并保留 previous diff 与 cumulative diff 语义。OCU 当前 LCS 风格实现只是 baseline；element identity、line budget、removed summary 和 fallback 条件必须用 official golden recording 校准。 +- 当前已有官方 non-recording surface fixture、Codex-hosted no-active status/stop fixture、宿主外 raw start/status/stop timeout fixture，以及 required `simple-action-stop` official successful recording fixture。前几类只能证明 surface、idle response 和宿主外边界；`simple-action-stop` 只满足最小 required successful recording 门槛，不能覆盖 keyboard、drag、cancel、timeout、截图阈值或 AX compact diff 算法级等价。 +- `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 仍是 recommended 采样清单。没有这些样本前，不能宣称 OCU 已完成所有 official successful recording 场景复刻。 +- 下一步不应再从聊天里拼命令。required 样本已入库时，先用 `make record-and-replay-official-golden-fixture-gate` / `make record-and-replay-official-golden-gate-audit` 留存 readiness 和 strict summary artifact，再用 OCU candidate ingest / pairing preflight 生成同场景 candidate，跑 fixture set gate 的 AX diff + suppressed stream 强对比；后续 recommended 场景继续走 capture packet verify / inspect / import wrapper。 + +## 决策摘要 + +- 第一阶段只做 macOS。 +- “直接复刻官方”在本仓库里的含义是复刻可观察协议、文件、事件、状态流转和生成 skill 的 handoff 行为；不得复制官方私有实现代码，也不得把 OCU 私有便利参数加入官方兼容 MCP surface。 +- 官方兼容 MCP surface 只保留 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三个无参数 tools。 +- 同一时间只允许一个 active recording；重复 start 返回 active session 是 runtime one-active 语义，agent 侧需要询问用户是使用该 active recording 还是等待它结束，不能静默当作新 demonstration。 +- `cancel`、`wait`、`validate`、`summarize`、`scaffold-skill`、控制条交互和 notify callback 都属于 OCU 扩展层，不能进入官方三件套 tool schema。 +- OCU 自己绘制开始确认 UI 和录制控制条，不能依赖 Codex.app 绘制 bar 或把用户点击转发回来。 +- `events.jsonl` 是后续生成 skill 的主证据，`metadata.json` / `session.json` 只提供状态、路径、timing 和计数。 +- 被用户取消的 recording 不能用于创建或更新 skill；上层应该提示重新录制。 +- 截图是上下文补充，不是每个事件默认主数据。 +- AX compact diff 是官方事件流的重要组成部分，最终必须用 official golden recording 校准，而不是长期停留在 full tree 或概念相似的 diff。 +- 独立 thin skill / plugin repo 只包装 OCU runtime，不复制本仓库录制逻辑。 + +## 后续推进协议 + +后续基于本文推进时，先把每个改动归到三类之一： + +- **官方兼容层**：只改 `event-stream mcp` 三件套、session 文件、官方已确认事件和 official golden 校准逻辑。这里不能增加新的 MCP tool、tool 参数、callback 参数或 OCU 私有状态依赖。 +- **OCU 扩展层**：只服务独立运行，包括开始确认 UI、录制控制条、`cancel`、`wait`、notify callback、validate / summarize / scaffold、standalone thin skill repo 和安装便利命令。这里可以比官方多，但必须在 docs 和测试里明确是 OCU 扩展。 +- **待官方校准 baseline**：已经有实现价值但没有 official golden 证明的字段、算法或触发策略。这里可以落代码和 smoke，但不能在文档或注释里宣称“已官方等价”。 + +每次推进都按这个顺序收口： + +1. 先补或引用证据：官方 fixture、asar 观察、二进制字符串、官方 skill 文案、或 OCU smoke 结果。 +2. 再做实现：官方兼容层保持三件套不变，扩展能力走 CLI / runtime / standalone repo。 +3. 再补验证：surface drift、recording validation、golden readiness、recording compare、skill scaffold 或 standalone repo smoke 中至少选中对应路径。 +4. 最后同步文档：如果字段、流程、边界或验收口径变化，同步更新 `record-and-replay-replication.md`、本 handoff、exec plan、reverse-engineering reference 和 history 中受影响的部分。 + +判断一个行为能否从 baseline 升级为“官方复刻完成”时，必须同时满足： + +- 有 official golden recording 或官方 non-recording fixture 证据。 +- OCU candidate 可以通过可复跑脚本与该证据比较，而不是只靠人工目测。 +- 文档里已经删除或更新对应的“待校准”标记。 + +没有 official golden 时，默认措辞保持为“OCU baseline”“官方风格”“待 official golden 校准”。 + +## 易混边界 + +- Codex.app 边界：当前可观察证据只支持把 Codex.app 视为宿主、feature gate 和通用 MCP elicitation UI 的提供方；录制控制条、事件文件、AX payload、截图上下文和 `event_stream_*` tool 字符串都落在 Computer Use runtime 侧。OCU 因此必须自己承担开始确认、录制中 bar、Done / Discard 和结束唤醒。 +- 官方兼容层边界：`open-computer-use event-stream mcp` 要尽量像官方 `SkyComputerUseClient event-stream mcp`，但只暴露三件套 no-arg tools。任何独立集成所需的 `wait`、callback、session index、validate 或 skill scaffold 都走 CLI / runtime 扩展。 +- Screenshot 口径：官方字符串确认 `screenshotNeededForContext`，但没有证据表明每个事件都截图。OCU 默认应最小化截图，只在上下文不足或测试显式要求时写入 `screenshots/`，并把截图失败降级为可诊断事件或 suppressed 记录。 +- Compact AX diff 口径：官方 diff marker 是 `~`、`+`、`-`，并有 previous / cumulative diff 和 fallback 线索。OCU 当前 LCS 风格 diff 是 baseline，不是最终官方等价算法；后续必须用 official golden recordings 校准字段、line budget、element identity、removed summary 和 fallback 条件。 +- Skill handoff 口径：录制结束后不能只交付 runbook。可用 recording-to-skill scaffold 生成草稿，但最终仍要由 agent 读取 `events.jsonl`、补齐业务输入、按 summary 的 `safetySignals` 和上下文确认高风险动作、按 `skill-creator` 完成可发现 skill。 +- Golden fixture 口径：已有 required `simple-action-stop` official recording fixture 时，只能把该场景覆盖到的生命周期、click、stop endReason 和 handoff shape 作为最小 official evidence；未被该 fixture 或后续 recommended fixture 覆盖的字段名、字段顺序、timeout endReason、raw event 结构和截图触发策略仍只能标成 OCU baseline 或待官方校准。 + +## 已沉淀的官方上下文 + +- 官方 `record-and-replay` 插件启动同一个 Computer Use runtime，只是参数为 `event-stream mcp`。 +- 官方 Record & Replay MCP server name 是 `Record & Replay`，protocol version 是 `2025-11-25`。 +- 官方 1.0.857 的非录制 surface fixture 已入库：`../references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json`。 +- 官方 raw start/status/stop 在宿主外当前会超时，边界 fixture 已入库：`../references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json`。`scripts/test-event-stream-probe-fixtures.py` 会确认三次 raw `tools/call` 都是 timeout、fixture 不含本机路径且没有 recording handoff paths；`make record-and-replay-baseline-smoke` 会把这些检查写入 `evidence.officialRawStartTimeout`，作为当前 usable baseline 的必需 non-recording evidence。 +- 通过 Codex-hosted Record & Replay tool 观察到 no-active `event_stream_status` / `event_stream_stop` response shape：text JSON 只有 `isRecording=false` 和 `maxDurationSeconds=1800`，fixture 已入库：`../references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json`。 +- Codex.app 侧目前只观察到 feature flag、插件可见性和通用 MCP elicitation/approval 处理；录制控制条、事件文件、AX diff、截图上下文和 `event_stream_*` 字符串在 Computer Use runtime 相关二进制里。 +- 官方字符串确认的事件名、文件名、control endReason 和 AX diff marker 已写入逆向参考文档。 + +## 当前 OCU Baseline + +当前分支已经围绕这个方向沉淀了 baseline 能力，后续推进时应先复用它们，而不是重新设计 surface： + +- `open-computer-use event-stream mcp` 暴露官方三件套。 +- 官方三件套运行时参数面已守住基本 `tools/call` contract：`params` 必须是 object，`params.name` 必须是非空 string，`arguments` 必须是 object，且 `event_stream_start/status/stop` 不接受任何字段；缺失 / 非 object params、缺失 / 非 string tool name、非 object arguments 或非空 object arguments 都通过 tool error 返回，且不创建 session 文件。 +- MCP no-active `event_stream_status` / `event_stream_stop` 已按官方观察返回 `isRecording=false` / `maxDurationSeconds=1800` 的最小 text JSON；CLI `event-stream status/stop --json` 仍保留 OCU 扩展字段。 +- CLI 扩展包含 `start/status/stop/cancel/wait/validate/summarize/scaffold-skill`。 +- session 产物包含 `events.jsonl`、`metadata.json`、`session.json`、`suppressed.jsonl`、`latest-session.json` 和 `active-session.json`。 +- 已有 OCU 自有开始确认和录制控制条 baseline。 +- 输入事件覆盖 click、context menu、drag、text input、submit、shortcut、window context、AX context、selection、terminal value、debug error 和 raw event 摘要。 +- AX payload 已有 full、previous diff 和 cumulative diff baseline。 +- `wait --session-id`、`waitTimedOut`、`waitSessionMatched` 和 `wait --notify-command` 已作为独立集成扩展。 +- summary / scaffold handoff 已有结构化 `runtimeInputs`、`safetySignals`、`summaryLimits`、完成态 `recordingIncomplete`、`sessionStartedNotFirst`、`sessionStartedCountInvalid`、`sessionEndedNotFinal`、`sessionEndedCountInvalid` 和 `blockingDiagnostics`，用于把文本输入 / selection 依赖转成运行时输入候选，把 submit、send/delete/save/upload 等可能改变外部状态的动作转成显式确认线索，在长录制摘要被截断时要求回读 `events.jsonl`，在 recording 仍 active、缺少 `session.started`、重复写入 `session.started`、`session.started` 不是首事件、缺少 `session.ended`、重复写入 `session.ended` 或 `session.ended` 后又出现事件时拒绝生成 skill 草稿，并在 Input Monitoring 不可用等阻断性诊断出现时拒绝生成 skill 草稿；默认只保留长度、目标摘要、原因、计数和敏感标记，不复制原始文本。 +- thin skill baseline 位于 `../../skills/open-computer-use-record-and-replay/SKILL.md`，通用 skill 的参考说明位于 `../../skills/open-computer-use/references/record-and-replay.md`。 +- standalone thin skill repo scaffold 入口是 `../../scripts/scaffold-record-and-replay-skill-repo.py`；生成 repo 自带 `scripts/verify-runtime.py`，用于在不启动录制的前提下验证 `open-computer-use event-stream mcp` 的官方兼容三件套 surface，包括 initialize capabilities、tool description、empty input schema、MCP annotations、no-active status/stop response shape、malformed `tools/call` contract 和被拒请求不创建 session 文件的副作用边界，也自带 `scripts/check.sh` 作为 package + runtime verifier + wait/notify contract smoke + synthetic recording-to-skill 的组合自检入口；其中生成态 package script 会校验 skill frontmatter 的 `name` 和非空 `description`，`scripts/verify-package-artifact.py` 会在打包后打开 `.skill` archive，确认 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` 的 frontmatter 和 Record & Replay handoff 片段仍存在，并输出 `checkedPackageArtifact=true` 供 source / npm staged scaffold smoke、baseline summary 和 artifact audit 消费；生成 repo 还带 SHA-pinned `.github/workflows/ci.yml` 在独立 repo 中复跑同一自检。生成 README 还固定了 recording-to-skill handoff：完整 OCU 产物用 `validate --strict-ocu --require-skill-draft `，只有 `eventsPath` 时用非 strict `validate --require-skill-draft `，并说明 events-only 可以从 `session.ended.endReason=recording_controls_cancelled` 推断取消态但不能证明 metadata/session alias 或 declared handoff paths。生成 repo 默认自检会运行 `scripts/verify-skill-workflow.py`、`scripts/wait-notify-contract-smoke.py` 和 `scripts/recording-to-skill-smoke.py`：workflow verifier 静态守住官方 handoff 与 OCU Independent Wait / Notify Integration 边界，wait/notify smoke 用不存在的 session 验证 callback skipped、不创建 session 文件和 `waitSessionMatched=false`，也用合成 completed session 验证成功 callback 能收到 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH`，失败 callback 非零退出会让 CLI 非零并返回 `reason=nonZeroExit`，callback 超时也会让 CLI 非零并返回 `reason=timeout`，recording-to-skill smoke 用临时合成 completed recording 验证安装态 strict validator、events-only validator、strict `declaredPaths` evidence 和 `scaffold-skill` 草稿生成路径，并用 cancelled recording 验证 strict / events-only validation 和 `scaffold-skill` 都会拒绝生成 skill；生成态 manifest 的 `extensionLayer` 也会机器可读地声明 `wait --notify-command`、`waitTimedOut` / `waitSessionMatched`、notify stdin/env、`callbackFailureMakesCliFail` 和 `callbackTimeoutMakesCliFail` 不属于官方 MCP surface，`mcpServer.rejectedRequestsDoNotCreateSessionFiles` 固定 malformed request 副作用边界，源码与 npm staged baseline 都会消费 `checkedPackageArtifact`、`checkedRequiresObjectParams`、`checkedRequiresStringToolName`、`checkedRequiresObjectArguments`、`checkedRejectsUnexpectedArguments`、`checkedRejectsNonObjectArguments`、`checkedRejectedRequestsDoNotCreateSessionFiles`、`checkedNotifySuppressedEventsPathEnv`、`checkedNotifyCallbackFailureExit`、`checkedNotifyCallbackFailureReason`、`checkedNotifyCallbackTimeoutFailureExit` 和 `checkedNotifyCallbackTimeoutReason`，源码 standalone baseline 还会消费 lifecycle smoke 的 one-active、idempotent stop 和 final status evidence，`recordingToSkill` 区块则机器可读地声明 strict validation、events-only validation、scaffold-skill handoff 和 cancelled recording 拒绝 contract。生成 repo 还带可选 `scripts/recording-lifecycle-smoke.py`，用于在本地显式 start/repeat start/status/stop/repeat stop/final status 一个最小 recording，验证 one-active、idempotent stop 和 completed status，再用安装态 validator 校验 session 文件；它不进入默认 `check.sh`，避免独立 repo CI 默认启动真实录制。 +- 生成态 runtime verifier 的 surface evidence 现在拆成三项机器字段：`checkedInitializeSurfaceContract` 固定 protocol / server name / capabilities / 无 `instructions`，`checkedToolMetadataContract` 固定三件套 tool name、description、input schema 和 annotations，`checkedToolInputSchemaNoArguments` 单独固定空 object schema 与 `additionalProperties=false`。source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费这三项字段。 +- 生成态 `scripts/verify-runtime.py` 和可选 `scripts/recording-lifecycle-smoke.py` 的 MCP response timeout 诊断必须带 runtime 路径、执行命令、`open-computer-use --version` 结果和设置 `OPEN_COMPUTER_USE_CLI` 指向当前 runtime 的修复提示；`OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS` 可用于本地快速复现旧全局安装或错误 launcher 导致的超时。源码 scaffold smoke 用假旧 runtime 覆盖该失败路径，npm staged smoke 也静态验证生成物包含这些诊断文本。 +- baseline summary 和 artifact audit 会把 source / npm staged 的 `checkedRuntimeTimeoutDiagnostics` 当作 `usableBaseline` 必需 evidence；缺失时 `missingUsableBaselineEvidence` 会分别报告 `standaloneSkillRepo.checkedRuntimeTimeoutDiagnostics` 和 `npmStagedSkillRepo.checkedRuntimeTimeoutDiagnostics`，避免 release / standalone audit 只验证 runtime surface，却漏掉旧 runtime 排障路径。 +- npm staged scaffold 还必须证明安装态 launcher 的 Python 依赖失败路径可诊断：`PYTHON` 指向不存在的文件或 Python 2 时应返回 command-specific `Python 3 is required to run` 和 `PYTHON=/path/to/python3` 修复提示。`checkedNpmPythonLauncherDiagnostics` 已进入 baseline summary、runner 早期断言和 artifact audit；缺失时 `missingUsableBaselineEvidence` 会报告 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics`。 +- 生成 repo README 必须把 Python 3 前置条件写清楚：自检脚本要求 `python3` 可用，通过 npm launcher 创建 repo 时如果 shell 找不到 Python 3，应设置 `PYTHON=/path/to/python3`。源码 standalone smoke 和 npm staged smoke 都输出 `checkedGeneratedReadmePrerequisites=true`，baseline summary 与 artifact audit 会在 source / npm staged 两侧消费该 evidence，缺失时报告 `standaloneSkillRepo.checkedGeneratedReadmePrerequisites` 或 `npmStagedSkillRepo.checkedGeneratedReadmePrerequisites`。 +- npm package build 必须携带 source baseline summary artifact：`scripts/npm/build-packages.mjs` 打包 `open-computer-use` 时会复制 `dist/record-and-replay-baseline-summary.json`，供安装态 scaffold 生成 `evidence/source-baseline-summary.json`。缺失该 artifact 时 build 直接以 `Missing Record & Replay baseline summary artifact` 失败，并要求先运行 `make record-and-replay-baseline-audit`；npm staged smoke 会临时移走该 artifact 覆盖负例，避免发布出缺 baseline evidence 的 npm 包。 + +## Standalone Repo Contract + +后续真正拆出独立 repo 时，以 scaffold 生成物的 `record-and-replay-skill-repo.json` 作为机器可读 contract： + +- `mcpServer` 只声明 `open-computer-use event-stream mcp`、initialize capabilities、`event_stream_start/status/stop` 三件套、tool metadata、object params contract、非空 string tool name contract、empty object argument contract 和 no-active response shape。 +- `officialEvidence` 声明该 standalone repo contract 目前基于官方 `record-and-replay/1.0.857` 的 non-recording surface fixture、Codex-hosted no-active status/stop fixture 和 hostless raw start/status/stop timeout 边界 fixture；它同时显式标记还没有 successful recording golden，并把 required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景写成机器可读清单。`officialEvidence.scenarioRecipes` 还会随 scaffold 固化每个场景的 capture goal、预期 action event、预期 endReason、evidence 要求和 OCU candidate 来源，避免独立 repo 把 no-active 或 timeout 边界证据误读成事件 schema 已官方等价，也避免后续只拿到场景名却丢失采样动作细节。`officialEvidence.sourceRepoBaselineChecks` 声明源仓 baseline 负责验证 baseline contract smoke、hostless raw timeout、official fixture set gate same-scenario compare policy、official capture packet preflight 和 OCU candidate pairing preflight；其中 `officialFixtureSetGate.sameScenarioComparePolicy` 机器声明 AX diff evidence / marker 与 suppressed event sequence / schema 都必须进入校准 gate，`officialGoldenCapturePreflight.requiredEvidence` 机器声明 `checkedOfficialCapturePacketInputSemanticGuard`、`checkedOfficialCapturePacketSetContractManifest`、`checkedOfficialCapturePacketPostCaptureWorkflow`、`checkedOfficialCapturePacketSetPostCaptureWorkflow`、`checkedOfficialCapturePacketStrictAuditHandoff` 和 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff` 必须由源仓 baseline 证明。`officialEvidence.sourceRepoBaselineAudit` 声明 release / standalone handoff 应在源仓运行 `make record-and-replay-baseline-audit`，官方 successful recording fixtures 入库后再运行 `make record-and-replay-official-golden-gate-audit`，并用 `make record-and-replay-baseline-audit-targets-smoke` 守住两个 audit target 的 dry-run 合同；默认 baseline 摘要产物是 `dist/record-and-replay-baseline-summary.json`，默认 strict 摘要产物是 `dist/record-and-replay-official-golden-gate-summary.json`，覆盖变量分别是 `RNR_BASELINE_SUMMARY_JSON` 和 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`，manifest 同时声明 strict 缺 official golden 时可运行的 `strictOfficialGoldenExpectedFailureAudit` 命令、`verifiesSummaryArtifactSeparation=true` 与 `verifiesSummaryEnvVarIsolation=true`；`officialEvidence.standaloneRepoBoundary` 明确独立 repo 默认自检不启动官方录制、不复制 preflight 脚本，也不复制 OCU runtime 源码。 +- `extensionLayer` 只声明 OCU 独立集成命令，例如 `wait --notify-command`、`waitTimedOut` / `waitSessionMatched`、notify stdin 和 `OPEN_COMPUTER_USE_EVENT_STREAM_*` 环境变量（包含 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH`）；它必须显式标记 `officialCompatibleMcpSurface=false`，避免独立 repo 把 listener/callback 误加到 MCP 三件套里。 +- `checks` 是默认发布/安装前必须跑的自检：`scripts/package-skill.sh`、`scripts/verify-manifest.py`、`scripts/verify-source-baseline-summary.py`、`scripts/verify-runtime.py`、`scripts/verify-skill-workflow.py`、`scripts/wait-notify-contract-smoke.py`、`scripts/recording-to-skill-smoke.py` 和组合入口 `scripts/check.sh`。这些检查都不应启动真实桌面录制；`verify-manifest.py` 只校验生成态 `record-and-replay-skill-repo.json` 的机器合同，`verify-source-baseline-summary.py` 只审计随 scaffold 复制的 `evidence/source-baseline-summary.json` 投影证据，`verify-runtime.py` 可以调用 no-active status/stop，因为这不会创建 session，`wait-notify-contract-smoke.py` 可以等待不存在的 session，因为这只验证 unmatched session / callback skipped 语义。 +- `optionalChecks` 只放需要本地桌面权限或会真实创建 recording session 的检查；当前只有 `scripts/recording-lifecycle-smoke.py`。 +- 默认 CI 只调用 `scripts/check.sh`，用于验证 skill frontmatter、runtime contract 和 recording-to-skill handoff。真实录制 lifecycle smoke 由开发者在本机或专门授权环境显式运行。 +- 生成 repo 不复制 OCU runtime 源码，也不假设 Codex.app 私有 UI；它只依赖已安装的 `open-computer-use` binary 和 OCU 自有控制条 / wait / validation / scaffold 能力。 + +补充：source standalone 和 npm staged baseline 不只消费 `recording-to-skill` handoff 总开关。`scripts/run-record-and-replay-baseline-smoke.sh` 和最终 summary 现在还要求 `checkedStrictValidation`、`checkedEventsOnlyValidation`、`checkedScaffoldSkill` 和 `checkedSkillCreatorHandoff` 这类具体 evidence，分别证明完整 OCU session strict gate、events-only gate、`scaffold-skill` 正路径和后续 `skill-creator` 交接文案都没有从独立 repo scaffold 中丢失；npm staged 路径同样证明这些证据能从安装态 launcher 生成的 repo 默认自检中传播出来。 + +## 后续最短路径 + +如果下一轮目标是继续实现官方复刻，先走这条路径： + +1. 用正常 Codex 宿主流程采集官方 successful recording，优先 stop / cancel / repeat start / no-active status-stop / 简单动作样本。 +2. 先用 `scripts/prepare-record-and-replay-official-golden-capture.py --scenario ` 做只读 preflight，确认当前缺口、`scenarioRecipe` 和下一条命令；需要交接采集输入时，可加 `--capture-packet-dir ` 生成包含 recipe、placeholder hosted JSON 和 verify/inspect/import/check wrappers 的 capture packet；需要一次准备全部推荐场景时，再加 `--capture-packet-recommended-scenarios` 生成 per-scenario 子目录、根级 manifest 和 `verify-all.sh` / `inspect-all.sh` / `import-all.sh` / `check-all.sh` / `ingest-ocu-candidates.sh` 批量 wrapper。单场景 `capturePacket` 与根级 `capturePacketSet` 都会用 `includeTranscript` / `requiresMcpTranscriptInput` 声明当前采集包是否要求 `inputs/mcp-transcript.json`，消费方不应靠猜文件是否存在或解析 README 判断。替换 packet 输入后先运行 `verify-inputs.sh` 或 `verify-all.sh`，再用 `scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario --inspect-only` 无写入检查 hosted JSON 是否能解析出 handoff path、recording input 和 MCP transcript 证据；检查通过后去掉 `--inspect-only` 正式脱敏导入并跑 readiness。recommended packet set 导入后还应运行根级 `ingest-ocu-candidates.sh`，批量生成 simple / drag 等可自动采样场景的 OCU candidate，并跳过 keyboard / cancel / timeout 这类 recording-required 或 no-action 场景。导入后的 `fixture-manifest.json` 必须保留 `scenario` 与 `scenarioRecipe`，fixture set gate 会拒绝 recipe drift；official / OCU direct ingest、coverage report 和 fixture set gate 的 scenario-specific readiness 参数也都从同一个 scenario catalog 生成。也可用 `--status-json -` 从 stdin 直接读取刚复制出来的 hosted JSON；如果 stdin 或转存 JSON 里的 handoff path 是相对路径，加 `--status-json-base-dir ` 固定解析基准；如果已有 raw MCP transcript，再加 `--mcp-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage`，或在 status JSON 已落文件时用 `--mcp-transcript -` 从 stdin 读取 transcript；如果同一 status JSON 已含 response-shape evidence，可用 `--use-status-json-as-transcript` 复用。采集 required 场景后可加 `--require-coverage`，让导入后 official scenario 覆盖或 required readiness 不足时直接返回非零。 +3. 用 `scripts/ingest-ocu-record-and-replay-candidate.py --recording --name --scenario --official-root --check-fixture-set`、`--smoke-json ` 或 `--run-action-smoke --scenario simple-action-stop|drag-stop` 导入 OCU candidate，再用 fixture set gate 对比 lifecycle、metadata 稳定字段、event sequence、replay-critical semantic fields、AX diff evidence / marker 和 suppressed stream sequence / schema。`keyboard-input-stop` 目前走已有 recording / smoke JSON 导入，不走 synthetic keyboard `--run-action-smoke`。 +4. 根据差异校准官方兼容层，保持 `event_stream_start/status/stop` 三件套无参数。 +5. 只有当独立运行体验受阻时才改 OCU 扩展层，例如控制条、`wait`、notify、validation 或 standalone repo scaffold。 + +如果下一轮目标是拆独立 repo，先走这条路径： + +1. 运行 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir ` 生成 repo。 +2. 在生成 repo 中运行 `./scripts/check.sh`，确认 package、runtime contract、skill workflow 和 synthetic recording-to-skill smoke 通过。 +3. 在有桌面权限的本机显式运行 `./scripts/recording-lifecycle-smoke.py`，确认安装态 runtime 可以真实 start / stop 最小 recording。 +4. 根据真实发布目标补 repo README、版本、安装说明和 release 流程，但不要把 OCU runtime 源码复制进独立 repo。 + +## 待解决问题 + +- 已有 required `simple-action-stop` official successful recording fixture，且当前 readiness / fixture set gate 可验证该最小门槛。当前也已有 official non-recording surface fixture、Codex-hosted no-active status/stop response fixture 和 raw start/status/stop timeout 边界 fixture；baseline summary 会分别消费 `evidence.officialSurfaceCompare`、`evidence.officialNoActiveResponse` 和 `evidence.officialRawStartTimeout`。`scripts/check-event-stream-official-fixture-coverage.py` 会单独报告 required official scenario 覆盖和 readiness；该 report 也会单独输出 recommended scenario 覆盖，默认清单是 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`。recommended 只指引后续 official golden 采样，不影响 required 覆盖的退出码。 +- 官方算法级 AX compact diff 仍未校准。 +- 鼠标、键盘、drag、terminal、selection、window、debug、secure input 和 raw event 字段仍需官方 recording 校准。 +- 截图触发阈值、字段名和 suppressed 行为仍需官方 recording 校准。 +- MCP elicitation 的 host schema baseline 已有，但官方 Record & Replay 真实业务 message / `_meta` 仍未确认。 +- timeout endReason 当前是 OCU baseline，未确认官方实际输出。 +- 独立 thin plugin / skill repo 尚未真正创建和发布验证；当前只有 scaffold 生成态 repo 的本地打包/frontmatter gate、完整 initialize / tool metadata / no-active response runtime contract verifier、可选最小 recording lifecycle smoke、组合 `check.sh` 和生成态 CI workflow smoke。 + +## 下一轮推进顺序 + +1. 补齐 recommended 官方 golden recordings。 + Required `simple-action-stop` 已入库；后续继续使用正常 Codex 宿主流程录制 keyboard、drag、cancel、timeout 等推荐样本。样本必须脱敏后再入库。 + +2. 导入和检查官方样本。 + 优先使用 `scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario --inspect-only` 检查 hosted official recording；确认 `stage=inspect`、`recordingInputInspection.*.exists=true` 和需要的 MCP transcript evidence 后，再去掉 `--inspect-only` 正式导入。也可以把 hosted JSON pipe 给 `--status-json -`。如果 hosted JSON 中的 `metadataPath` / `sessionPath` / `eventsPath` 是相对路径，传 `--status-json-base-dir `，避免按当前工作目录误解析。正式导入会调用 `scripts/import-event-stream-fixture.py` 脱敏入库，把 `scenario` / `scenarioRecipe` 写进 manifest，并按 scenario 跑 `scripts/check-event-stream-golden-readiness.py`。如果只有 session 目录、`metadata.json` 或 `session.json`，可用 `--recording `；如果另有 raw MCP transcript，则加 `--mcp-transcript --require-mcp-transcript-evidence --check-fixture-set --check-coverage`，或者在 status JSON 已经来自文件时用 `--mcp-transcript -` 从 stdin 读取单独 transcript。`--status-json -` 和 `--mcp-transcript -` 不能同时使用；同一 status JSON 含 transcript / response-shape evidence 时，改用 `--use-status-json-as-transcript` 直接复用。导入后可用同一导入命令加 `--require-coverage`，或单独用 `scripts/check-event-stream-official-fixture-coverage.py --require-readiness` 验证 required scenario 覆盖和 readiness，再用 `scripts/check-event-stream-official-fixture-set.py --official-root ` 做集合级 readiness；后续有 OCU candidate fixture 时再加 `--candidate-root ` 做同 scenario 强对比。 + +3. 对比 OCU candidate。 + 优先用 `scripts/ingest-ocu-record-and-replay-candidate.py --recording --name --scenario ` 导入已有 OCU recording；如果 candidate 来自 `make event-stream-action-smoke`，可把 action smoke stdout 保存成 JSONL 后用 `--smoke-json `，脚本会从 `recordingsRoot` / `sessionId` 找到 session 目录，并自动消费同一 JSON 里的 `mcpTranscriptPath`。因为 action smoke 默认会清理临时目录，手动保存 stdout 给后续导入时需要同时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP=1` 或 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR=`；否则优先用 `--run-action-smoke` 让导入脚本自己跑一次 opt-in 真实输入 smoke，并保留同 session MCP transcript 到导入完成。`--run-action-smoke` 会按 `--scenario simple-action-stop|drag-stop` 触发同名真实输入采样，分别生成 click 或 drag candidate；其它非 keyboard scenario 回退到默认 mixed action smoke，只适合作为通用监听链路证据。`keyboard-input-stop` 因当前 macOS 对 synthetic keyboard event 的 session tap 过滤不稳定，先通过 `--recording` 或保留的 `--smoke-json` 导入并跑 readiness。导入时若已有其它 local MCP transcript，也可显式加 `--mcp-transcript --require-mcp-transcript-evidence`；official fixture 已入库后再加 `--official-root --check-fixture-set` 直接跑 same-scenario gate。底层仍使用 `scripts/compare-event-stream-recordings.py` 比较 event sequence、schema、metadata keys、stable metadata values、final session evidence、replay-critical semantic fields 和 AX diff evidence；官方 fixture 入库后优先打开 `--require-same-metadata-keys`、`--require-same-metadata-values`、`--require-final-session-evidence` 和 `--require-semantic-fields`，确保 metadata/session 布局键、state / active / endReason / count 这类稳定状态值、最终 `session.ended` 闭环、target AX、drag 起止点、keyboard 目标、AX treeLines 等关键证据没有从 OCU candidate 中丢失或漂移。 + +4. 校准官方兼容层。 + 优先校准 MCP response、session 文件字段、endReason、事件字段、AX payload 和 screenshot context。保持官方三件套无参数。 + +5. 强化 OCU 扩展层。 + 在不污染官方 surface 的前提下，继续打磨控制条、wait/notify、取消路径、权限错误和 standalone thin skill repo。 + +6. 拆真实 standalone repo。 + 以 `scripts/scaffold-record-and-replay-skill-repo.py` 的生成物为蓝本创建独立 repo，只声明 runtime contract 和 thin skill；发布前至少跑 `scripts/check.sh`，本地具备桌面权限时再跑可选 `scripts/recording-lifecycle-smoke.py`。 + +## 常用验证入口 + +- `make event-stream-surface-smoke` +- `make event-stream-no-active-smoke` +- `make event-stream-smoke` +- `make event-stream-smoke-matrix` +- `make event-stream-action-smoke` +- `make event-stream-skill-scaffold-smoke` +- `make record-and-replay-skill-repo-smoke` +- `make codex-record-and-replay-installer-smoke` +- `make event-stream-fixture-smoke` +- `make event-stream-official-fixture-ingest-smoke` +- `make event-stream-official-fixture-coverage-smoke` +- `make event-stream-ocu-candidate-ingest-smoke` +- `make event-stream-golden-readiness-smoke` +- `make event-stream-official-fixture-set-smoke` +- `make event-stream-compare-smoke` +- `make event-stream-probe-fixture-smoke` +- `make record-and-replay-baseline-smoke` / `scripts/run-record-and-replay-baseline-smoke.sh`：本机 opt-in baseline 候选验证，串起 baseline contract smoke、默认 matrix、截图上下文 smoke、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、official fixture set gate smoke、official fixture coverage report、hosted official fixture ingest smoke、OCU candidate ingest smoke、official golden capture preflight smoke、OCU candidate pairing preflight smoke、standalone repo scaffold smoke 和 npm staging scaffold smoke；脚本会先把 `scripts/test-record-and-replay-baseline-contract.py` 输出写入 `evidence.baselineContract`，证明 shared `REQUIRED_BASELINE_CHECKS`、standalone / npm staged smoke required keys、summary evidence keys 和 standalone lifecycle summary rename mapping 已在同一轮 baseline 中自检；脚本会解析 matrix / screenshot / action smoke 的 JSON 输出，在最终摘要中写入脱敏的 `evidence.eventStreamMatrix`、`evidence.screenshotContextSmoke` 和 `evidence.realInputActionSmoke`，证明 lifecycle / no-active / timeout / wait / approval / elicitation / app-agent wait-notify 路径、`OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 下的 `screenshotNeededForContext` 标记、当前环境可截图时的 `screenshotPath` 校验，以及真实 CGEvent 输入录制到 skill 草稿链路都跑过；同一 baseline 还会显式跑 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop` 的 OCU action candidate smoke，并要求 `evidence.realInputActionSmoke.checkedSimpleActionStopCandidate=true` 与 `checkedDragStopCandidate=true`，证明当前 OCU 至少能生成 click / drag 两类同场景 candidate 证据；脚本会解析官方 surface compare 的 JSON，要求 local / official 两个 label 都匹配官方 fixture，并在最终摘要中写入脱敏的 `evidence.officialSurfaceCompare`；脚本也会解析 official fixture set gate smoke 和 official fixture coverage report 的 JSON，最终摘要中的 `evidence.officialFixtureSetGate` 会证明集合级 gate、`simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` scenario policy、candidate readiness 负例和 candidate pairing 自检都跑过，同时暴露仓库真实 official recording fixture required / recommended scenario 覆盖；required `simple-action-stop` 已入库且 ready 时会输出 required coverage/readiness 通过，推荐清单未齐时仍会输出 `repoHasRecommendedOfficialScenarioCoverage=false` / `missingRepoRecommendedOfficialScenarios=[...]`；脚本会解析 hosted official fixture ingest 与 OCU candidate ingest 的 JSON 摘要，并在 `evidence.fixtureIngestPipelines` 中要求 official ingest、`sessionDirectoryPath` handoff、candidate ingest、smoke JSON import、official/candidate pairing、candidate redaction、keyboard readiness 和 drag readiness 都为 true；脚本会解析 official capture / OCU pairing preflight smoke 的 JSON 摘要，并在 `evidence.preflightPipelines` 中要求 capture packet Make targets、capture packet、capture packet 后续 handoff scripts、recommended packet set、缺样本/缺插件/coverage error、keyboard recording-required、缺 candidate、paired candidate 和 compare 成功路径都被覆盖；也会解析 standalone / npm 两个 scaffold smoke 的 JSON 摘要，要求 runtime contract、wait/notify、recording-to-skill handoff、declared handoff paths、scaffold failure exit、cancelled recording guard、lifecycle smoke、安装态 manifest contract、官方 evidence 场景清单、官方 evidence source repo audit 命令、安装态生成 repo 默认 `check.sh` evidence、`skill-creator` handoff evidence 和官方 handoff guard evidence 都为 true，再输出带 `checkedOfficialEvidenceScenarioManifest` / `checkedOfficialEvidenceAuditManifest` 的 `evidence.standaloneSkillRepo` / `evidence.npmStagedSkillRepo` 机器可读 JSON 摘要；依赖桌面输入权限、官方 bundled cache 和当前 `dist/` artifacts，不进入默认 CI。 +- `make record-and-replay-baseline-audit`：等价于 baseline smoke 加 `--summary-json`，默认把最终 summary 落到 `dist/record-and-replay-baseline-summary.json`；可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖。它不改变 baseline 退出语义，只是把 release / standalone audit 证据落盘。 +- `make record-and-replay-official-golden-capture-preflight`：实际运行 `scripts/prepare-record-and-replay-official-golden-capture.py`，查看当前缺口和下一条 inspect/import 命令。 +- `make record-and-replay-official-golden-capture-packet`：生成单个 official capture packet；默认 `RNR_SCENARIO=simple-action-stop`，默认写到系统临时目录，可用 `RNR_PACKET_DIR=` 覆盖。该入口不启动官方录制，只写 scenario recipe、`capture-contract.json`、hosted status/transcript JSON placeholder 和 verify/inspect/import/check wrapper；`verify-inputs.sh` 会先检查输入存在、JSON 可解析且不再是 `_placeholder=true`，再按 `capture-contract.json` 检查 status JSON 至少包含 `eventsPath`，并且包含 `metadataPath` 或 `sessionPath` 这类 handoff 文件 evidence；`suppressedEventsPath` / `sessionDirectoryPath` 只作为 optional evidence 记录，避免官方最小包被误拒。transcript enabled 时还要检查 MCP transcript evidence。`capture-contract.json` 和返回 JSON 还会写 `postCaptureWorkflow`，把录制后的替换输入、verify、inspect、import、coverage / fixture-set gate、可选 OCU candidate ingest、strict audit 和 strict expected-failure audit 排成机器可读步骤。消费 hosted JSON 的 `inspect-only.sh` / `import-fixture.sh` wrapper 也会先跑同一语义校验，并拒绝仍带 `_placeholder=true` 的输入文件。单场景 packet 还会生成并在 README / JSON 中暴露 `check-fixture-set.sh`、`strict-golden-gate.sh`、`strict-expected-failure-audit.sh` 和可用时的 `ingest-ocu-candidate.sh`，用于 official fixture 导入后的 fixture-set gate、strict audit artifact 刷新、required official golden 缺口的 expected-failure audit 和 same-scenario OCU candidate 导入；`strict-golden-gate.sh` 实际运行 `make record-and-replay-official-golden-gate-audit`，`strict-expected-failure-audit.sh` 实际运行 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing`，并由 baseline summary / artifact audit 消费 `checkedOfficialCapturePacketStrictAuditHandoff`、`checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff` 和 `checkedOfficialCapturePacketPostCaptureWorkflow`。离线生成时可用 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`，测试路径可用 `RNR_FIXTURE_ROOT` / `RNR_OFFICIAL_PLUGIN_ROOT` 覆盖。 +- `make record-and-replay-official-golden-capture-packet-set`：生成 required + recommended capture packet set，包含 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel` 和 `timeout` 子目录以及根级 `verify-all.sh` / `inspect-all.sh` / `import-all.sh` / `check-all.sh` / `ingest-ocu-candidates.sh`；同样不启动官方录制，并支持同一组 `RNR_*` 覆盖变量。 +- capture packet set 的根级 `verify-all.sh` 会逐个运行子 packet 输入校验和 `capture-contract.json` 语义校验；根级 `capture-packets.json` 会内联 `captureContracts` / `captureContractPaths` / `postCaptureWorkflow`，把每个 scenario 的 expected action event、expected `endReason`、handoff path evidence、transcript requirement 和录制后步骤顺序留在集合级 manifest，方便后续独立 repo 或 skill 直接消费；根级 `inspect-all.sh` / `import-all.sh` 会在执行任何子 packet wrapper 前，先全量检查所有场景的 hosted status response 与 MCP transcript 输入是否仍带 `_placeholder=true`，再由子 wrapper 执行 handoff / transcript semantic guard；根级 `ingest-ocu-candidates.sh` 会批量运行存在 `ingest-ocu-candidate.sh` 的子场景，并明确跳过 keyboard / cancel / timeout 这类不生成 synthetic candidate wrapper 的场景。如果 packet set 用 `--no-include-transcript` 生成，根级 wrapper 只检查 status response，不会要求不存在的 transcript 输入。official capture preflight smoke 和 baseline summary 都要求单场景 verify wrapper、单场景 inspect/import wrapper、批量 verify wrapper、批量 OCU candidate handoff wrapper、批量 contract manifest、批量 post-capture workflow、批量 wrapper 的占位拒绝行为、单场景 no-transcript packet、no-transcript packet set 行为，以及这层根级 preflight-all 行为，避免批量导入时前几个场景已写入、后续场景才因为占位 JSON 失败的半批次状态；baseline summary 还消费 `checkedOfficialCapturePacketInputSemanticGuard`、`checkedOfficialCapturePacketSetContractManifest`、`checkedOfficialCapturePacketPostCaptureWorkflow` 和 `checkedOfficialCapturePacketSetPostCaptureWorkflow`，防止 capture packet 退回只检查 JSON 形状、只保留子目录合同，或丢掉录制后的机器可读 handoff 顺序。 +- `make record-and-replay-official-golden-capture-preflight-smoke`：快测 official golden capture preflight 的 JSON 输出合同。 +- `make record-and-replay-official-golden-fixture-gate`：只跑仓库 official successful recording fixture 覆盖与 required readiness；比完整 baseline strict gate 更快，适合导入官方样本后立即验证 fixture 是否合格。 +- `make record-and-replay-official-golden-gate-audit`：等价于 strict official golden gate 加 `--summary-json`，默认写 `dist/record-and-replay-official-golden-gate-summary.json`，可用 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=` 覆盖；用于 required official successful recording 入库后的 release gate 留证。缺 required 样本时仍应非零退出，同时写出失败摘要，但默认不会覆盖 `dist/record-and-replay-baseline-summary.json`。 +- `make record-and-replay-ocu-candidate-pairing-preflight`：实际运行 `scripts/prepare-record-and-replay-ocu-candidate-pairing.py`，在 official fixture ready 后查看 same-scenario OCU candidate ingest/compare 下一步。 +- `make record-and-replay-ocu-candidate-pairing-preflight-smoke`：快测 OCU candidate pairing preflight 的 JSON 输出合同。 + +补充：`scripts/run-record-and-replay-baseline-smoke.sh` 现在也会显式消费 `scripts/compare-event-stream-no-active.py` 的 JSON 输出，并在最终摘要写入 `evidence.officialNoActiveResponse`。该 evidence 固定 Codex-hosted no-active `event_stream_status` / `event_stream_stop` 的 `isRecording=false` / `maxDurationSeconds=1800` 最小 text JSON，并确认本地对比不会创建 session 文件；它只证明 no-active response shape，不代表 official successful recording fixture 已入库。 + +补充：baseline 最终摘要的顶层 `status` 会直接区分 `usableBaseline`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady`、`officialGoldenGatePassed` 和默认模式下的 `officialGoldenRequirementSatisfied`，并输出 `requiresOfficialGoldenCapture`、`missingRequiredOfficialSuccessfulRecordingScenarios`、`notReadyRequiredOfficialSuccessfulRecordingScenarios` 与 `missingRecommendedOfficialSuccessfulRecordingScenarios`。`standaloneRepoBaselineReady` 表示源码与 npm staged standalone scaffold 的默认 contract 已可用；`officialSuccessfulRecordingEquivalenceReady` 只有在 usable baseline 和 required official successful recording golden 同时成立时才会为 true。`officialSuccessfulRecordingGoldenComplete` / `officialGoldenGatePassed` 只有在 required official scenario 覆盖和 required fixture readiness 都通过时才会为 true；默认模式允许缺 official golden，所以 `officialGoldenRequirementSatisfied=true` 只表示当前没有强制 golden gate。只有 scenario manifest 存在但 readiness 不合格时仍会失败，并通过 `notReadyRequiredOfficialSuccessfulRecordingScenarios` 指出需要修复或重新采集的官方场景。`usableBaseline` 由各 smoke / scaffold evidence 推导；若必需 evidence 缺失，摘要会输出 `missingUsableBaselineEvidence` 并返回非零。摘要还会输出机器可读 `nextActions`，把修 baseline evidence、采 required official golden、准备 recommended capture packet set 和 scaffold standalone repo 的下一步命令直接列出。这让后续发版或拆 repo 时不用从 `evidence.officialFixtureSetGate` 里间接推断当前缺口，也不会在非 golden 的其它 baseline 证据不完整时误报可用。 + +补充：`realInputActionSmoke` 不再只用 `skillPath` / `mcpTranscriptPath` 间接证明真实输入录制到 skill 草稿链路。action smoke 的最终 JSON 还会输出 `checkedMcpResponseShapesCaptured`、`checkedSkillReadinessCanCreateDraft`、`checkedSkillCreatorFinalizationHandoff` 和 `checkedGeneratedSkillPathRedaction`，baseline summary 会把它们都纳入 `usableBaseline` 必需 evidence，确认同一份真实录制同时覆盖三件套 response shape、skill readiness、`skill-creator` 收尾文案和路径脱敏。完整 baseline audit 还会额外运行 `simple-action-stop` 与 `drag-stop` action candidate smoke，并把 `checkedSimpleActionStopCandidate` / `checkedDragStopCandidate` 纳入 `usableBaseline` 必需 evidence；这证明 OCU candidate 采样链路已经能覆盖 click 与 drag，但 official successful recording equivalence 仍必须等待官方 `simple-action-stop` golden fixture 入库后再判定。 + +补充：`usableBaseline` 对 source standalone 与 npm staged recording-to-skill 证据的最低要求包括 strict validation、events-only validation、scaffold-skill 正路径和 skill-creator handoff。缺任一项时，最终摘要会在 `missingUsableBaselineEvidence` 中报告对应的 `standaloneSkillRepo.*` 或 `npmStagedSkillRepo.*` concrete evidence 字段。 + +补充：baseline 摘要里的 official capture `nextActions` 不再要求调用方手拼 preflight 参数。缺 required official golden 时会先给出 `make record-and-replay-official-golden-capture-packet RNR_SCENARIO= RNR_PACKET_DIR=`，再给出 packet 内 `verify-inputs.sh`、`inspect-only.sh` / `import-fixture.sh` 和 `make record-and-replay-official-golden-gate-audit`；recommended 场景未齐时会给出 `make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR=` 以及 `verify-all.sh`、`inspect-all.sh` / `import-all.sh` / `ingest-ocu-candidates.sh`。 + +补充：需要把 baseline 最终摘要作为 release / standalone audit evidence 留存时，运行 `scripts/run-record-and-replay-baseline-smoke.sh --summary-json `。脚本仍会把同一份机器可读 JSON 打到 stdout，并保持原有退出码；生成 summary 后会立即用 `scripts/check-record-and-replay-baseline-summary.py` 审计同一份摘要，audit 输出走 stderr，不改变 stdout / 文件中的最终 summary 合同；严格 official golden 模式失败时，也会在生成最终摘要后返回非零，方便调用方同时拿到失败证据和下一步 `nextActions`。落盘后仍可用 `scripts/check-record-and-replay-baseline-summary.py ` 重新做默认审计：要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致，顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查，且不能带未知或重复 check；缺失、未知和重复项会分别在 audit JSON 的 `declaredChecks.missingRequired`、`declaredChecks.unknown` 和 `declaredChecks.duplicates` 中列出。`scripts/record_and_replay_baseline_contract.py` 是顶层 `checks` 的唯一源码，builder 负责按该 tuple 顺序输出，audit 负责按同一合同复审。审计也要求 usable baseline、standalone repo baseline、官方 non-recording / raw timeout boundary 成立，并直接检查 baseline contract、event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence，也会检查派生状态不变量：`requiresOfficialGoldenCapture` 必须和 official golden 是否完成一致，`officialSuccessfulRecordingEquivalenceReady` 必须等于 usable baseline + official golden complete，`standaloneRepoBaselineReady` 不能在 `usableBaseline=false` 时为 true，golden complete 不能同时带 required gaps / coverage errors；同时会检查 required / recommended official capture 与 standalone repo `nextActions` 是否仍包含 packet Make、verify、inspect、import、`make record-and-replay-official-golden-gate-audit`、recommended 根级 `ingest-ocu-candidates.sh`、source `record-and-replay-baseline-audit`、generated `check.sh` 和 lifecycle smoke 命令。默认模式允许缺 official successful recording golden，同时要求 `officialSuccessfulRecordingEquivalenceReady=false`。如果要做 official golden release gate，改用 `scripts/check-record-and-replay-baseline-summary.py --require-official-golden`，它会要求 golden gate、required scenario readiness 和 equivalence ready 全部成立。如果要审计 `make record-and-replay-official-golden-gate-audit` 在缺 required official fixture 时写出的预期失败摘要，使用 `scripts/check-record-and-replay-baseline-summary.py --allow-strict-official-golden-missing`；该模式要求 strict summary 失败只由 required official scenario missing / not-ready 解释，且 baseline/standalone/官方 non-recording/raw timeout evidence 仍完整、无 coverage errors。默认 CI 里的 runner 级 stub smoke 会验证落盘 JSON 与 stdout 最终摘要一致、runner 确实调用 summary audit，并验证严格模式缺 official golden 时仍保留非零退出码；summary audit smoke 会验证 builder 产出的顶层 `checks` 与 shared contract 顺序完全一致，并覆盖默认 / 严格审计口径、预期 strict missing 审计口径、顶层 summary / checks 不变量、unknown / duplicate checks 负例、baseline contract evidence 负例、派生状态不变量、官方边界 evidence 负例、preflight evidence 负例、usable baseline direct evidence 负例、nextActions command 负例和 audit target evidence 负例。 + +补充：为了让 release / standalone audit 不需要手写 runner 参数，Makefile 还提供 `make record-and-replay-baseline-audit` 和 `make record-and-replay-official-golden-gate-audit`。默认 baseline audit 使用 `RNR_BASELINE_SUMMARY_JSON` 指定落盘路径，默认是 `dist/record-and-replay-baseline-summary.json`；strict official golden audit 使用 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 指定落盘路径，默认是 `dist/record-and-replay-official-golden-gate-summary.json`。前者保持默认 baseline 缺 official golden 仍可通过的语义，后者保持 strict official golden 缺 required 样本时非零退出的语义，并且默认不会覆盖可用 baseline artifact。 + +补充：`make record-and-replay-baseline-audit-targets-smoke` / `scripts/test-record-and-replay-baseline-audit-make-targets.py` 用 `make -n` 检查上述两个 audit target 的命令展开，确认 baseline 默认/自定义 summary 路径、strict 默认/自定义 summary 路径、strict 与 baseline 默认 artifact 分离、baseline/strict summary 覆盖变量互不污染，以及 strict audit target 的 `--require-official-golden` 都未漂移。该 smoke 不启动完整 baseline 或官方录制，已接入默认 CI；完整 `scripts/run-record-and-replay-baseline-smoke.sh` 也会运行它，并在最终摘要的 `evidence.preflightPipelines` 写入 `checkedBaselineAuditMakeTargets`、`checkedBaselineAuditDefaultSummaryPath`、`checkedBaselineAuditCustomSummaryPath`、`checkedBaselineAuditIgnoresStrictSummaryVar`、`checkedBaselineAuditStrictOfficialGoldenTarget`、`checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPath`、`checkedBaselineAuditStrictOfficialGoldenCustomSummaryPath`、`checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar` 和 `checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath`。这些字段属于 `usableBaseline` 必需 evidence，避免 release / standalone audit 的短命令漂移但完整 baseline 仍误报可用。 + +补充：`scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden` 和 `make record-and-replay-official-golden-gate` 复用同一条 baseline 聚合验证，但会在 required official successful recording fixture 缺失或 required readiness 未通过时返回非零，并输出 `officialGoldenRequirementSatisfied=false`。当前没有 `simple-action-stop` 官方 successful recording 入库时，该严格 gate 预期失败；它用于后续 official golden 入库后的 release / 拆 repo 验收，不改变默认 `make record-and-replay-baseline-smoke` 的“baseline 可用但 golden 未完成”语义。 + +## 下轮读文档顺序 + +1. `record-and-replay-handoff.md` +2. `record-and-replay-replication.md` +3. `record-and-replay-official-golden-capture.md` +4. `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md` +5. `../exec-plans/active/20260626-record-and-replay-event-stream.md` +6. `../../skills/open-computer-use-record-and-replay/SKILL.md` +7. `../../skills/open-computer-use/references/record-and-replay.md` diff --git a/docs/design-docs/record-and-replay-official-golden-capture.md b/docs/design-docs/record-and-replay-official-golden-capture.md new file mode 100644 index 00000000..1a8eca2e --- /dev/null +++ b/docs/design-docs/record-and-replay-official-golden-capture.md @@ -0,0 +1,317 @@ +# Record & Replay Official Golden Capture + +## 状态 + +本文是后续采集官方 successful recording golden fixture 的操作入口。完整方案看 `record-and-replay-replication.md`，推进总入口看 `record-and-replay-handoff.md`，官方逆向证据看 `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`。 + +当前仓库已经有官方 1.0.857 non-recording surface、Codex-hosted no-active status/stop response、宿主外 raw start timeout 边界证据，以及最小 required `simple-action-stop` official successful recording fixture。该 required 样本来自正常 Codex-hosted Record & Replay 流程，入库后通过 readiness；recommended `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 仍未齐。未被 official successful recording 样本覆盖的事件字段、AX compact diff、截图触发、timeout endReason 或 raw event 结构仍只能标为 OCU baseline，直到本页流程导入并通过 gate。 + +## 采集目标 + +先采 required，再采 recommended： + +- Required：`simple-action-stop`，一次最小点击动作，然后通过官方录制控制结束。 +- Recommended：`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`。 + +`prepare-record-and-replay-official-golden-capture.py` 和 +`prepare-record-and-replay-ocu-candidate-pairing.py` 都会在 JSON 输出里带 +`scenarioRecipe`，用于把每个 scenario 的人工采集动作、预期动作事件、 +预期 `endReason`、OCU candidate 来源方式和注意事项机器可读化。正式导入 +后的 `fixture-manifest.json` 也会写入同一份 `scenarioRecipe`，fixture set +gate 会拒绝 recipe 与 scenario 不匹配的样本。后续录官方样本时优先按这个 +字段执行，不要只依赖聊天上下文或记忆。 +同一份 scenario catalog 也生成 official / OCU direct ingest 和 fixture set +gate 的 readiness 参数，覆盖 required / recommended 清单、预期 action +event、预期 `endReason`、动作类 full AX payload、生命周期类 no-action 放行和 +需要 MCP transcript 时的 completed response-shape evidence；不要在单个导入 +脚本里另写一套 scenario policy。 +coverage report 也会在只扫描 scenario 覆盖时校验 `scenarioRecipe` drift； +因此 manifest 里只有正确 scenario 名但 recipe 与 catalog 不一致的样本不能 +作为 official golden 覆盖。 +完整 baseline 摘要还会把 coverage report 的错误原样放到 +`status.officialFixtureCoverageErrors` 和 +`evidence.officialFixtureSetGate.coverageErrors`；严格 gate 失败时如果看到 +`coverageErrors=...`，优先修复或重新导入对应 fixture，再继续 OCU candidate +对比。 +采集前 `prepare-record-and-replay-official-golden-capture.py` 会把同类错误提升到 +顶层 `coverageErrors` 和 `nextActions`；official fixture 入库后的 +`prepare-record-and-replay-ocu-candidate-pairing.py` 也会先跑 coverage report， +并通过 `officialCoverageErrors` 阻止带坏 manifest 的 official fixture 进入 +candidate pairing 判断。 + +每个 successful recording fixture 至少要能证明： + +- `event_stream_start` / active `event_stream_status` / `event_stream_stop` / final `event_stream_status` 的 hosted MCP response shape。 +- 返回的 `eventsPath` 可解析，并且至少有 `metadataPath` 或 `sessionPath` 指向官方 handoff 文件；`suppressedEventsPath` 在官方 hosted 最小包里可能不存在，导入 fixture 时会补空 `suppressed.jsonl`。 +- `events.jsonl` 以唯一的 `session.started` 开头，以唯一的 `session.ended` 结束。 +- 动作类场景包含对应动作事件和 AX payload。 +- stop / cancel 场景的 `endReason` 与 scenario policy 一致。 + +官方 bundled skill 要求 `event_stream_start` 成功后结束 turn，让用户完成录制后再回来；因此 official successful recording gate 不要求额外 `repeatStartResponseShape`。重复 start / repeat stop 仍由本地 raw probe 和 OCU lifecycle smoke 覆盖 one-active / idempotent 行为，但不是 Codex-hosted official capture 的必需证据。 + +## 采集前检查 + +在采集官方样本前，先跑只读 preflight。它不会启动录制，也不会写 fixture；输出 JSON 会报告本机官方 plugin cache、当前 official scenario 覆盖、下一条 inspect-only / import / OCU candidate 命令： + +```bash +./scripts/prepare-record-and-replay-official-golden-capture.py +``` + +如果要把后续人工采集后的导入动作落成一个可交接目录，可显式生成 capture packet： + +```bash +./scripts/prepare-record-and-replay-official-golden-capture.py \ + --capture-packet-dir /tmp/ocu-rnr-official-simple-action-stop +``` + +也可以用 Make 入口生成同一类单场景 packet；默认场景是 required +`simple-action-stop`，默认目录是系统临时目录，可用变量覆盖： + +```bash +make record-and-replay-official-golden-capture-packet +make record-and-replay-official-golden-capture-packet \ + RNR_SCENARIO=drag-stop \ + RNR_PACKET_DIR=/tmp/ocu-rnr-official-drag-stop +``` + +如果只想在没有官方 bundled plugin cache 的环境里预生成 packet，可加 +`RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`。测试或离线准备时也可以用 +`RNR_FIXTURE_ROOT=` 和 `RNR_OFFICIAL_PLUGIN_ROOT=` 覆盖扫描路径。 + +该参数仍不会启动官方录制，也不会写 fixture。它只会写入 `README.md`、`preflight.json`、`scenario-recipe.json`、`capture-contract.json`、`inputs/event_stream_stop-response.json` / `inputs/mcp-transcript.json` 占位文件，以及 `verify-inputs.sh`、`verify-workflow.sh`、`inspect-only.sh`、`import-fixture.sh`、`check-coverage.sh`、`check-fixture-set.sh`、`strict-golden-gate.sh`、`strict-expected-failure-audit.sh` 和可用时的 `ingest-ocu-candidate.sh` wrapper。`verify-workflow.sh` 不读取 hosted JSON，只检查 `postCaptureWorkflow` 里的 input / command 与生成的文件、可执行 bit、transcript 开关一致。`strict-golden-gate.sh` 保留短文件名，但实际运行 `make record-and-replay-official-golden-gate-audit`，用于在 official fixture 入库后同步刷新 strict summary artifact；`strict-expected-failure-audit.sh` 审计已落盘的 `dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing`，用于 required official golden 尚缺时证明 strict 失败是预期缺口而不是 baseline 回归。official capture preflight smoke 会输出 `checkedCapturePacketStrictAuditHandoff=true`、`checkedCapturePacketStrictExpectedFailureAuditHandoff=true`、`checkedCapturePacketPostCaptureWorkflow=true` 和 `checkedCapturePacketWorkflowVerifier=true`，并由 baseline artifact 消费为 `checkedOfficialCapturePacketStrictAuditHandoff`、`checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`、`checkedOfficialCapturePacketPostCaptureWorkflow` 与 `checkedOfficialCapturePacketWorkflowVerifier`。返回 JSON 的 `capturePacket.includeTranscript` / `capturePacket.requiresMcpTranscriptInput` 会机器可读地声明当前 packet 是否要求 `inputs/mcp-transcript.json`，并同时给出 `captureContractPath`、`statusResponseInputPath`、`mcpTranscriptInputPath`、`verifyWorkflowShell`、`checkFixtureSetShell`、`strictGoldenGateShell`、`strictExpectedFailureAuditShell`、`postCaptureWorkflow` 和可选 `ingestOcuCandidateShell`；如果单场景 packet 使用 `--no-include-transcript` 生成，`mcpTranscriptInputPath=null`，wrapper 只校验 status response。 + +`capture-contract.json` 是录制后替换输入的机器合同，记录当前 scenario、fixture name、capture goal、预期 action event、预期 `endReason`、status JSON 必须出现的 handoff path key，以及是否必须提供 MCP transcript evidence。它还写入有序 `postCaptureWorkflow`，把录完官方样本后的步骤固定为替换 hosted status JSON、按需替换 MCP transcript、`verify-inputs.sh`、`inspect-only.sh`、`import-fixture.sh`、coverage / fixture-set gate、可用时的 OCU candidate ingest、strict golden audit 和 strict expected-failure audit；后续独立 repo、skill 或人工交接不应再从 README 文案里推断顺序。录完官方样本前也可以运行 `verify-workflow.sh`，先证明 `postCaptureWorkflow` 中每个 command/input 都落到了当前 packet 的真实 wrapper 和输入文件。录完官方样本后，如果保存的是 hosted `event_stream_start`、active `event_stream_status`、`event_stream_stop` 和 final `event_stream_status` 的独立 JSON，先从仓库根目录运行 `scripts/finalize-record-and-replay-official-capture-packet.py --packet-dir --start-json --status-json --stop-json --final-status-json `；它会生成 `inputs/event_stream_stop-response.json` 和 `inputs/mcp-transcript.json`，并立即跑 `verify-workflow.sh` / `verify-inputs.sh`。也可以手工用真实 hosted JSON 替换占位文件，再运行 `verify-inputs.sh`；它现在不只确认输入存在、JSON 可解析且不再是 `_placeholder=true`，还会检查 status JSON 至少包含 `eventsPath`，并且包含 `metadataPath` 或 `sessionPath` 这类 Record & Replay handoff 文件 evidence；`suppressedEventsPath` 和 `sessionDirectoryPath` 会作为 optional handoff evidence 记录，但不再阻断官方最小包。transcript enabled 时还会检查 `inputs/mcp-transcript.json` 至少包含 `startResponseShape` / `statusResponseShape` / `stopResponseShape` / `finalStatusResponseShape` / `transcript` 等 MCP transcript evidence。`inspect-only.sh` 和 `import-fixture.sh` 在执行导入命令前也会先调用同一语义校验，避免把合法但明显不是 successful recording handoff 的 JSON 送入导入链路。随后运行 `inspect-only.sh`，确认无误后再运行 `import-fixture.sh`、`check-fixture-set.sh`、`strict-golden-gate.sh`，并在存在 `ingest-ocu-candidate.sh` 时导入 same-scenario OCU candidate；`strict-golden-gate.sh` 应刷新 `dist/record-and-replay-official-golden-gate-summary.json`，不要只跑不落盘的 strict gate。required official golden 尚缺时，先让 strict gate audit 写出失败 summary，再运行 `strict-expected-failure-audit.sh` 证明失败只由缺样本解释。这个 `ingest-ocu-candidate.sh` 会显式把 candidate 写到当前 official fixture root 下的 `ocu-candidates/`，而不是落回仓库默认 candidate 目录，确保自定义 `RNR_FIXTURE_ROOT` 时 official fixture 与 OCU candidate 仍在同一组样本根目录下配对。OCU candidate 导入是后续校准动作，不是 required official golden strict gate 的硬条件；需要比较时使用 candidate 导入输出里的 `fixtureSetGateShell` 或 `pairingPreflightShell` 显式运行强对比。消费 hosted JSON 的 wrapper 仍会先检查 status response 和 MCP transcript 输入是否仍带 `_placeholder=true`，如果任一输入仍是占位文件会直接失败并提示替换对应 input。生成的 wrapper 会写入创建 packet 时的仓库路径；如果把 packet 移到别处或要针对另一个 checkout 运行，显式设置 `REPO_ROOT=/path/to/repo`。 + +如果要一次准备 required + recommended 的全部采集包： + +```bash +./scripts/prepare-record-and-replay-official-golden-capture.py \ + --capture-packet-dir /tmp/ocu-rnr-official-golden-packets \ + --capture-packet-recommended-scenarios +``` + +等价 Make 入口： + +```bash +make record-and-replay-official-golden-capture-packet-set +make record-and-replay-official-golden-capture-packet-set \ + RNR_PACKET_DIR=/tmp/ocu-rnr-official-golden-packets +``` + +这会在目标目录下为 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel` 和 `timeout` 各生成一个子目录，并写入根级 `capture-packets.json`。仍然只生成占位输入和 wrapper，不启动官方录制。`capture-packets.json` 和命令返回值都会声明 `includeTranscript` / `requiresMcpTranscriptInput`，每个子 `capturePacket` 也会带同样字段，方便后续交接脚本直接判断是否要等待或校验 transcript 输入。根级 `capture-packets.json` 还会写入 `captureContracts`、`captureContractPaths` 和按 scenario 分组的 `postCaptureWorkflow`，把每个子目录的 `capture-contract.json` 合同与录制后步骤内联到 packet set manifest；后续独立 repo、skill 或批量采集脚本可以只读根 manifest 就知道每个 scenario 的 expected action event、expected `endReason`、status handoff path evidence、transcript evidence requirement，以及替换输入、inspect-only、导入、OCU candidate ingest 和 strict audit 的执行顺序。批量目录还会生成根级 `verify-all.sh`、`verify-workflow.sh`、`inspect-all.sh`、`import-all.sh`、`check-all.sh`、`ingest-ocu-candidates.sh` 和 `strict-expected-failure-audit.sh`，用于在替换完各场景输入后按场景顺序批量执行子目录 wrapper，或在 strict summary 已落盘后审计 required official golden 缺口是否是预期失败；其中 `verify-all.sh` 会逐个运行子 packet 的 `verify-inputs.sh`，`verify-workflow.sh` 会检查根级 manifest、每个子 `capture-contract.json` 和各 wrapper 脚本一致，`inspect-all.sh` / `import-all.sh` 会先检查所有子目录的 hosted status response 与 MCP transcript 输入是否仍是 `_placeholder=true`，任一场景未替换就直接退出，不会先执行前置场景的子 wrapper；`ingest-ocu-candidates.sh` 只运行存在 `ingest-ocu-candidate.sh` 的子场景，导入 candidate 后输出校准用配对命令，并跳过 keyboard / cancel / timeout 这类需要真实录制或不生成 synthetic candidate wrapper 的场景。若生成 packet set 时使用 `--no-include-transcript`，这些 manifest 字段会是 `false`，根级 wrapper 也只检查 status response 输入，不会错误要求不存在的 `inputs/mcp-transcript.json`。baseline smoke、summary builder 和 artifact audit 通过 `checkedOfficialCapturePacketSetContractManifest`、`checkedOfficialCapturePacketSetPostCaptureWorkflow` 与 `checkedOfficialCapturePacketSetWorkflowVerifier` 守住集合级合同、有序 handoff 和 workflow verifier，避免后续只保留子目录文件而丢失可机器消费的根级上下文。该 Make 入口同样支持 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN`、`RNR_FIXTURE_ROOT` 和 `RNR_OFFICIAL_PLUGIN_ROOT`。 + +默认 scenario 来自 `scripts/record_and_replay_scenarios.py` 的 +`DEFAULT_REQUIRED_SCENARIOS[0]`，当前是 required `simple-action-stop`。official +capture preflight、OCU candidate pairing preflight、official ingest、OCU candidate +ingest、fixture coverage report 和 fixture set gate 都应该复用这份 catalog，不要 +在单个脚本里再手写另一套默认场景清单。如果只想生成命令而本机暂时没有官方 +bundled plugin cache，可加： + +```bash +--allow-missing-official-plugin +``` + +需要把 preflight 当成 release 前硬 gate 时，加： + +```bash +--require-ready +``` + +再确认导入工具链 smoke 可用： + +```bash +make event-stream-official-fixture-ingest-smoke +make event-stream-official-fixture-coverage-smoke +make event-stream-official-fixture-set-smoke +make event-stream-ocu-candidate-ingest-smoke +``` + +如果要确认当前 OCU baseline 仍可用，运行： + +```bash +make record-and-replay-baseline-smoke +``` + +需要把最终机器摘要留作发版或独立 repo audit evidence 时，运行： + +```bash +scripts/run-record-and-replay-baseline-smoke.sh --summary-json /tmp/ocu-rnr-baseline-summary.json +``` + +也可以使用 Make 包装入口。默认 baseline audit 写入 +`dist/record-and-replay-baseline-summary.json`，可用 `RNR_BASELINE_SUMMARY_JSON=` +覆盖： + +```bash +make record-and-replay-baseline-audit +make record-and-replay-baseline-audit \ + RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-baseline-summary.json +``` + +该命令应输出 `status.usableBaseline=true`。required `simple-action-stop` 官方样本已入库且 readiness 通过时，`status.officialSuccessfulRecordingGoldenComplete=true` / `status.requiresOfficialGoldenCapture=false` 是正确状态；如果后续在其它 checkout 或新环境里 required 样本缺失，则应回到 `status.officialSuccessfulRecordingGoldenComplete=false` / `status.requiresOfficialGoldenCapture=true`。 + +## 官方采集流程 + +官方 successful recording 必须从正常 Codex 宿主 Record & Replay 流程采集，不要把宿主外 raw MCP start timeout 当成 successful recording。 + +1. 在 Codex 中启动官方 Record & Replay skill / MCP tool。 +2. 按目标 scenario 做最小动作。 +3. 通过官方录制控制结束，或按场景取消 / 等待 timeout。 +4. 保存 `event_stream_start`、active `event_stream_status`、最终 `event_stream_stop` 和完成态 final `event_stream_status` 返回 JSON。 +5. 如果有同 session MCP transcript，也保存为单独 JSON；如果没有，用 `scripts/finalize-record-and-replay-official-capture-packet.py` 从第 4 步的 hosted JSON 生成 packet 所需 transcript evidence。 + +如果前面生成了 capture packet，把第 4 步保存的 JSON 写入 `inputs/event_stream_stop-response.json`，把第 5 步 transcript 写入 `inputs/mcp-transcript.json`，然后先运行 `verify-inputs.sh` 或根级 `verify-all.sh`。不要把原始 hosted JSON、截图、隐私文本或本机绝对路径提交进仓库；正式入库必须通过 inspect-only 与脱敏导入脚本。 + +不要把本机绝对路径、原始截图、原始终端内容、密码字段或隐私文本直接复制进文档。正式入库必须走导入脚本脱敏。 + +## Inspect-Only + +先用 inspect-only 确认 hosted JSON 能解析出录制路径和 evidence，不写 fixture。优先使用 preflight JSON 里的 `commands.inspectOnlyShell`，或手动运行： + +```bash +python3 scripts/ingest-official-record-and-replay-fixture.py \ + --status-json \ + --name official-simple-action-stop-1.0.857 \ + --scenario simple-action-stop \ + --inspect-only +``` + +如果 status JSON 来自剪贴板或临时 stdin： + +```bash +python3 scripts/ingest-official-record-and-replay-fixture.py \ + --status-json - \ + --name official-simple-action-stop-1.0.857 \ + --scenario simple-action-stop \ + --inspect-only +``` + +如果 hosted JSON 里的 handoff path 是相对路径，加： + +```bash +--status-json-base-dir +``` + +如果同一 status JSON 已包含 response-shape evidence，可以加： + +```bash +--use-status-json-as-transcript --require-mcp-transcript-evidence +``` + +如果 transcript 是单独文件： + +```bash +--mcp-transcript --require-mcp-transcript-evidence +``` + +Inspect-only 通过时，重点看返回 JSON 里的 `recordingInputInspection`、handoff path 是否存在，以及 MCP transcript evidence 是否满足当前 scenario。 + +## 正式导入 + +Inspect-only 通过后，去掉 `--inspect-only` 正式导入： + +```bash +python3 scripts/ingest-official-record-and-replay-fixture.py \ + --status-json \ + --name official-simple-action-stop-1.0.857 \ + --scenario simple-action-stop \ + --mcp-transcript \ + --require-mcp-transcript-evidence \ + --check-fixture-set \ + --check-coverage +``` + +导入 required 场景时，可加 `--require-coverage`，让 required scenario 覆盖或 readiness 不足时直接失败： + +```bash +--require-coverage +``` + +正式导入后至少运行： + +```bash +python3 scripts/check-event-stream-official-fixture-coverage.py --require-readiness +python3 scripts/check-event-stream-official-fixture-set.py --official-root docs/references/codex-computer-use-reverse-engineering/fixtures/recordings +``` + +同一检查也有更短的 Make 入口： + +```bash +make record-and-replay-official-golden-fixture-gate +``` + +需要机器可解析 JSON 时，直接使用上面的 `scripts/check-event-stream-official-fixture-coverage.py --require-readiness`；Make 在失败时会追加自身错误行，适合作为 human / release gate。 + +当 required official successful recording 入库并通过 readiness 后，严格 gate 应该通过；日常留证优先运行 audit 入口： + +```bash +make record-and-replay-official-golden-gate-audit +``` + +如果只想跑不落盘的 human gate，也可以用： + +```bash +make record-and-replay-official-golden-gate +``` + +需要把 strict gate 的机器摘要保存到自定义路径时，用： + +```bash +make record-and-replay-official-golden-gate-audit \ + RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-official-golden-summary.json +``` + +不带覆盖变量时，strict gate audit 默认写入 +`dist/record-and-replay-official-golden-gate-summary.json`，避免缺 required official +fixture 时的预期失败摘要覆盖 baseline 可用证据。 + +## OCU Candidate 对比 + +官方 fixture 入库后，可以先运行只读 pairing preflight。它不会启动 action smoke，也不会写 candidate fixture；只检查 official fixture readiness，并输出下一条 OCU candidate ingest / fixture set gate 命令： + +```bash +./scripts/prepare-record-and-replay-ocu-candidate-pairing.py +``` + +或通过 Make： + +```bash +make record-and-replay-ocu-candidate-pairing-preflight +``` + +官方 fixture 入库后，为同一 scenario 导入 OCU candidate： + +```bash +python3 scripts/ingest-ocu-record-and-replay-candidate.py \ + --recording \ + --name ocu-simple-action-stop \ + --scenario simple-action-stop \ + --official-root docs/references/codex-computer-use-reverse-engineering/fixtures/recordings \ + --check-fixture-set +``` + +如果需要重新采 OCU click / drag candidate，可用真实输入 action smoke： + +```bash +python3 scripts/ingest-ocu-record-and-replay-candidate.py \ + --run-action-smoke \ + --scenario simple-action-stop \ + --name ocu-simple-action-stop \ + --official-root docs/references/codex-computer-use-reverse-engineering/fixtures/recordings \ + --check-fixture-set \ + --require-mcp-transcript-evidence +``` + +`keyboard-input-stop` 当前不要走 synthetic keyboard `--run-action-smoke`。该场景先用已有 OCU recording 或保留的 action smoke JSONL 导入，避免 macOS 过滤合成键盘事件造成不稳定。 + +## 完成判定 + +一个 scenario 可以从 OCU baseline 升级为官方校准完成，必须同时满足： + +- official fixture 已脱敏入库，并带正确 `scenario` 与 `scenarioRecipe` manifest。 +- `check-event-stream-official-fixture-coverage.py --require-readiness` 通过。 +- `check-event-stream-official-fixture-set.py` 对 official fixture 通过。 +- 同 scenario OCU candidate 可通过 fixture set gate 或 recording compare。 +- 相关文档删除或更新对应 “待 official golden 校准” 标记。 + +没有通过这些 gate 前，文档和代码注释都应继续使用 “OCU baseline”“官方风格” 或 “待 official golden 校准”。 diff --git a/docs/design-docs/record-and-replay-replication.md b/docs/design-docs/record-and-replay-replication.md new file mode 100644 index 00000000..7b17eed2 --- /dev/null +++ b/docs/design-docs/record-and-replay-replication.md @@ -0,0 +1,247 @@ +# Record & Replay Replication + +## 状态 + +本文记录 2026-06-26 形成的 Record & Replay 复刻方案。它是后续实现和评审的设计入口；证据细节看 `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`，执行拆解看 `../exec-plans/active/20260626-record-and-replay-event-stream.md`。 + +当前方向是直接复刻官方 Record & Replay 的可观察行为，再补 OCU 为了独立运行必须拥有的控制条、等待和通知能力。官方兼容层和 OCU 扩展层必须分开维护。 + +“直接复刻官方”不是把所有 OCU 已实现的便利能力伪装成官方协议，也不是复制官方私有代码。这里的复刻边界是：用官方版本的可观察 MCP transcript、session 文件、`events.jsonl`、AX payload、截图上下文、control endReason 和 skill handoff 行为约束 OCU；未被官方样本确认的字段必须继续标记为 OCU baseline 或待校准。 + +## 目标 + +- 先在 macOS 上复刻官方 `record-and-replay` event stream 的 MCP surface、session 文件、事件语义、AX payload、compact diff 和截图上下文。 +- 让 `open-computer-use event-stream mcp` 可以作为官方 `SkyComputerUseClient event-stream mcp` 的开源替代。 +- 让独立 plugin / skill repo 只依赖 OCU runtime，不依赖 Codex.app 绘制 UI 或转发录制结束交互。 +- 所有偏离官方的能力必须明确标记为 OCU 扩展，不能混进官方兼容 MCP surface。 + +## 已确定的方案口径 + +本轮决策用于约束后续实现： + +- 第一阶段只聚焦 macOS，不把 Windows / Linux recording 纳入当前验收范围。 +- 接受增加 `event-stream` 子命令；官方兼容入口是 `open-computer-use event-stream mcp`，独立集成入口是 `open-computer-use event-stream start/status/stop/cancel/wait/validate/summarize/scaffold-skill`。 +- 目标是直接复刻官方可观察行为，再补 OCU 独立运行所需能力；不是先设计一套“更合理”的新协议，也不是让上层 skill/plugin 依赖聊天约定来解释录制产物。 +- Codex.app 不作为 OCU 的运行依赖。即使官方宿主会提供 feature gate 或 MCP elicitation UI，OCU 也必须自己处理开始确认、录制控制条、Done / Discard、结束状态和 listener 唤醒。 +- `cancel`、`wait`、notify callback、validate / summarize / scaffold 和 standalone repo 自检都只属于 OCU 扩展层；官方兼容 MCP surface 不因此增加工具或参数。 +- AX compact diff、截图上下文、事件字段和 timeout endReason 都必须最终由 official golden recordings 校准；在此之前只能称为 OCU baseline。 +- 独立 thin skill / plugin repo 的职责是包装和消费 OCU runtime，不复制录制实现，也不假设 Codex.app 私有 UI 或回调存在。 + +## 复刻与扩展的接口合同 + +后续实现和评审默认使用这份接口合同判断行为归属： + +- **官方兼容合同**：`open-computer-use event-stream mcp` 的 server name、protocol、initialize response、`tools/list`、tool annotations、3 个 no-arg tools、session 文件布局、`events.jsonl` 主证据、`session.started` / `session.ended` 生命周期事件、官方已确认动作事件、AX payload、截图上下文和 skill handoff 都要向官方样本收敛。 +- **OCU 扩展合同**：`event-stream cancel/wait/validate/summarize/scaffold-skill`、开始确认 UI、录制中控制条、wait listener、notify callback、`latest-session.json` / `active-session.json`、standalone repo 自检和 recording-to-skill scaffold 是 OCU 为独立运行提供的能力，可以比官方多，但必须在文档、脚本和输出字段中保持可识别。 +- **校准合同**：没有 official golden recording 的字段或算法不能升级为“官方等价”。当前 AX compact diff、截图触发阈值、drag / terminal / selection / debug / secure input / raw event 字段、timeout endReason 和 MCP elicitation 业务 message / `_meta` 都仍在校准合同下。 +- **独立 repo 合同**：后续真实 repo 应只包含 thin skill、runtime verifier、skill workflow verifier、package/self-check、synthetic recording-to-skill smoke 和可选 lifecycle smoke。默认 CI 不启动真实录制；真实桌面 lifecycle 只作为本地或授权环境里的 opt-in 检查。 + +## Baseline 版本定义 + +第一波 baseline 版本按“官方可观察行为优先 + OCU 独立运行补齐”定义，避免实现时把可用性、独立集成能力和官方等价混为一谈： + +- **必须具备**：官方三件套 MCP surface、no-active response 对齐、session 文件模型、基础输入事件、AX full / compact diff baseline、按需截图上下文、开始确认、OCU 自有控制条、wait/notify、recording validation、summary、skill scaffold 和 standalone repo scaffold。 +- **必须可验证**：默认 / matrix / screenshot / real-input action smoke、official surface compare、no-active compare、hostless raw timeout fixture、official capture preflight、OCU candidate pairing preflight、source standalone repo smoke、npm staged standalone repo smoke，以及 baseline summary artifact audit。 +- **必须如实报告缺口**：required `simple-action-stop` official successful recording fixture 已入库并通过 readiness 后，baseline 可以把最小 required golden 标为完成；但 recommended `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 未齐和算法级校准未完成时，仍必须报告 recommended gap，不能把全部 official successful recording schema 或 AX compact diff / screenshot 策略宣称为等价。 +- **不得做的事**：不得把 `cancel`、`wait`、notify callback、validation、summary 或 scaffold 参数加入 `event_stream_start/status/stop` 的 tool schema；不得依赖 Codex.app 私有 UI 绘制录制条或转发 Done / Discard；不得把 hostless raw timeout 或 no-active response fixture 当成 successful recording 证据。 + +## 后续推进基准 + +后续基于这份方案继续做 baseline 或拆独立 repo 时，默认按下面顺序推进： + +- **先复刻官方 successful recording**：required `simple-action-stop` official golden 已通过正常 Codex hosted Record & Replay 流程采集和脱敏入库；之后继续补 `keyboard-input-stop`、`drag-stop`、`cancel` 和 `timeout`。采集包只负责准备 wrapper 和占位输入，不启动官方录制；正式入库必须先跑 inspect-only，再脱敏导入，再跑 strict audit gate 留存 summary artifact。 +- **再做 same-scenario OCU candidate**：official fixture 入库后，用 `scripts/ingest-ocu-record-and-replay-candidate.py` 生成同 scenario candidate。candidate 导入不作为 required official golden strict gate 的硬条件；它会输出 `fixtureSetGateShell` / `pairingPreflightShell`，需要校准 OCU 与官方事件形态时再显式运行集合级强对比。 +- **fixture set gate 的 candidate compare 默认走强策略**：同 scenario compare 必须要求 AX diff evidence、AX diff marker 一致、suppressed event sequence 一致和 suppressed schema 一致；如果官方样本没有 diff 或 suppressed stream，gate 应显示对应强策略已打开但不臆造官方字段。这样后续官方样本一旦出现 compact diff 或 suppressed fallback，OCU candidate 缺失或漂移会直接失败。真实 hosted official 样本会包含录制控制条、结束点击等宿主事件，synthetic OCU candidate 可能天然不具备相同事件序列，因此 candidate compare 是校准/差距分析门，不是 required official golden 完成条件。 +- **OCU 扩展只补独立运行体验**：开始确认 UI、录制条、Done / Discard、`wait --session-id`、notify callback、standalone repo verifier 和 recording-to-skill scaffold 都留在 OCU 扩展层；它们可以是 baseline 可用性的必要条件，但不能作为 official successful recording 等价证据。 +- **严格区分两个完成状态**：`usableBaseline=true` 只说明当前 OCU / standalone baseline 可用；只有 required official successful recording 覆盖和 readiness 都通过后，才能把 `officialSuccessfulRecordingEquivalenceReady` 置为 true。same-scenario OCU candidate compare 的结果继续作为校准 evidence 单独解读。 + +## 已确认的官方事实 + +- 官方 `record-and-replay` 插件启动的是 `Codex Computer Use.app` 里的 `SkyComputerUseClient event-stream mcp`。 +- 官方普通 Computer Use 和 Record & Replay 共用同一个 Computer Use runtime，只是 MCP mode 不同。 +- 当前对 Codex.app asar 的观察只确认了 `recordAndReplay` feature gate、插件可见性和通用 MCP elicitation / approval panel 逻辑；没有证据表明 Codex.app 自己实现 `event_stream_*` tools、写 recording files 或绘制录制控制条。 +- 官方 Record & Replay MCP server name 为 `Record & Replay`，协议版本为 `2025-11-25`。 +- 官方 MCP surface 只有 3 个无参数 tools：`event_stream_start`、`event_stream_status`、`event_stream_stop`。 +- 官方 skill 的 concurrent recording 语义是“一次只允许一个 active recording”：`event_stream_start` 如果报告已有 active recording，agent 不应静默当作新的 demonstration 继续推进，而应告知用户已有录制正在进行，并询问要使用这个 active recording 还是等它结束。 +- 官方 start tool description 和 skill 流程都要求录制最长 30 分钟；agent 在 start 成功后应结束当前 turn、请用户完成后回来，并明确告知这个时限。 +- 官方 skill 把 `events.jsonl` 当主输入，`metadata.json` 或 `session.json` 提供 timing、路径和状态。 +- 官方 strings 确认了 `events.jsonl`、`metadata.json`、`suppressed.jsonl`、`session.json`、`eventsPath`、`metadataPath`、`suppressedEventsPath` 等产物或字段。 +- 官方 strings 确认录制控制条文案和结束原因：`recording_controls_stopped`、`recording_controls_cancelled`。 +- Required `simple-action-stop` official successful recording fixture 已入库：`docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/`，可用于校准最小 stopped click scenario、hosted response-shape evidence 和 `recording_controls_stopped` handoff。 +- 官方 strings 确认事件名至少包括 `session.started`、`mouse.click`、`mouse.context_menu`、`mouse.drag`、`keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut`、`terminal.value_changed`、`selection.changed`、`AX.focusedWindowChanged`、`window.changed` 和 `debug.error`，并出现过 `secureInput` 线索。 +- 官方 AX compact diff 使用 `~`、`+`、`-` 表示 changed、added、removed，并存在 diff 过大时回退或跳过 diff render 的行为。 +- 官方 strings 确认 `screenshotNeededForContext` 和 `accessibilityInspectorPayload`，但没有证据表明每个事件都默认截图。 +- 当前没有确认独立 scroll event 名称，也没有确认 timeout 对外 endReason。 + +## 官方兼容层 + +这一层只服务于“像官方一样被 MCP host 使用”。 + +- `event-stream mcp` 只暴露 `event_stream_start`、`event_stream_status`、`event_stream_stop`。 +- 这 3 个 tools 保持无参数，不加入 `cancel`、`wait`、callback、webhook 或 OCU 私有参数。 +- `start` 已有 active session 时返回当前 session，不新建并行录制；agent-facing skill 应把这个结果解释为需要用户选择“使用当前录制”或“等待当前录制结束”，不能静默把它当成本次新录制。 +- `status` 返回当前或最近 session，必须包含可读取的 metadata 和 events 路径。 +- `stop` 幂等；无 active session 时返回最近可解释状态。 +- session 文件至少包含 `events.jsonl`、`metadata.json`、`session.json` alias 和 `suppressed.jsonl`。 +- `events.jsonl` 是后续生成 skill 的事实来源，metadata 只提供会话边界、计数、路径和 endReason。 +- 事件 schema、字段名、顺序、timing 和 AX payload 以 official golden recordings 为最终校准依据。 + +## OCU 扩展层 + +这一层只服务于“脱离 Codex.app 后仍能独立运行”。 + +- OCU 必须自己承载开始确认 UI、录制中控制条、Done / Discard、`wait` 唤醒和 notify callback；独立 plugin / skill 不应依赖 Codex.app 替它画 bar 或转发录制结束交互。 +- CLI 扩展保留在 `open-computer-use event-stream start/status/stop/cancel/wait --json`。 +- `cancel` 只作为 OCU CLI/runtime 能力，不暴露为官方兼容 MCP tool。 +- `wait` 是第一版 listener 模型：调用方启动录制后阻塞等待，用户点击 Done / Discard 或外部 stop/cancel 后返回最终 status；`wait --json` 额外带 `waitTimedOut` 和 `waitSessionMatched` 作为 OCU 扩展字段，供独立集成方区分结束唤醒、超时返回和 session id 不匹配。 +- `wait --session-id ` 会优先读取对应 session 目录里的完成态 metadata，因此即使 latest session 已被后续录制覆盖，也能立即返回目标历史 session；如果指定的 session id 不存在或当前 active session 明显不匹配，会返回 `waitTimedOut=true` / `waitSessionMatched=false`，避免独立集成方无限等待错误 id。 +- `wait --notify-command ''` 是本地 callback 模型：只在 `waitTimedOut=false` 时同步执行，stdin 写入最终 status JSON，同时通过 `OPEN_COMPUTER_USE_EVENT_STREAM_*` 环境变量暴露 session、state、endReason、metadataPath、sessionPath、eventsPath 和 suppressedEventsPath；callback 默认 10 秒超时，失败会让 CLI 返回 error。 +- `validate`、`summarize` 和 `scaffold-skill` 是安装态 recording-to-skill 辅助命令：分别校验 session 结构、摘要事件流、输出 `skillReadiness` 质量信号、结构化 `runtimeInputs`、结构化 `safetySignals`、结构化 `summaryLimits`、完成态 `recordingIncomplete`、`sessionStartedNotFirst`、`sessionStartedCountInvalid`、`sessionEndedNotFinal`、`sessionEndedCountInvalid` 和阻断性 `blockingDiagnostics`，并生成第一版带 workflow readiness / runtime inputs / confirmation signals / summary limits / agent replay procedure / verification checklist / `skill-creator` finalization 指引的 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`。`runtimeInputs` 当前从文本输入和 selection 事件提取运行时值候选，默认只保留长度、目标摘要和敏感标记，不复制原始文本；`safetySignals` 当前从 `keyboard.submit` 和按钮/控件标题里的 send/delete/save/upload/publish 等关键词提取显式确认线索，只保留行号、事件类型、原因、目标摘要和窗口上下文；`summaryLimits` 会记录 action sequence、runtime inputs、safety signals、target/focused elements、diagnostics 和截图路径等高容量字段的上限、源计数、保留计数和省略计数，避免长录制草稿静默漏步骤；`blockingDiagnostics` 当前把 `inputMonitorsUnavailable` 这类会让 demonstration 不可信的诊断转成硬拒绝信号。scaffold 生成前必须先等价执行 `validate --require-skill-draft`，拒绝 event count、metadata/session alias、路径、截图引用、缺失或重复 `session.started`、`session.started` 非首事件、重复 `session.ended`、`session.ended` 后仍有事件等结构不一致的 recording；完整 OCU session 在 `--strict-ocu` 下还会输出 `declaredPaths`，并要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个 handoff path 都已声明且可解析到存在文件，事件 payload 中的 `screenshotPath` 也必须解析到当前 session 目录内的存在文件；通过后再从结构化字段渲染 Runtime Inputs、Confirmation Signals 和 Summary Limits，旧 summary 才回退到 action sequence 推断。生成的 replay procedure 会先提示检查 connector / API / 专用工具，再把 Computer Use 用于 UI 依赖、视觉验证或没有语义工具覆盖的部分；finalization checklist 会明确最终交付必须完成 `skill-creator` workflow 和 validation，成为实际可发现 skill，而不是停在 Markdown 草稿。未完成、取消录制、缺失或重复 `session.started`、`session.started` 非首事件、重复 `session.ended`、`session.ended` 非最终事件、结构不一致或存在阻断性诊断时，scaffold 路径必须拒绝生成草稿并提示用户 stop、修复权限或重新录制。`validate --require-skill-draft` 是同一条 handoff 的前置 gate：它要求至少一个高层动作、recording 已完成、未取消、`session.started` 只出现一次且为首事件、`session.ended` 只出现一次并收尾且没有阻断性诊断，允许其它 `needsReview` 草稿继续生成。action smoke 会用真实录制产物断言生成草稿和 summary 不泄漏本机绝对路径，且 `skillReadiness.canCreateSkillDraft=true`。这些都属于 OCU 扩展层，不进入官方三件套 MCP surface。 +- 源码 checkout 额外提供 `scripts/check-event-stream-golden-readiness.py` 作为官方样本校准 gate：它不替代 runtime `validate`，只用于判断脱敏 fixture 是否具备 final state、`metadata.json` / `session.json` alias 一致性、metadata event/suppressed counts 与文件行数一致、metadata 声明的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 可解析 evidence、唯一且位于首事件的 `session.started`、唯一且位于最终事件的 `session.ended`、动作证据、AX payload、可选 expected `endReason`、可选 full/diff/cumulative AX 证据、可选 `suppressed.jsonl` fallback 证据，以及可选 `mcp-transcript.json` 中的 `event_stream_start/status/stop` response 证据，避免把结构合法但信息不足的录制当成 golden baseline。默认会拒绝同时存在但内容不同的 metadata/session alias，也会拒绝已声明但与文件行数不一致的 `eventCount` / `suppressedEventCount`，并拒绝已经声明但不存在的 handoff path；需要证明官方 handoff alias、计数字段或路径字段完整时可加 `--require-session-alias` / `--require-metadata-counts` / `--require-handoff-paths`。 +- `install-codex-record-and-replay-mcp` 是 Codex 接入便利命令：它只写入独立 `record_and_replay` MCP server，启动 `open-computer-use event-stream mcp`，不改变普通 `install-codex-mcp` 的 9-tool Computer Use server。 +- npm 安装态提供 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `，等价于源码 checkout 里的 `scripts/scaffold-record-and-replay-skill-repo.py --output-dir `；npm 包会携带 thin skill 源和 scaffold 脚本，让真实独立 repo 创建不依赖本仓库源码 checkout。该安装态 helper 当前通过 Python 3 运行 bundled scaffold 脚本，launcher 会优先使用 `PYTHON`，否则在 POSIX 上尝试 `python3` / `python`，在 Windows 上尝试 `python`；缺失或版本不是 Python 3 时返回 command-specific 诊断，不再退化成通用 spawn 错误或晚到 scaffold 脚本失败。 +- npm staged standalone smoke 会把这条 launcher 诊断做成机器证据：`PYTHON` 指向不存在的可执行文件或 Python 2 都必须失败，并包含 `Python 3 is required to run` 与 `PYTHON=/path/to/python3` 修复提示。baseline summary、runner 早期断言和 artifact audit 会消费 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics`，缺失时 `usableBaseline=false`。 +- 控制条由 OCU 自己绘制，显示录制状态、计时、Done、Discard，并支持拖拽。 +- 控制条点击直接进入 OCU runtime，不依赖 Codex.app 转发。 +- 开始录制前的用户确认也应由 OCU runtime 自己承载，不能假设 Codex.app 一定会替独立使用方展示 approval UI。 +- 后续如果需要跨进程长期订阅，可以继续在 OCU 扩展层增加 webhook 或本地 socket；不要修改官方三件套 tool schema。 +- `latest-session.json`、`active-session.json`、`sessionPath` 和调试环境变量可以作为 OCU 集成便利设施，但上层官方兼容 skill 不应必须依赖它们。 +- standalone thin skill repo 的机器可读 contract 要区分默认自检、OCU 扩展层和可选真实录制检查：`checks` 包含 skill package、`manifestContract`、`sourceBaselineSummaryEvidence`、`readmeHandoffContract`、runtime contract、skill workflow verifier、wait/notify contract smoke、synthetic recording-to-skill smoke 和组合 `check.sh`，可以进入默认 CI 且不启动真实录制；生成态 `scripts/verify-manifest.py` 会直接读取 `record-and-replay-skill-repo.json`，校验 `kind`、`runtimeDependency`、官方 `record-and-replay/1.0.857` evidence、`hasSuccessfulRecordingGolden` 与 copied source baseline summary 一致、required / recommended scenario 清单、`scenarioRecipes`、`sourceRepoBaselineAudit`、`strictOfficialGoldenExpectedFailureAudit`、artifact / env var 隔离、官方三件套 no-arg surface、OCU `extensionLayer` 边界和 recording-to-skill strict handoff 没有漂移，并输出 `checkedManifestContract=true`。生成态 `scripts/verify-source-baseline-summary.py` 会读取随 scaffold 复制的 `evidence/source-baseline-summary.json`，该文件是源仓 `dist/record-and-replay-baseline-summary.json` 的脱敏投影；verifier 校验默认 baseline 可用、standalone repo baseline ready、official golden 状态自洽（required gap 明确，或 official golden gate / successful recording equivalence 已完成）、source / npm staged standalone evidence 保留 fixture-set compare policy，并输出 `checkedSourceBaselineSummaryEvidence=true` 和 `checkedSourceBaselineSummaryOfficialGoldenState=true`。如果 source summary 仍带 required gap 却声明 equivalence ready，生成态 verifier 会失败。生成态 `scripts/verify-readme-handoff.py` 会直接读取 README，校验官方 evidence 文件、baseline / strict audit 命令、strict expected-failure 审计命令、required / recommended successful recording 场景、recording-to-skill 命令和 wait/notify 边界仍写给使用方，并输出 `checkedReadmeHandoffContract=true`、`checkedReadmeOfficialEvidenceHandoff=true`、`checkedReadmeOfficialGoldenGap=true` 和 `checkedReadmeWaitNotifyBoundary=true`。`officialEvidence.sourceRepoBaselineAudit` 声明源仓 release / standalone handoff 的两个审计入口：默认可用 baseline 使用 `make record-and-replay-baseline-audit`，官方 successful recording 入库后的 strict gate 使用 `make record-and-replay-official-golden-gate-audit`，audit target dry-run 使用 `make record-and-replay-baseline-audit-targets-smoke`，默认 baseline summary artifact 是 `dist/record-and-replay-baseline-summary.json`，复制到 standalone repo 的投影证据是 `copiedBaselineSummaryEvidence=evidence/source-baseline-summary.json`，默认 strict summary artifact 是 `dist/record-and-replay-official-golden-gate-summary.json`，覆盖变量分别是 `RNR_BASELINE_SUMMARY_JSON` 和 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`，strict 缺 official golden 时的 expected-failure 审计命令通过 `strictOfficialGoldenExpectedFailureAudit` 暴露，并通过 `verifiesSummaryArtifactSeparation=true` / `verifiesSummaryEnvVarIsolation=true` 机器声明源仓会守住两个 artifact 和两个变量互不污染；`extensionLayer` 显式列出 `wait --notify-command` 等独立集成命令，并标记这些能力不属于官方兼容 MCP surface，同时声明 notify callback 会收到 status stdin、metadata/session/events/suppressed handoff 环境变量，且非零退出或超时都必须让 CLI 失败。runtime contract verifier 可以调用 no-active status/stop 来守住官方 response shape，因为该路径不会创建 session，也会调用缺失 / 非 object `params`、缺失 / 非 string `params.name`、非 object `arguments` 和非空 `arguments` 的 malformed request，确认这些错误都走 tool error 且没有 session 文件副作用。生成态 manifest 也用 `mcpServer.rejectedRequestsDoNotCreateSessionFiles=true` 固定这个副作用边界，baseline summary 会消费 `checkedManifestContract`、`checkedSourceBaselineSummaryEvidence`、`checkedSourceBaselineSummaryOfficialGoldenState`、README handoff evidence、`checkedOfficialEvidenceAuditManifest`、`checkedRequiresObjectParams`、`checkedRequiresStringToolName`、`checkedRequiresObjectArguments`、`checkedRejectsUnexpectedArguments`、`checkedRejectsNonObjectArguments` 和 `checkedRejectedRequestsDoNotCreateSessionFiles`，避免独立 repo 后续只验证 schema 错误或 preflight 清单，却漏掉 generated manifest / README / source baseline summary 自校验、source repo audit handoff 或不创建 session 文件的约束。wait/notify contract smoke 会等待一个故意不存在的 session，验证 `waitTimedOut=true`、`waitSessionMatched=false`、notify callback 被跳过且不会创建 recording session 文件；也会用合成 completed session 运行成功 callback，要求 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` 指向 session 内 `suppressed.jsonl`，再验证失败 callback 非零退出返回 `reason=nonZeroExit` 且 CLI 非零、超时返回 `reason=timeout` 且 CLI 非零。baseline summary 会消费 standalone / npm staged 的 `checkedNotifySuppressedEventsPathEnv`、`checkedNotifyCallbackFailureExit`、`checkedNotifyCallbackFailureReason`、`checkedNotifyCallbackTimeoutFailureExit` 与 `checkedNotifyCallbackTimeoutReason` evidence。recording-to-skill smoke 会用 completed recording 验证 strict/events-only validation 与 scaffold-skill 正路径，也会用 cancelled recording 验证 strict/events-only validation 和 scaffold-skill 都拒绝生成 skill。`optionalChecks` 只放会创建本地 recording session 或依赖桌面权限的 lifecycle smoke；源码 standalone baseline 会进一步消费该 smoke 输出的 one-active、idempotent stop 和 final status 语义 evidence，npm staged 默认自检仍不启动真实录制。skill workflow verifier 检查 `SKILL.md` 是否保留官方 handoff 约束（三件套 MCP、start 后结束 turn、不轮询、用户完成后 stop、取消后不再 stop、validate/summarize/scaffold 前置门禁），也检查 Independent Wait / Notify Integration 仍说明 `waitTimedOut` / `waitSessionMatched`、notify stdin/env 和“不把 listener 放进官方 MCP surface”的边界。 +- standalone source summary projection 还必须保留 `preflightPipelines.checkedOfficialCapturePacketSetContractManifest=true`、`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow=true`、`preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff=true` 和 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff=true`，生成态 verifier 输出 `checkedSourceBaselineSummaryOfficialGoldenState=true`、`checkedSourceBaselineSummaryCapturePacketSetContractManifest=true`、`checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true`、`checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 和 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`；source / npm staged scaffold smoke、baseline summary 和 artifact audit 都消费这些 evidence，避免独立 repo 只证明 source summary 存在，却漏掉 official golden 状态自洽检查、recommended packet set 的集合级 `captureContracts` 合同、录制后 `postCaptureWorkflow` 有序 handoff、official fixture 入库后刷新 strict summary artifact 的 handoff，或 strict 缺样本 expected-failure audit handoff。 +- standalone manifest 现在还会把源仓 official fixture set gate 的 same-scenario compare policy 写入 `officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy`，固定 `requiresAxDiffEvidence`、`requiresSameAxDiffMarkers`、`requiresSameSuppressedEventSequence` 和 `requiresSameSuppressedSchema`；`officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 也会固定 `checkedOfficialCapturePacketInputSemanticGuard`、`checkedOfficialCapturePacketSetContractManifest`、`checkedOfficialCapturePacketPostCaptureWorkflow`、`checkedOfficialCapturePacketSetPostCaptureWorkflow`、`checkedOfficialCapturePacketStrictAuditHandoff` 和 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，声明源仓 baseline 必须证明 capture packet 输入语义 guard、recommended packet set 集合级 contract manifest、录制后有序 workflow、strict audit artifact 刷新 handoff 和 strict expected-failure audit handoff。生成态 `scripts/verify-manifest.py` 会输出 `checkedOfficialFixtureSetComparePolicy=true`，source / npm staged scaffold smoke 与 baseline summary 则消费 `checkedOfficialFixtureSetComparePolicyManifest=true`，避免未来独立 repo 只知道要跑 fixture set gate，却丢失 AX compact diff、suppressed stream 强对比边界或 official capture packet set 合同边界。 +- baseline summary 现在还会消费 source / npm staged 的 `checkedInitializeSurfaceContract`、`checkedToolMetadataContract` 和 `checkedToolInputSchemaNoArguments`，把 initialize surface、完整三件套 tool metadata 和 no-arg schema 从宽泛 `checkedRuntimeContract` 中拆出来，避免独立 repo release audit 无法判断具体哪一层官方 surface 被验证过。 +- standalone repo 的 runtime verifier 和 optional lifecycle smoke 需要把旧 runtime / 错误 launcher 的 MCP response timeout 诊断做成合同：错误信息应包含 runtime path、`event-stream mcp` command、`--version` 输出和 `OPEN_COMPUTER_USE_CLI` 修复提示；`OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS` 只用于测试或本地排障快速缩短等待时间。source scaffold smoke 用 fake `0.1.51` runtime 覆盖超时负例，npm staged scaffold smoke 至少验证生成态脚本包含这些诊断字段。 +- baseline summary 会把 source 与 npm staged 的 `checkedRuntimeTimeoutDiagnostics` 纳入 `usableBaseline` 必需 evidence，artifact audit 也会直接复审这两个字段；这让独立 repo 后续不能只保留成功路径 runtime contract，而丢掉旧安装/错误 launcher 的可诊断失败路径。 +- standalone repo 生成态 README 也要把 Python 3 依赖写成用户可见合同：自检脚本要求 `python3`，npm launcher 场景要提示 `PYTHON=/path/to/python3` 修复方式。source scaffold smoke、npm staged scaffold smoke、baseline summary 和 artifact audit 都消费 `checkedGeneratedReadmePrerequisites`，避免 Python 依赖只出现在失败诊断里而没有进入独立 repo handoff 文档。 +- standalone thin skill repo 的 recording-to-skill smoke 还必须把截图路径边界传播到导出的独立 repo：合成 completed recording 正路径继续验证 strict/events-only/scaffold handoff，另造一个 `screenshotPath` 指向 session 目录外但实际存在文件的负例，要求安装态 strict validator 拒绝并输出明确诊断。生成态 manifest 的 `recordingToSkill.strictValidation.requiresScreenshotPathsInsideSession=true` 是机器可读 contract；baseline / npm staged smoke 都要消费 `checkedScreenshotPathContainment=true` evidence，避免主仓 validator 已加 gate 但独立 repo 自检没有继承。 +- npm staged standalone smoke 在调用安装态 scaffold 前也要直接验证同一个 screenshot containment 负例；如果 staged `dist/Open Computer Use.app` 里的 native binary 还没有该 validator gate，应在 preflight 阶段提示重建 `dist` artifacts，而不是等生成 repo 的 `scripts/check.sh` 才失败。 +- npm package build 不能静默漏带 source baseline summary artifact。`scripts/npm/build-packages.mjs` 打包 `open-computer-use` 时必须复制 `dist/record-and-replay-baseline-summary.json`，因为安装态 scaffold 需要从它生成 `evidence/source-baseline-summary.json` 并运行 `scripts/verify-source-baseline-summary.py`；缺失时 build 会以 `Missing Record & Replay baseline summary artifact` 失败并提示先跑 `make record-and-replay-baseline-audit`。npm staged scaffold smoke 会临时移走该 artifact 覆盖负例，确保 release / standalone audit 不是只在源码 checkout 路径可用。 + +## Start Approval 与录制 UI + +官方 strings 能确认 `Record & Replay approval cancelled via MCP elicitation.` 和 `Record & Replay approval denied via MCP elicitation.`；`Codex.app` asar 中也能看到通用 MCP elicitation/approval panel 逻辑。但当前可观察证据没有显示 `Codex.app` 实现 `event_stream_*` tool 或绘制录制控制条。更合理的边界是： + +- `Codex.app` 是宿主，负责 feature gate、插件可见性和通用 MCP elicitation panel。 +- `Codex Computer Use.app` / event-stream runtime 负责录制能力、控制条、事件文件和结束原因。 +- 官方 Record & Replay skill 在自然语言层面要求“用户准备好后再调用 start”，runtime 侧也存在 approval cancelled / denied 的错误语义。 + +OCU 需要同时满足两类使用方： + +- 官方兼容 MCP host：仍然只看到 `event_stream_start/status/stop` 三个无参数 tools。以后如果实现 MCP elicitation，应作为 start 内部流程，不改变 tool schema。 +- 独立 plugin / skill / CLI 使用方：不能依赖 Codex.app 的 approval UI，所以 OCU 应提供自有的开始确认 UI 或可配置 approval policy，并让控制条、Done / Discard 和 `wait` 都直接落到 OCU runtime。 + +推荐实现顺序： + +1. 保持现有默认自动化路径可跑，测试和 smoke 可关闭 start approval。 +2. 在 `EventStreamService.start()` 前增加可替换的 start approval gate，先沉淀 approved / denied / cancelled 三种结果和官方错误文案。 +3. 在 macOS interactive runtime 中接入 OCU 自有开始确认浮层；用户确认后才创建 session 和安装事件 monitor，用户拒绝或取消时不写 session。开始确认文案应告知录制时限，默认是官方一致的 30 分钟上限。 +4. MCP elicitation baseline 已接入：在线性 stdio server 内补 server-initiated request/response 支持，只在支持 `capabilities.elicitation` 的 host 上启用；不支持时回退到 OCU 自有 UI 或配置策略。当前已按 Codex.app host schema 使用显式 `mode: "form"` + 空对象 schema，真实官方 Record & Replay 业务 message / `_meta` 仍待 golden 样本校准。 +5. 把 approval outcome 写入 status / error，但不要暴露新的官方 MCP tool，也不要把 host-specific callback 放进官方兼容层。 + +## 实现分层 + +### Runtime + +`EventStreamService` 是录制状态机，负责 active session、文件写入、事件 append、结束原因和 listener 唤醒。它必须运行在持久进程内；短生命周期 CLI 只能做调试，不能作为真实录制状态源。 + +关键约束： + +- start/status/stop/cancel/wait 通过 app-agent 访问同一份进程内状态。 +- `wait` 不能阻塞其它连接的 stop/cancel。 +- `finish()` 必须先写最终 `session.ended` 和 metadata，再暴露 inactive 状态。 +- 权限缺失、AX snapshot 失败、截图失败、事件压缩失败和 Input Monitoring monitor 安装失败都应可恢复，并进入 status、主事件的低敏 `debug.error` 或 suppressed stream。 + +### Event Stream + +第一阶段必须覆盖官方已确认事件名： + +- session：`session.started`、`session.ended` +- mouse：`mouse.click`、`mouse.context_menu`、`mouse.drag` +- keyboard：`keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut` +- terminal：`terminal.value_changed` +- selection：`selection.changed` +- window：`window.changed` +- AX：`AX.focusedWindowChanged` +- debug：`debug.error` + +OCU baseline 已有点击、右键、拖拽、文本输入、提交、快捷键、terminal value changed、selection changed、window changed、debug error、experimental raw events 和 AX focused window。`mouse.drag` 当前按 mouse down / dragged / up 的起止位置、距离和时长记录，字段结构仍需 official golden recordings 校准。`terminal.value_changed` 当前只在明显终端 app / 终端 focused element 且 AX value 变化时记录，payload 写 `valueHash`、`valueLength` 和脱敏后的 `focusedAccessibilityElement`，不写终端缓冲区全文；事件触发时机、字段名和官方是否记录明文仍需 official golden recordings 校准。`selection.changed` 当前在 focused AX element 暴露非空 `selectedText` 且 selection signature 变化时记录选择内容；如果此前已记录过选择而当前选择为空，也会写 `selectionCleared=true` 的清空事件，避免后续 skill 误用 stale selection；字段名、触发时机和清空选择时的官方行为仍需校准。`window.changed` 当前在录制开始和输入事件前 app/window context 变化时写入，并继续补 `AX.focusedWindowChanged` 作为上下文；字段名、是否初始帧也写入、以及与 AX event 的顺序仍需 official golden recordings 校准。`debug.error` 当前在 AX snapshot、AX diff cumulative fallback、screenshot 写入失败和 Input Monitoring monitor 全部安装失败这类上下文采集降级时写入主事件流，payload 只保留 `subsystem`、`reason`、`errorType` 和低敏 context；Input Monitoring 不可用时会写 `subsystem=inputMonitoring` / `reason=inputMonitorsUnavailable`，避免独立集成方把没有输入事件的录制误判为正常空录制；完整错误细节仍进入 `suppressed.jsonl`，字段仍需官方样本校准。`secureInput` 当前作为安全 baseline：focused AX element 暴露 protected/secure/password 信号时，键盘文本事件只写 `secureInput=true` 和长度，不写明文 `text`，raw key event 也只写 redaction 标记和字符数；字段仍需官方样本校准。`experimentalRawEvents` 当前作为低层 NSEvent 摘要附在点击、拖拽和键盘事件上；scroll wheel 在没有确认官方独立 scroll event 名称前只写 raw-only `experimentalRawEvents` 事件，且 opt-in action smoke 已用真实 CG scroll wheel 验证该 raw-only 路径会入流。为了让生成 skill 不漏掉滚动意图，summary/scaffold 会把 `reason=scrollWheel` 的 raw-only event 当作 replay action 渲染为 Scroll step；这仍是 OCU baseline，待官方 scroll schema 校准。 + +### AX Payload + +AX payload 是复刻质量的关键，不应长期停留在每次 full tree。 + +- 首帧或无法 diff 时输出 full tree。 +- 同一 app/window 的后续 payload 输出 compact diff。 +- compact diff 使用官方风格的 `~`、`+`、`-`。 +- diff payload 保留 previous diff 作为主 `renderedText/treeLines`，并额外输出从 initial tree 到 current tree 的 cumulative diff 字段。 +- diff 过大、removed summary 超预算或 baseline 不稳定时回退 full tree，并记录 suppressed reason。 +- 后续用 official golden fixtures 校准 element identity、line budget、removed range summary、cumulative diff 和 fallback 条件。 + +### Screenshot Context + +截图是上下文补充，不是默认每事件主数据。 + +- 默认策略应最小化截图采集。 +- 当前 OCU 提供 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never` 供验证和调试。 +- payload 标记 `screenshotNeededForContext` 和 `screenshotAvailable`，写入成功时提供 `screenshotPath`。 +- 官方触发阈值、路径字段和 suppressed 行为仍待 golden recordings 校准。 + +### Recording Controls + +控制条属于 OCU 独立性要求，不属于官方 MCP surface 扩展。 + +- Done 结束为 `recording_controls_stopped`。 +- Discard 结束为 `recording_controls_cancelled`。 +- 控制条必须能在普通窗口、多显示器、Space 切换和权限缺失场景下给出明确结果或错误。 +- 自动化和 headless 验证可关闭控制条。 + +## 推进顺序 + +1. 固化官方 golden recordings：stop、cancel、重复 start、无 active status/stop、最长时长或模拟 timeout；入库时同时带上脱敏 MCP transcript，先跑结构 validator，再跑 golden readiness gate，确认动作类样本有一致的 `metadata.json` / `session.json` alias、可信的 metadata event/suppressed counts、唯一且位于首事件的 `session.started` 和唯一且位于最终事件的 `session.ended`，并在样本用于校准 AX / screenshot / 压缩 fallback 时显式要求 suppressed evidence，最后才进入 OCU candidate compare。 +2. 对齐 MCP response 和 session 文件 schema：字段名、状态流转、endReason、路径命名、idempotent 行为。 +3. 补 start approval gate：先沉淀 denied / cancelled 语义，再接 OCU 自有开始确认 UI，最后评估是否实现 MCP elicitation。 +4. 对齐事件 schema：鼠标、键盘、AX、drag、raw events，确认是否存在 scroll event。 +5. 校准 AX compact diff：从当前轻量实现推进到官方样本可比较的算法级行为。 +6. 强化 OCU 扩展层：控制条场景、wait listener、notify callback、后续订阅模型和权限错误。 +7. 拆独立薄 plugin / skill repo：只包装 OCU runtime，不复制录制逻辑；以 scaffold 生成的 `record-and-replay-skill-repo.json` 为 contract，保持 `checks` / `optionalChecks` 分层，发布前跑默认 `scripts/check.sh`，有桌面权限时再显式跑可选 lifecycle smoke。 + +## 验收门槛 + +- `tools/list` 与官方当前 baseline 的 name、description、input schema 和 annotations 对齐。 +- no-active `event_stream_status` / `event_stream_stop` 的 MCP text JSON 与 Codex-hosted 官方观察对齐为 `isRecording=false` / `maxDurationSeconds=1800`。 +- OCU golden tests 能用官方样本约束 session 文件布局、MCP response 和关键事件 schema。 +- OCU smoke 稳定覆盖 start/status/stop、session alias、AX payload、截图上下文、timeout、wait listener 和 notify callback。 +- opt-in action smoke 覆盖真实 CGEvent click / keyboard event 录制，并用同一份 recording 生成 `SKILL.md` 草稿，证明 baseline 不只停留在 synthetic fixture。 +- opt-in baseline smoke 的机器可读摘要能区分多类证据：OCU 当前可用性、截图上下文 baseline、官方 non-recording surface 对齐、官方 no-active status/stop response 对齐、官方 raw start-timeout 边界、official fixture set gate 准备情况、official/OCU fixture ingest pipeline 准备情况，以及 official capture / OCU pairing preflight 准备情况；顶层 `status` 还会直接输出 `usableBaseline`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady`、`officialGoldenGatePassed`、`officialGoldenRequirementSatisfied`、缺失的 required / recommended official successful recording scenarios，以及已覆盖但未通过 readiness 的 `notReadyRequiredOfficialSuccessfulRecordingScenarios`。如果 coverage report 发现 recipe drift、重复 scenario 或 manifest 错误，摘要会同步输出 `status.officialFixtureCoverageErrors` 与 `evidence.officialFixtureSetGate.coverageErrors`。`usableBaseline` 必须由这些 evidence 计算得到，必需 evidence 缺失时输出 `missingUsableBaselineEvidence` 并返回非零；`standaloneRepoBaselineReady=true` 只表示 standalone scaffold baseline 与 npm staged scaffold baseline 已可交付，不表示全部官方 successful recording schema 已校准。摘要的 `nextActions` 会按当前状态给出下一步命令：baseline evidence 缺失时先修 baseline，缺 required golden 时生成 official capture packet，运行 packet 内 verify / inspect / import wrapper，并以 `make record-and-replay-official-golden-gate-audit` 刷新 strict summary artifact；recommended 未齐时生成 recommended capture packet set 并运行 verify-all / inspect-all / import-all / ingest-ocu-candidates，standalone baseline ready 时先运行 `make record-and-replay-baseline-audit` 留存 source summary artifact，再给出 scaffold repo、generated self-check 和 opt-in lifecycle smoke 命令。默认模式允许缺 official golden，因此 `officialGoldenRequirementSatisfied=true` 只表示当前运行没有强制要求 golden；`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady` 和 `officialGoldenGatePassed` 仍要求 required official scenario 覆盖和 required fixture readiness 同时通过。`scripts/run-record-and-replay-baseline-smoke.sh --summary-json ` 会把同一份最终 JSON 额外落盘，供 release / standalone audit 保存证据，同时继续向 stdout 打印并保持原退出码；默认 CI 的 runner 级 stub smoke 会验证落盘 JSON 与 stdout 最终摘要一致，并验证严格模式失败时仍保留非零退出码。`scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden` / `make record-and-replay-official-golden-gate` 会把同一套 baseline 证据提升为严格 gate：没有 required official successful recording fixture、readiness 未通过或 coverage report 有错误时，最终 JSON 写入 `ok=false`、`strictOfficialGoldenRequired=true`、`officialGoldenRequirementSatisfied=false`、`officialGoldenGatePassed=false` 并返回非零；stderr 会分别列出 `missing=...`、`notReady=...` 与 `coverageErrors=...`，方便判断是要补采官方样本还是修复已有 fixture。需要留存 strict artifact 时使用 `make record-and-replay-official-golden-gate-audit`；missing-golden `nextActions` 默认指向这个 audit 入口。 +- opt-in baseline smoke 对真实输入 action 录制到 skill 草稿链路会消费具体 evidence：`checkedMcpResponseShapesCaptured`、`checkedSkillReadinessCanCreateDraft`、`checkedSkillCreatorFinalizationHandoff` 和 `checkedGeneratedSkillPathRedaction` 任一缺失都会让 `usableBaseline=false`。baseline runner 还会在默认 mixed action smoke 之外运行 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop`，要求 `checkedSimpleActionStopCandidate` 和 `checkedDragStopCandidate` 同时成立；缺少 click / drag 同场景 OCU candidate 时也会让 `usableBaseline=false`。这避免只因为产出了 `skillPath` / `mcpTranscriptPath` 就把 action smoke 当成完整可用证据，也避免后续拆独立 repo 时缺少可与 official `simple-action-stop` / `drag-stop` golden 配对的 OCU 候选链路。 +- opt-in baseline smoke 对 source standalone 和 npm staged recording-to-skill 证据都会检查具体语义，而不是只看总 handoff 开关：`checkedStrictValidation`、`checkedEventsOnlyValidation`、`checkedScaffoldSkill` 和 `checkedSkillCreatorHandoff` 任一缺失都会让 `usableBaseline=false`，避免独立 repo 默认自检保留了文案但丢失完整 session gate、events-only gate、草稿生成正路径或 skill-creator 收尾指引；npm staged 路径还证明这些证据能通过安装态 launcher 生成的 repo 自检传播到 release / standalone audit summary。 +- baseline 摘要里的 official capture `nextActions` 优先输出 Make packet 入口：required 缺口使用 `make record-and-replay-official-golden-capture-packet RNR_SCENARIO= RNR_PACKET_DIR=`，随后在 packet 目录里运行 `verify-inputs.sh`、`inspect-only.sh` / `import-fixture.sh`；recommended 缺口使用 `make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR=`，随后运行 `verify-all.sh`、`inspect-all.sh` / `import-all.sh` / `ingest-ocu-candidates.sh`。这让 release / standalone audit 可以直接复用同一套 packet wrapper，而不是重新拼 preflight 参数。 +- baseline 摘要也会要求单场景 `verify-inputs.sh` 存在并拒绝占位/非法输入，要求 `inspect-only.sh` / `import-fixture.sh` 和 packet set 根级 `inspect-all.sh` / `import-all.sh` 拒绝 `_placeholder=true` 输入；如果输入校验 wrapper 缺失或失效，`preflightPipelines.checkedOfficialCapturePacketVerifyInputs` / `checkedOfficialCapturePacketSetVerifyAll` 会让 usable baseline 失败。capture packet 现在还会写 `capture-contract.json`，机器声明 scenario、fixture name、预期 action event、预期 `endReason`、status handoff path evidence 和 transcript evidence requirement；`verify-inputs.sh`、`inspect-only.sh` 和 `import-fixture.sh` 会基于该合同确认 hosted status JSON 至少包含 `eventsPath`，并且包含 `metadataPath` 或 `sessionPath` 这类 successful recording handoff 文件证据；`suppressedEventsPath` 和 `sessionDirectoryPath` 只作为 optional evidence 记录，兼容官方最小 completed session。transcript enabled 时还要确认 MCP transcript evidence。`preflightPipelines.checkedOfficialCapturePacketInputSemanticGuard` 会把这条语义校验纳入 usable baseline 必需 evidence，避免合法 JSON 但不是 Record & Replay recording handoff 的输入进入导入链路。如果正式导入 wrapper 不再拒绝未替换的 hosted status/transcript 占位输入，`preflightPipelines.checkedOfficialCapturePacketImportPlaceholderGuard` 会让 usable baseline 失败；如果批量 wrapper 不再拒绝占位输入，`preflightPipelines.checkedOfficialCapturePacketSetRootPlaceholderGuard` 会让 usable baseline 失败；如果根级 `inspect-all.sh` / `import-all.sh` 没有在执行任何子 wrapper 前全量检查所有场景输入，导致可能先导入前置场景再在后续占位输入处失败，`preflightPipelines.checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard` 会让 usable baseline 失败。单场景 packet 还必须在 README 和 `capturePacket.*Shell` 字段里暴露 `check-fixture-set.sh`、`strict-golden-gate.sh` 和可用时的 `ingest-ocu-candidate.sh`，`preflightPipelines.checkedOfficialCapturePacketHandoffScripts` 会把这条采样后 handoff 纳入 usable baseline 必需 evidence；其中 `strict-golden-gate.sh` 必须运行 `make record-and-replay-official-golden-gate-audit` 并由 `preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff` 守住，避免采到 official golden 后只跑不落盘的 strict gate。单场景 `capture-contract.json` 和 `capturePacket.postCaptureWorkflow` 还必须声明录制后有序步骤，`preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow` 会守住替换 hosted JSON、verify、inspect、import、coverage/fixture-set、可选 OCU candidate ingest、strict gate 和 strict expected-failure audit 的机器可读顺序。`preflightPipelines.checkedOfficialCapturePacketOcuCandidateOutputDir` 进一步要求 `ingest-ocu-candidate.sh` 使用当前 official fixture root 下的 `ocu-candidates/` 作为输出目录，避免自定义 fixture root 时 official fixture 与 OCU candidate 分散到不同根目录；`preflightPipelines.checkedOfficialCapturePacketSetOcuCandidateHandoff` 要求 recommended packet set 根级生成 `ingest-ocu-candidates.sh`，并在批量采样后自动运行所有可生成 synthetic OCU candidate 的子场景、跳过 recording-required 或 no-action 场景，避免 recommended set 校准还要人工逐个进入子目录找 wrapper。recommended packet set 的根级 `capture-packets.json` 还会写 `captureContracts` / `captureContractPaths` / `postCaptureWorkflow`，`preflightPipelines.checkedOfficialCapturePacketSetContractManifest` 会要求这些集合级合同存在并匹配 scenario recipe，`preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow` 会要求每个 scenario 的 post-capture workflow 与子 `capture-contract.json` 一致，使独立 repo 或 skill 能直接从根 manifest 消费 expected action event、expected `endReason`、handoff path evidence、transcript requirement 和录制后的执行顺序，而不是靠 README 或遍历子目录。单场景 `capturePacket`、根级 `capturePacketSet` 和根级 `capture-packets.json` 都会声明 `includeTranscript` / `requiresMcpTranscriptInput`，`preflightPipelines.checkedOfficialCapturePacketTranscriptManifest` 会把这条机器可读合同纳入 usable baseline 必需 evidence，并由 summary smoke 覆盖缺失负例。`preflightPipelines.checkedOfficialCapturePacketNoTranscript` 守住单场景 `--no-include-transcript` packet：只生成 status response input，`mcpTranscriptInputPath=null`，单场景 wrapper 不要求 transcript。`preflightPipelines.checkedOfficialCapturePacketSetNoTranscript` 另外守住 `--no-include-transcript` 批量 packet set：manifest 表示不需要 transcript，根级 wrapper 只检查 status response，不会要求未生成的 `inputs/mcp-transcript.json`。 +- 单场景 capture packet 现在还会生成 `strict-expected-failure-audit.sh`，根级 packet set 也会生成同名 wrapper；它运行 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing`，用于 strict gate audit 已写出失败 summary 后确认 required official golden 缺口是预期失败。official capture preflight smoke 输出 `checkedCapturePacketStrictExpectedFailureAuditHandoff=true`，summary builder 投影为 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，artifact audit 通过 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence` 守住这条 handoff。 +- baseline runner 在生成摘要后会立即用 `scripts/check-record-and-replay-baseline-summary.py` 做 artifact 审计，audit 输出写到 stderr，stdout 仍保持最终 summary JSON，避免破坏机器消费合同。落盘后的 summary 也可单独用 `scripts/check-record-and-replay-baseline-summary.py ` 重审。默认审计不重跑桌面 smoke，要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致、顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查且不带未知或重复 check、usable baseline、standalone repo baseline、official non-recording baseline 和 raw timeout boundary 成立，并直接审计所有 usable baseline 子证据：event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence；同时会审计派生状态和 `nextActions` 的命令内容，确认 `requiresOfficialGoldenCapture`、`officialSuccessfulRecordingEquivalenceReady`、`standaloneRepoBaselineReady` 与 required scenario gaps / coverage errors 保持一致，缺 required golden 时仍给出 packet Make、`verify-inputs.sh`、inspect/import 和 `make record-and-replay-official-golden-gate-audit` strict audit step，缺 recommended golden 时仍给出 packet set Make、`verify-all.sh`、inspect-all/import-all 和 `ingest-ocu-candidates.sh`，standalone ready 时仍给出 source `record-and-replay-baseline-audit`、scaffold、generated `check.sh` 和 lifecycle smoke。`declaredChecks` 诊断会分别列出 `missingRequired`、`unknown` 和 `duplicates`；`scripts/record_and_replay_baseline_contract.py` 是顶层 `checks` 的唯一源码，builder 按该 tuple 顺序输出，audit 按同一合同复审，summary audit smoke 还会直接断言 builder 输出顺序等于 shared contract，并覆盖 unknown / duplicate 负例。`--require-official-golden` 模式要求 summary 本身来自 strict gate，并要求 golden gate、required scenario gaps、coverage errors 和 equivalence ready 全部满足；`--allow-strict-official-golden-missing` 用来审计 strict gate 在 required official fixture 缺失或 not-ready 时写出的预期失败摘要，要求 strict mode、`ok=false`、baseline evidence 完整、无 coverage errors，并且失败由 required scenario gap 解释。该脚本用于 release / standalone audit 阶段消费 `--summary-json` 产物，避免人工误读 `usableBaseline=true`、派生状态组合、缺失或污染的顶层 check 声明、漂移的 handoff 命令，以及 strict 失败摘要到底是缺 official golden 还是 baseline 回归。 +- Makefile 也提供 audit 包装入口：`make record-and-replay-baseline-audit` 会运行默认 baseline 并把 summary 写到 `RNR_BASELINE_SUMMARY_JSON`，默认路径为 `dist/record-and-replay-baseline-summary.json`；`make record-and-replay-official-golden-gate-audit` 会运行 strict official golden gate 并把 summary 写到 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`，默认路径为 `dist/record-and-replay-official-golden-gate-summary.json`。两者只包装现有 runner，不改变默认 baseline / strict golden gate 的退出语义，且默认 strict 失败摘要不会覆盖 baseline 可用证据。默认 CI 不运行完整 audit target，但会运行 `scripts/test-record-and-replay-baseline-audit-make-targets.py` 的 dry-run smoke，确认 Make target 展开仍传递 `--summary-json`，strict target 仍传递 `--require-official-golden`，baseline/strict 默认 artifact 分离，并且 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 不影响 baseline audit、`RNR_BASELINE_SUMMARY_JSON` 不影响 strict audit。完整 baseline runner 也会消费这份 dry-run smoke，把 baseline 默认/自定义 summary path、baseline 忽略 strict summary 变量、strict 默认/自定义 summary path、strict 忽略 baseline summary 变量、strict 独立 summary path 和 strict audit target 写入 `evidence.preflightPipelines` 并纳入 `usableBaseline`。 +- 自有控制条能完成 Done / Discard，并把 endReason 写入最终 metadata。 +- Codex 用户可以通过 `open-computer-use install-codex-record-and-replay-mcp` 安装独立 R&R MCP surface，无需手写 `mcp_servers.record_and_replay` 配置。 +- docs 明确标记“已确认官方行为”“OCU baseline”“待官方校准”。 + +## 当前缺口 + +- opt-in baseline smoke 现在会把 Codex-hosted no-active fixture 对比写入 `evidence.officialNoActiveResponse`，要求 `event_stream_status` / `event_stream_stop` 都是 `isRecording=false` / `maxDurationSeconds=1800` 的最小 text JSON，并确认该对比路径不会创建 session 文件。这补齐了官方 non-recording response 证据；required successful recording evidence 由 `official-simple-action-stop-1.0.857` fixture 和 coverage/readiness gate 单独证明。 +- Required `simple-action-stop` official successful recording 已入仓并通过 readiness；recommended `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 仍未齐。 +- 已有 hosted official recording ingest、OCU candidate ingest、fixture import、compare、golden readiness gate 和集合级 official fixture set gate；`scripts/ingest-official-record-and-replay-fixture.py` 可从 official `event_stream_stop` / `event_stream_status` 返回 JSON 中递归提取 `metadataPath` / `sessionPath` / `eventsPath` / `sessionDirectoryPath`，支持 `--status-json -` stdin 输入，并可用 `--status-json-base-dir` 为相对 handoff path 指定解析基准；当 hosted JSON 只给 `sessionDirectoryPath` 时，会把该目录作为 recording input 交给 fixture import 读取官方最小 `events.jsonl + session.json` 包。`--mcp-transcript -` 也可在 status JSON 来自文件时从 stdin 读取单独 transcript，随后调用 import 脚本脱敏入库，并按 scenario 运行 readiness；`scripts/ingest-ocu-record-and-replay-candidate.py` 可从 `--recording`、action smoke JSONL 或 opt-in `--run-action-smoke` 生成 `source=ocu` candidate fixture，并可用 `--official-root --check-fixture-set` 立即跑同 scenario official-vs-candidate gate。OCU candidate ingest 输出会同时带 `commands.pairingPreflightShell` 和 `commands.fixtureSetGateShell`，后续 official fixture 入库或 candidate 重采后可以直接复跑配对 preflight 与集合级 gate；baseline summary 通过 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff` 和 `checkedOcuCandidateIngestHandoffCommands` 把 official directory handoff 与 candidate 后续 handoff 纳入 usable baseline evidence。两者带 `--mcp-transcript` 时还可要求 response-shape evidence。OCU candidate ingest 已对 `simple-action-stop` / `keyboard-input-stop` / `drag-stop` 三类 stopped action scenario 做场景级 validation/readiness，分别要求 click / text input / drag 事件、AX context、stopped endReason、full AX payload和 completed MCP response-shape evidence。readiness 已要求动作类 golden 具备 `metadata.json` / `session.json` alias 一致性、metadata count 一致性、declared handoff path evidence、`session.started` 首事件 / 唯一性和 `session.ended` 最终事件 / 唯一性，也可通过 `--require-session-alias` 要求 alias 文件完整，通过 `--require-metadata-counts` 要求 `eventCount` / `suppressedEventCount` 字段完整，通过 `--require-handoff-paths` 要求 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 字段完整，通过 `--require-suppressed-events` / `--require-suppressed-event-type` 要求 suppressed fallback evidence，compare 的 `--require-final-session-evidence` 也已扩展为 start/end lifecycle evidence。official fixture set smoke 已覆盖 `timeout` 场景只要求生命周期 evidence、允许 no action / no AX 且暂不固定 endReason。`scripts/run-record-and-replay-baseline-smoke.sh` 会把 official fixture set gate 的自检结果写入 `evidence.officialFixtureSetGate`，并扫描仓库真实 official recording fixture scenario 覆盖；同一 baseline 摘要也会把截图上下文 smoke写入 `evidence.screenshotContextSmoke`，要求 `always` policy 下出现 `screenshotNeededForContext`，并在当前环境返回截图数据时要求 `screenshotPath` 存在；同一摘要还会把 raw start-timeout fixture smoke 写入 `evidence.officialRawStartTimeout`，要求三次官方宿主外 raw `tools/call` timeout、无 recording handoff paths 且 sanitized fixture 不含本机路径；同一摘要还会把 hosted official fixture ingest smoke 和 OCU candidate ingest smoke 写入 `evidence.fixtureIngestPipelines`，要求 official ingest、official `sessionDirectoryPath` handoff、candidate ingest、candidate ingest 后续 pairing/gate handoff 命令、smoke JSON import、candidate redaction、official/candidate pairing、keyboard readiness 和 drag readiness 均已通过；同一摘要还会把 official golden capture preflight 与 OCU candidate pairing preflight smoke 写入 `evidence.preflightPipelines`，要求缺 official 样本、采集包 Make targets、采集包、批量采集包、`--require-ready` 失败、缺官方插件、keyboard 非 synthetic action smoke、coverage error、缺 candidate、paired candidate 和 candidate compare 等下一步操作合同都被覆盖。required 覆盖仍以 `simple-action-stop` 作为最小门槛，recommended 覆盖会额外报告 `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 是否齐备，用于指导后续 official golden 采样。 +- recording compare 现在除主 `events.jsonl`、metadata、AX diff、semantic fields 和 MCP response shape 外，也能用 `--require-handoff-paths` 比较 official baseline 与 OCU candidate 是否都声明并保留可解析 handoff path，用 `--require-same-suppressed-event-sequence` / `--require-same-suppressed-schema` 比较 `suppressed.jsonl` 的降级、压缩和错误证据。集合级 official fixture set gate 的 same-scenario compare 已默认打开 `--require-ax-diff-evidence`、`--require-same-ax-diff-markers`、`--require-same-suppressed-event-sequence` 和 `--require-same-suppressed-schema`，并把 `checkedAxDiffComparisonPolicy`、`checkedSuppressedStreamComparisonPolicy` 与 `checkedAxDiffComparisonFailure` 写入 baseline summary evidence。后续 official golden 入库后，如果官方样本包含 compact diff、cumulative diff 或 suppressed fallback，OCU candidate 缺失对应 evidence 会直接失败；动作类 handoff 样本仍应打开 handoff path gate，确保 candidate 不只保留 metadata key，还保留可被 skill 消费的路径。该强对比用于校准和差距分析；required official golden strict gate 只要求官方 required fixture 覆盖与 readiness。 +- AX compact diff 仍是轻量官方风格实现；previous diff 和 cumulative diff 已有 baseline 字段，但不是已确认等价的官方算法。 +- `mouse.drag`、`terminal.value_changed`、`selection.changed`、`window.changed`、`debug.error`、Input Monitoring 不可用时的 debug payload、`secureInput`、`experimentalRawEvents` 和 input 前 window context 检测已有可用 baseline，但字段名、距离阈值、terminal value 触发时机和脱敏策略、selection 触发时机与清空字段、window 事件顺序、debug payload、secure input 字段、raw payload 与尚未确认的 scroll event schema 仍需官方样本校准。 +- start approval gate、OCU 自有开始确认弹窗和协议级 MCP elicitation request/response 已有 baseline；host 可渲染 schema 已校准，官方 Record & Replay 真实业务 message / `_meta` 仍未按 golden 完整复刻。 +- 已新增 `scripts/probe-event-stream-recording.py`，可对 local OCU 或官方 bundled client 复跑 initialize/tools/start/status/stop transcript 探测；当前本机官方 1.0.857 raw surface 可正常返回，raw `event_stream_start` / `event_stream_status` / `event_stream_stop` 仍在宿主外超时，没有产出 official recording paths。该 timeout transcript shape 已保存为 `../references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json`，并由 `scripts/test-event-stream-probe-fixtures.py` 校验不含本机路径、三次 `tools/call` 都是 timeout、且没有 `metadataPath` / `eventsPath` / `sessionPath` 这类 recording handoff paths；baseline summary 会把这些校验作为 `evidence.officialRawStartTimeout` 的必需 evidence。 +- timeout endReason 当前是 OCU baseline，未确认官方实际输出。 +- 独立 plugin / skill repo 尚未创建；当前 scaffold 生成态 repo 已带 package frontmatter gate、package artifact verifier、runtime contract verifier、skill workflow verifier、wait/notify contract smoke、组合 `check.sh`、可选 `recording-lifecycle-smoke.py` 和 GitHub Actions CI workflow，可本地或在干净 runner 上验证 `open-computer-use event-stream mcp` 的官方三件套 surface（含 initialize capabilities、tool description、empty input schema、MCP annotations 和 no-active response shape）、thin skill 官方 handoff 语义、OCU wait/notify 扩展边界，并产出且审计 `.skill` archive。`scripts/verify-package-artifact.py` 会验证 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` 的 frontmatter 和 Record & Replay handoff 片段仍存在，并通过 `checkedPackageArtifact=true` 进入 source / npm staged scaffold smoke、baseline summary 和 artifact audit；`record-and-replay-skill-repo.json` 也会写入 `officialEvidence`、`extensionLayer` 和 `recordingToSkill`，让独立 repo 的默认 contract 机器可读地区分官方 1.0.857 surface / no-active / raw timeout 证据、required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景清单、每个场景的 `scenarioRecipes` 采集合同、源仓负责的 official capture / OCU candidate pairing preflight baseline checks、standalone 默认自检不启动官方录制且不复制 preflight/runtime 源码的边界、`wait --notify-command` 扩展边界，以及 recording-to-skill handoff 门禁。生成态 README 的 required / recommended official scenario 文案也从同一 scenario catalog 生成，source scaffold、npm staged scaffold 和 baseline summary 都消费 `checkedGeneratedReadmeScenarioList`，避免 manifest 正确但用户文档里的官方采样清单漂移。源码 checkout 和 npm 安装态都可以生成这套骨架。具备本地桌面权限时也可以显式运行 lifecycle smoke，验证安装态 runtime 能创建最小 recording、重复 start 返回同一 active session、stop 幂等、停止后的 status 指向同一个 completed session，并用 validator 校验 session 文件。但还没有真实外部 repo 的发布、安装和端到端业务录制验证。 +- Official capture packet 本身现在也带 `verify-workflow.sh`。单场景 packet 会用它校验 `capture-contract.json` 的 `postCaptureWorkflow`、input 文件、wrapper command、可执行 bit 和 transcript 开关一致；recommended packet set 会用根级 `verify-workflow.sh` 校验 `capture-packets.json`、每个子 `capture-contract.json` 和子 wrapper 一致。源仓 baseline 消费 `checkedOfficialCapturePacketWorkflowVerifier` / `checkedOfficialCapturePacketSetWorkflowVerifier`，standalone source summary 继承 `checkedSourceBaselineSummaryCapturePacketWorkflowVerifier` / `checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier`，避免后续只拿到 hosted official JSON 时才发现 packet 执行计划已经漂移。 + +## 关联文档 + +- `record-and-replay-handoff.md` +- `../exec-plans/active/20260626-record-and-replay-event-stream.md` +- `../references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md` +- `../ARCHITECTURE.md` +- `../QUALITY_SCORE.md` diff --git a/docs/exec-plans/active/20260626-record-and-replay-event-stream.md b/docs/exec-plans/active/20260626-record-and-replay-event-stream.md new file mode 100644 index 00000000..52a504f5 --- /dev/null +++ b/docs/exec-plans/active/20260626-record-and-replay-event-stream.md @@ -0,0 +1,578 @@ +# Record & Replay Event Stream + +## 目标 + +在 `open-codex-computer-use` 内复刻官方 Record & Replay 的 macOS event-stream 能力，优先对齐官方 `record-and-replay` 1.0.857 的 MCP surface、文件产物和事件语义，再补充独立于 Codex.app 的录制控制条、等待命令和结束通知能力。后续可以把薄插件和 skill 单独放到一个 repo，只依赖本仓库提供的 runtime。 + +## 范围 + +本阶段只覆盖 macOS。目标是让使用方可以通过 `open-computer-use event-stream mcp` 获得官方兼容的 3 个 MCP tool，并可以通过本仓库扩展 CLI 控制录制生命周期。 + +包含： + +- 官方兼容 MCP tools：`event_stream_start`、`event_stream_status`、`event_stream_stop`，保持无参数 surface。 +- 本地 CLI 扩展：`event-stream start/status/stop/wait/validate/summarize --json`，用于非 MCP 调用方和独立 skill/plugin 集成。 +- Codex 安装便利命令：`install-codex-record-and-replay-mcp`，写入独立 `record_and_replay` server，启动 `open-computer-use event-stream mcp`。 +- 录制产物：`events.jsonl`、`metadata.json`、`session.json`、`suppressed.jsonl`，以及 status 中的路径字段。 +- 事件采集：鼠标、键盘文本输入、AX 事件、app/window attribution、截图上下文标记。 +- 官方风格 AX compact diff：支持 full tree、diff from previous、cumulative diff，并保留必要的降级策略。 +- OCU 自有录制控制条：显示录制中、可拖拽、Stop/Cancel、计时，并由 OCU 自己接收交互。 +- 等待和通知：录制结束后唤醒 `wait` 调用方，供上层 skill 或插件继续生成后续资产。 + +不包含： + +- Windows / Linux 录制能力。 +- 与 Codex.app 私有 UI、feature flag 或插件宿主耦合。 +- 直接复制官方私有实现代码。 +- HTTP webhook 或复杂多 listener 协议；当前第一版已包含本地 `wait --notify-command` callback baseline，但它属于 OCU 扩展层，不进入官方兼容 MCP surface。 + +## 背景 + +官方 `record-and-replay` 插件本质上仍依赖 `Codex Computer Use.app` 内的 `SkyComputerUseClient`，只是以 `event-stream mcp` 子命令启动另一组 MCP tools。逆向上下文已沉淀在： + +- `docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md` +- `docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json` +- `docs/design-docs/record-and-replay-replication.md` + +关键事实： + +- 官方插件路径为 `openai-bundled/record-and-replay/1.0.857/`。 +- 官方 `.mcp.json` 启动命令为 `SkyComputerUseClient event-stream mcp`。 +- 官方普通 `computer-use` 插件使用同一个 client，只是参数为 `mcp`。 +- `Codex.app` asar 里只观察到 `recordAndReplay` feature flag、插件可见性和通用 MCP elicitation/approval 逻辑，没有发现 `event_stream_*` tool 实现或录制控制条文案。 +- 录制控制条、`event_stream_*` tools、`events.jsonl` / `metadata.json` / `suppressed.jsonl` 字符串均出现在 `Codex Computer Use.app` 相关二进制中。 +- 官方二进制字符串包含 `Record & Replay approval cancelled via MCP elicitation.` 和 `Record & Replay approval denied via MCP elicitation.`，说明 start 前 approval 有 runtime 可观察错误语义；但独立 OCU 不能依赖 Codex.app 的通用 panel，因此需要自有 start approval gate 或 UI。 +- 当前已保存官方 1.0.857 的 `initialize` / `tools/list` normalized fixture；它不包含录制输出，仍不能替代后续 official golden recordings。 +- 当前已新增 raw probe 脚手架 `scripts/probe-event-stream-recording.py`；本机官方 1.0.857 在宿主外可返回 initialize / tools-list，但 raw `event_stream_start` / `event_stream_status` / `event_stream_stop` 仍超时，没有产出 official recording paths。该 timeout transcript shape 已脱敏保存为 `docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json`。 +- 2026-06-27 通过 Codex-hosted Record & Replay MCP tool 观察到 no-active `event_stream_status` / `event_stream_stop` 的 text JSON 都是 `{"isRecording": false, "maxDurationSeconds": 1800}`，没有 `state` / `active` / path 字段；该 response shape 已保存为 `docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json`，并用于校准 OCU MCP idle/no-active 返回。 + +## 方案 + +### 1. 固化官方样本 + +先用当前安装的官方 1.0.857 作为 baseline，采集最小录制样本： + +- 正常 stop：点击、键盘输入、窗口切换各至少一次。 +- cancel：通过官方控制条取消一次。 +- 无 active session 时调用 status/stop。 +- active session 时重复 start。 + +样本要作为 golden fixtures 保存到 `docs/references/` 或 `test-fixtures/` 的脱敏目录中，记录官方版本号、采集日期和脱敏方式。如果后续需要拉取 LFS 最新 bundle，应保留版本化对比，而不是覆盖 baseline。 + +### 2. 增加 event-stream 子命令 + +在现有 CLI 分层上增加： + +- `open-computer-use event-stream mcp` +- `open-computer-use event-stream start --json` +- `open-computer-use event-stream status --json` +- `open-computer-use event-stream stop --json` +- `open-computer-use event-stream cancel --json` +- `open-computer-use event-stream wait --json [--session-id ...] [--timeout ...] [--notify-command '']` +- `open-computer-use event-stream validate --json [--strict-ocu] [--require-event-type ...] [--require-skill-draft] ` +- `open-computer-use event-stream summarize --json [--include-text] [--require-action] ` +- `open-computer-use event-stream scaffold-skill --json --skill-name --output-dir [--description ] [--overwrite] [--include-text] ` + +MCP 模式只暴露官方兼容 3 tools。扩展命令留在 CLI，不污染官方兼容 surface。 + +### 3. 建立 session 文件模型 + +建议每次录制创建独立 session 目录，包含： + +- `events.jsonl`：append-only 事件流，作为主产物。 +- `metadata.json`：session id、start/end 时间、endReason、event count、路径和状态。 +- `session.json`：与 `metadata.json` 内容一致的兼容 alias；官方 skill 文案和二进制字符串都提到过 `metadata.json or session.json`。 +- `suppressed.jsonl`：被限流、合并、隐私策略或体积策略丢弃的事件。 +- `screenshots/`：仅在 `AX.focusedWindowChanged` payload 判断需要额外视觉上下文且当前 snapshot 有截图数据时写入的 PNG，不作为每个事件的默认产物。 +- `latest-session.json` / `active-session.json`：OCU 扩展索引文件，供 CLI `status` / `wait` 在非 MCP 场景下查找当前或最近 session。 + +`event_stream_start` 如果已有 active session，应返回当前 active session，而不是新建。agent-facing skill 不能把这个结果静默当成本次新 demonstration；它应该告知已有录制正在进行，并询问用户要使用这个 active recording 还是等待它结束。`event_stream_status` 返回当前或最近一次 session。`event_stream_stop` 要 idempotent，无 active session 时返回最近状态。 + +`wait` 用作 OCU 扩展 listener：通过 app-agent 访问长驻 recorder 状态时，`status/stop/cancel/wait` 不应持有全局环境覆盖锁，避免一个长时间 `wait` 阻塞另一个连接上的 `stop` / `cancel`。`finish()` 写最终 metadata 前也不能先清空 active session，否则并发 `wait` 可能读到旧的 active metadata 并误报 `recording_process_unavailable`。`wait --json` 返回体额外包含 `waitTimedOut` 和 `waitSessionMatched`，这是 OCU 扩展字段，不属于官方兼容 MCP surface。`wait --session-id ` 会优先读取目标 session 目录里的完成态 metadata，即使 latest session 已被后续录制覆盖，也能返回指定历史 session；如果指定 id 不存在或当前 active session 明显不匹配，会返回 `waitTimedOut=true` / `waitSessionMatched=false`，避免独立集成方无限等待错误 id。`wait --notify-command ''` 也是 OCU 扩展：仅当 `waitTimedOut=false` 时同步执行本地回调，stdin 写最终 status JSON，并通过 `OPEN_COMPUTER_USE_EVENT_STREAM_*` 环境变量暴露 session、state、endReason、metadataPath、sessionPath 和 eventsPath；回调默认 10 秒超时，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS` 覆盖，失败会让 CLI 返回 error 并在 JSON 里写入 `notification`。`validate --json`、`summarize --json` 和 `scaffold-skill --json` 是本地只读/文件生成命令，不通过 app-agent 代理；前者用于检查 session 文件一致性、required event type，以及可选 `--require-skill-draft` 的 skill 草稿最低证据门槛，中者用于生成 skill 前快速审计事件流，后者把可审计 summary 写成第一版 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`。`--require-skill-draft` 要求至少一个高层动作、recording 已完成、未取消、`session.started` 只出现一次且为第一条事件、`session.ended` 只出现一次并收尾，且没有 `inputMonitorsUnavailable` 等阻断性诊断；active recording、`active=true`、缺少 `session.started`、`session.started` 非首事件或缺少 `session.ended` 的 event stream 都会被拒绝。它允许其它 `needsReview` 的 recording 进入草稿生成，避免把尚待官方校准的 AX / screenshot 细节误设成硬门槛。`sessionPath` 是 OCU 便利字段，指向 `metadata.json` 的 `session.json` alias；官方兼容层仍以 `metadataPath` / `eventsPath` 为主。 + +### 4. 复刻事件语义 + +第一版事件类型优先覆盖官方已观察字符串： + +- `session.started` +- `mouse.click` +- `mouse.context_menu` +- `mouse.drag` +- `keyboard.text_input` +- `keyboard.submit` +- `keyboard.shortcut` +- `terminal.value_changed` +- `selection.changed` +- `AX.focusedWindowChanged` +- `debug.error` +- AX tree full snapshot +- AX tree compact diff + +每个事件应带 timestamp、app/window attribution、必要的 AX context，以及是否需要截图上下文的标记。截图不应默认每个事件都采集，应按官方线索使用 `screenshotNeededForContext` 类字段触发。 + +OCU 当前输入事件上下文： + +- `mouse.click` 写入 y-down screen-state `location`，同时保留原始 AppKit `appKitLocation` 供后续校准。 +- `mouse.click` 会尽量通过 `AXUIElementCopyElementAtPosition` 写入 `targetAccessibilityElement`，包含 role/title/label/value/selectedText/frame/windowFrame/actions 等轻量字段。 +- 右键点击按官方字符串记录为 `mouse.context_menu`，同时保留 `button=right` 和目标 AX 摘要。 +- `keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut` 会尽量从 system-wide focused element 写入 `focusedAccessibilityElement`，用于识别文本输入、提交和快捷键目标。 +- 官方字符串中还出现 `window.changed`、`selection.changed`、`mouse.drag`、`terminal.value_changed`、`debug.error`、`secureInput` 和 `experimentalRawEvents`；当前 baseline 已补 `window.changed`、`mouse.drag`、轻量 terminal value changed、轻量 selection changed 与 selection clear、低敏 debug error、Input Monitoring monitor 全部安装失败时的 `inputMonitoring/inputMonitorsUnavailable` debug event、secure input 脱敏和低层 raw event 摘要，但 window / drag / terminal / selection / debug / input monitoring failure / secure input / raw 字段结构仍需 official golden recording 校准。没有观察到明确 scroll event 名称，OCU 暂时只把 scroll wheel 写成 raw-only `experimentalRawEvents` 事件，不臆造 scroll event type。 +- target/focused summary 只作为可消费上下文；命中失败不阻断录制。 + +OCU 当前截图策略： + +- `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never` 控制 Record & Replay 截图上下文，默认 `auto`。 +- `always`：只要 snapshot 带有 screenshot PNG，就为 AX payload 写入 `screenshots/--context.png`。 +- `never`：只保留 AX payload，不写截图。 +- `auto`：当 AX tree 很短或缺少 focused summary 时认为上下文不足，标记 `screenshotNeededForContext=true` 并尝试写截图。 +- payload 会包含 `screenshotNeededForContext`、`screenshotAvailable`，写入成功时额外包含 `screenshotPath`。 +- 如果策略要求截图但当前没有截图数据，或写文件失败，事件不中断；写文件失败会进入 `suppressed.jsonl`。 + +### 5. 复刻 AX compact diff + +实现目标是可用官方样本做 golden comparison，而不是只实现概念相似的 diff。 + +已观察到的官方 diff 语义： + +- `diffFromPrevious` +- `DifferenceBaseline` +- `UIElementRenderDifference` +- `UIElementRenderDifferenceBuffer` +- `The following is a diff from the previous accessibility tree` +- `The following is a cumulative diff from the initial accessibility tree` +- compact render 使用 `~`、`+`、`-` 表示 changed / added / removed。 +- diff 过大或 removed element summary 超预算时，官方会跳过 diff render 并回退。 + +OCU 实现应先复用现有 AX snapshot renderer，再增加稳定 element identity、tree diff 和 line budget。无法稳定 diff 时回退 full tree，并在 metadata 或 suppressed stream 中记录原因。当前 baseline 已输出 previous diff，并额外输出从 initial tree 到 current tree 的 cumulative diff 字段；字段名、line budget 和算法仍待 official golden recording 校准。 + +### 6. 增加 OCU 自有控制条 + +因为本仓库不能依赖 Codex.app 替我们绘制录制 UI，macOS runtime 需要自己绘制浮层： + +- 显示 “Record & Replay is recording your actions” 或等价文案。 +- 提供 Stop、Cancel、timer、拖拽移动。 +- Stop 结束为 `recording_controls_stopped`。 +- Cancel 结束为 `recording_controls_cancelled`，并保留 cancellation 状态供上层 skill 识别。 +- 控制条点击直接进入 OCU runtime，不通过 Codex.app。 + +控制条应只在交互式录制时显示；headless 或 fixture 模式允许关闭。 + +### 7. 补 start approval gate + +官方 skill 在流程上要求用户准备好后才调用 `event_stream_start`，官方 runtime 字符串也确认存在 approval cancelled / denied 错误。OCU 需要把这个流程落在自己这边，但不能改官方兼容 MCP surface。 + +第一阶段建议做成 runtime 内部 gate： + +- `event_stream_start` 仍保持无参数。 +- 已有 active session 时直接返回 active status，不重复弹 approval。 +- 无 active session 时先进入 approval gate，只有 approved 后才创建 session、写 `session.started`、安装事件 monitor 和显示录制控制条。 +- denied / cancelled 不创建 session，MCP 返回 error result；错误文案尽量对齐官方已观察字符串。 +- 自动化和 smoke 默认可配置为 auto-approve，避免本地 CI 因 UI 阻塞。 + +第二阶段再接 macOS OCU 自有开始确认 UI： + +- UI 由 OCU runtime 绘制，不依赖 Codex.app。 +- 确认 UI 和录制中控制条是两个阶段：前者决定是否开始，后者负责 Done / Discard。 +- 如果宿主支持 MCP elicitation，可以在 gate 内优先走 MCP elicitation；宿主不支持或独立 CLI 使用时回退 OCU 自有 UI / policy。 + +验收点： + +- approval approved / denied / cancelled 均有单测。 +- denied / cancelled 时没有 session 目录和 active session。 +- interactive 手工 smoke 能看到 OCU 自有开始确认，确认后才开始记录事件。 +- MCP tools/list 不因 approval 增加任何参数或工具。 + +## 风险 + +- 官方 JSON schema 仍需要实际样本校准，不能只靠 strings 推断字段。 +- AX diff 的 element identity、removed range summary 和 line budget 可能影响 replay skill 生成质量。 +- 录制会涉及 Accessibility、Screen Recording、Input Monitoring 等系统权限，权限缺失或 input monitor 安装失败时必须给出可恢复状态；当前 baseline 会用低敏 `debug.error` 标记 input monitor 完全不可用。 +- 截图采集有隐私和体积风险，需要默认最小化并保留 suppressed 记录。 +- 浮层窗口可能被不同 Space、全屏应用或多显示器场景影响，需要单独 smoke。 + +## 里程碑 + +- [ ] 采集并脱敏官方 1.0.857 golden recordings。 +- [x] 增加 `event-stream mcp` 和官方兼容 3 tools。 +- [x] 增加 session 文件模型与 CLI `start/status/stop/wait`。 +- [x] 接入鼠标、键盘、AX focused window 基础事件。 +- [x] 增加 input monitor handler 级回归，验证 mouse/key handlers 会把 `mouse.click` / `keyboard.text_input` 写入 `events.jsonl`。 +- [x] 接入轻量 `selection.changed` baseline。 +- [x] 接入轻量 `terminal.value_changed` baseline。 +- [x] 接入轻量 `debug.error` baseline。 +- [x] 实现 OCU 自有录制控制条及 stop/cancel endReason。 +- [x] 接入 AX full payload 与轻量 `~` / `+` / `-` compact diff fallback。 +- [x] 接入轻量 cumulative AX diff baseline。 +- [x] 增加 AX payload 服务级回归，验证同一 app/window context 下 `AX.focusedWindowChanged` 会把 full payload、previous diff 和 cumulative diff 写入实际 `events.jsonl`。 +- [x] 实现截图上下文策略，并支持 `auto|always|never` 环境开关。 +- [x] 增加截图上下文服务级回归，验证 AX payload 需要截图时会把 PNG 写入 session `screenshots/` 并在 `screenshotPath` 中引用。 +- [x] 实现默认 30 分钟最长录制限制，并为测试/本地调试提供 `OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS` 覆盖。 +- [x] 增加 start approval gate，并接入官方 approval cancelled / denied 错误语义 baseline。 +- [x] 接入 OCU 自有开始确认 UI baseline，保证独立使用方不依赖 Codex.app。 +- [x] 接入 MCP elicitation approval baseline；host 声明 `capabilities.elicitation` 时通过 `elicitation/create` 请求用户确认，不改变官方 3 个 no-arg tools。 +- [x] 增加 runtime `event-stream validate --json` / `summarize --json`，让独立 skill/plugin 在安装态也能校验和审计 recording，不依赖源码脚本。 +- [x] 增加 `event-stream validate --require-skill-draft`，让独立 skill/plugin 在 scaffold 前可显式拒绝无动作、未完成、已取消、`session.started` 起点异常、`session.ended` 收尾异常或存在阻断性诊断的 recording。 +- [x] 增加仓库内独立 `open-computer-use-record-and-replay` thin skill，并让 skill 打包脚本自动扫描多 skill，为后续独立 repo 验证提供可安装 baseline。 +- [x] 增加源码 checkout 内的 `scripts/scaffold-event-stream-skill.py`，把 validated recording summary 转成可审阅的 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`，让 recording-to-skill handoff 有可复跑脚手架。 +- [x] 增加 runtime `event-stream scaffold-skill --json`，让安装态 OCU 也能从 recording 生成第一版 skill 草稿，不要求独立 plugin / skill repo checkout 本仓库源码。 +- [x] 固定取消录制的 recording-to-skill guard：`recording_controls_cancelled` / cancelled metadata 会让 summary 输出 `skillReadiness.status=insufficient` 和 `canCreateSkillDraft=false`，源码 Python scaffold 与 runtime CLI scaffold 都拒绝生成 skill 草稿。 +- [x] 把真实输入 action smoke 的 simple / drag OCU candidate 纳入完整 baseline audit：默认 `mixed-action-stop` 硬门槛覆盖 click、AX context、MCP transcript 和 recording-to-skill handoff，并继续 best-effort 发送 scroll wheel；捕获到的 scroll raw-only 进入 summary/scaffold，但不作为 release hard gate。`OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop` 分别覆盖 click / drag 同场景候选，最终摘要通过 `checkedSimpleActionStopCandidate` / `checkedDragStopCandidate` 作为 `usableBaseline` 必需 evidence。 +- [x] 增加独立 Record & Replay surface drift smoke，用官方 1.0.857 non-recording fixture 校验 OCU `event-stream mcp`，并支持 opt-in 复核本机官方 bundle。 +- [x] 增加独立 Record & Replay no-active drift smoke，用 Codex-hosted 官方 1.0.857 no-active fixture 校验 OCU MCP `event_stream_status` / `event_stream_stop` 的最小 text JSON，并确认该路径不会创建 session 文件。 +- [x] 固定官方三件套 no-arg runtime contract：`event_stream_start/status/stop` 的 MCP `params` 必须是 object，`params.name` 必须是非空 string，`arguments` 必须是 object 且为空；缺失 / 非 object params、缺失 / 非 string tool name、非 object 或非空 object 参数返回 tool error 且不创建 session 文件。 +- [x] 增加 `open-computer-use install-codex-record-and-replay-mcp`，让 Codex 用户可一键安装独立 Record & Replay MCP surface。 +- [x] 增加 standalone thin skill repo scaffold，可生成只依赖 `open-computer-use` runtime 的独立 Record & Replay skill 仓库骨架，并内置 package / runtime contract verifier / self-check 门禁 / CI workflow。 +- [ ] 用官方 golden fixtures 校准 AX compact diff 到官方算法级行为。 +- [ ] 基于 golden fixtures 做 schema / 行为对齐测试。 +- [ ] 创建真实独立薄插件/skill repo 并发布/安装验证，只依赖 OCU runtime 即可工作。 + +## 后续推进入口 + +后续工作按“先官方样本、再 OCU 校准、最后薄插件/skill 集成”的顺序推进。每一步都应该把观察事实和实现差异继续写回本 plan 或 reverse-engineering reference，避免只依赖聊天上下文。 + +下一轮开始时先读 `docs/design-docs/record-and-replay-handoff.md`。该文档是本 plan 的短入口，固定了当前决策、官方兼容层 / OCU 扩展层边界、待官方校准 baseline 的措辞和每次推进必须遵守的证据、实现、验证、文档同步闭环。 + +当前 baseline 推进基准： + +- 先交付可用的 macOS baseline：官方三件套 MCP surface、session 文件、基础事件、AX payload / compact diff baseline、按需 screenshot context、OCU 自有开始确认与录制控制条、wait/notify、validation、summary、skill scaffold 和 standalone repo scaffold。 +- baseline summary 必须能区分 `usableBaseline` / `standaloneRepoBaselineReady` 和 `officialSuccessfulRecordingEquivalenceReady`。缺 official successful recording golden 时，前两者可以为 true，但后者必须为 false。 +- 完整 baseline audit 不只看默认真实输入 smoke 是否产出 `skillPath` / `mcpTranscriptPath`，还必须消费 `checkedMcpResponseShapesCaptured`、`checkedSkillReadinessCanCreateDraft`、`checkedSkillCreatorFinalizationHandoff`、`checkedGeneratedSkillPathRedaction`、`checkedSimpleActionStopCandidate` 和 `checkedDragStopCandidate`。这证明 OCU candidate 采样、MCP transcript 和 recording-to-skill handoff 可用，但仍不能宣称 official successful recording schema 等价。 +- 拆独立 repo 前，默认自检只验证 package、runtime contract、skill workflow、wait/notify 和 synthetic recording-to-skill；真实 recording lifecycle smoke 保持 opt-in。 +- 任何要宣称官方等价的事件字段、AX compact diff 算法、screenshot 触发策略或 endReason，都必须先有 official golden recording 或官方 fixture 证据，并能用脚本和 OCU candidate 对比。 + +需要为 release 或 standalone repo audit 留存 baseline 机器证据时，使用 `scripts/run-record-and-replay-baseline-smoke.sh --summary-json `。它仍打印同一份最终 JSON 到 stdout，并保持默认 / 严格 official golden gate 的退出码；runner 会在生成 summary 后自动调用 `scripts/check-record-and-replay-baseline-summary.py` 审计同一份摘要，audit 输出走 stderr。落盘后也可用 `scripts/check-record-and-replay-baseline-summary.py ` 做默认 artifact 复审；审计会固定顶层 `baseline=record-and-replay`，要求顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致，要求顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查，并直接检查 event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence；只有 official successful recording golden 入库并要求官方等价时，才使用 `scripts/check-record-and-replay-baseline-summary.py --require-official-golden`。 + +### P0：官方 golden recordings + +目标是把官方 observable behavior 固化成可比较样本，而不是继续只靠 `strings` 推断。 + +- 使用当前官方版本录制最小 stop / cancel / repeat start / no active stop / status 样本。 +- 如需拉取更新版本，应保留版本号、采集日期、系统版本和脱敏说明，不覆盖 1.0.857 baseline。 +- 样本至少覆盖 `events.jsonl`、`metadata.json` 或 `session.json`、`suppressed.jsonl`、MCP start/status/stop response。 +- 样本保存前要脱敏 app title、用户输入、窗口内容和截图；无法脱敏的原始数据不进仓库。 +- 官方样本入库使用 `scripts/import-event-stream-fixture.py --name --source official --official-plugin-version ... [--mcp-transcript ]`，默认把已知 metadata path 改成 fixture 相对路径，替换用户文本 / 标题 / timestamp / session id / app attribution，并移除 screenshot path；对于事件或 suppressed event 里的通用 `path` / 未知 `*Path` 字符串只保留 redacted marker 和长度，避免本机路径通过非标准字段漏进 fixture。如果传入 MCP probe output，会额外生成脱敏 `mcp-transcript.json`，保留 `event_stream_start/status/stop` response shape 作为官方对外行为证据。 +- 样本落盘后先用 `open-computer-use event-stream validate --json ` 或 `scripts/validate-event-stream-recording.py ` 做结构检查；OCU 自身产物用 `--strict-ocu`，官方样本先不启用该开关，避免把 OCU alias/index 约束误当成官方事实。独立 skill/plugin 集成优先调用 runtime CLI，源码脚本只作为仓库内交叉检查。 +- 结构检查通过后，用 `scripts/check-event-stream-golden-readiness.py --require-fixture-manifest --require-source official --require-official-plugin-version ... --require-session-alias --require-metadata-counts --require-handoff-paths --require-mcp-transcript --require-mcp-tool-response event_stream_start --require-mcp-tool-response event_stream_status --require-mcp-tool-response event_stream_stop` 判定样本是否足够作为动作类 golden；默认要求 final state、同时存在的 `metadata.json` / `session.json` 内容一致、已声明的 `eventCount` / `suppressedEventCount` 与文件行数一致、已声明的 handoff path 指向存在文件、`session.started` 恰好一次且为首事件、`session.ended` 恰好一次且为最终事件、至少一个动作事件和 AX payload，`--require-session-alias` 进一步要求两份 alias 文件都存在，`--require-metadata-counts` 进一步要求两项 count 字段都存在，`--require-handoff-paths` 进一步要求 fixture metadata 中的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 四个字段都存在并可解析。注意这四个字段是脱敏 fixture 的归一化合同，不要求 official hosted 原始 JSON 同时返回 `suppressedEventsPath`；官方最小包可只有 `eventsPath` 和 `metadataPath` / `sessionPath`，导入器会补空 `suppressed.jsonl`。若样本要校准 AX / screenshot / 压缩 fallback，可额外要求 `--require-suppressed-events --require-suppressed-event-type `，确保 suppressed evidence 不只停留在文件里。若样本来自 raw probe 并要校准 one-active、idempotent stop 或 completed status，可额外要求 `--require-mcp-response-shape repeatStartResponseShape --require-mcp-response-shape repeatStopResponseShape --require-mcp-response-shape finalStatusResponseShape`，确保这些 completed lifecycle 证据没有只停留在 transcript 里。tool 级 `--require-mcp-tool-response event_stream_start` 表示该 tool 至少有一次 result/error response；repeat shape 的 timeout 不会覆盖已成功的 start tool 证据，具体 repeat 语义必须用 `--require-mcp-response-shape` 单独要求。no-active status/stop 等生命周期样本需要显式加 `--allow-no-action --allow-no-ax-payload`，并按场景只要求对应 MCP response，避免把非动作样本误当成可校准事件语义的 golden。若 transcript 中 required tool 或 response shape 只有 timeout shape，readiness gate 会明确报 `MCP tool timed out` 或 `MCP response shape timed out`，该样本只能作为宿主外边界证据，不能当成成功 response golden。 +- 官方样本与 OCU 候选录制的差异用 `scripts/compare-event-stream-recordings.py --require-same-event-sequence --require-same-metadata-keys --require-same-metadata-values --require-handoff-paths --require-final-session-evidence --require-same-schema --require-semantic-fields --require-ax-diff-evidence` 量化；校准过程中可以先去掉 require 开关只看 JSON diff。`--require-same-metadata-keys` 守住 metadata/session 文件布局键；`--require-same-metadata-values` 守住 `state`、`active`、`endReason`、`eventCount`、`suppressedEventCount` 这类稳定状态值，避免 cancel / stop / count 漂移只因 key set 一致而漏掉；`--require-handoff-paths` 不比较路径字符串本身，但要求 baseline 与 candidate 都声明 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 且解析后文件存在，避免 candidate 保留 key 却丢掉可消费路径；`--require-final-session-evidence` 因兼容性保留原 flag 名，但现在守住 start/end lifecycle evidence：`session.started` 个数、首事件位置、`session.ended` 个数、最后事件位置和 endReason 集合，避免录制边界丢失；`--require-semantic-fields` 只盯 replay-critical 字段，后续官方 fixture 入库后可在不要求完整 schema 一字不差的情况下，先保证 target AX、drag 起止点、keyboard 目标、AX treeLines / cumulative treeLines 等关键证据没有丢失。若官方样本包含 `suppressed.jsonl` 降级 / 压缩 / 截图失败证据，可额外打开 `--require-same-suppressed-event-sequence --require-same-suppressed-schema`，要求 candidate 保留 suppressed stream 的事件顺序和 schema。若官方 fixture 和 OCU candidate 都带 `mcp-transcript.json`，可额外打开 `--require-mcp-response-shapes --require-same-mcp-response-schema`，要求 candidate 保留 baseline 中出现的 start/repeat/status/stop/final response shape、result/error/timeout 状态和 response schema。脱敏导入官方 fixture 时会保留 `treeLines` / `cumulativeTreeLines` 的 `+` / `-` / `~` marker，但移除原始文本，保证 AX diff 结构可比较且不泄漏内容。 +- 采集前可先运行 `scripts/probe-event-stream-recording.py --target official` 确认官方 bundled client 的 non-recording surface 可用；如果要尝试 raw start/status/stop，用 `scripts/probe-event-stream-recording.py --target official --start-stop --timeout --output `，脚本会记录 transcript，并在 start 返回或超时时尝试 status 和 stop。当前本机宿主外官方 raw start/status/stop 仍超时，因此不能把该路径当作 golden recording 采集成功条件。 + +### P1：官方兼容层校准 + +目标是让 `open-computer-use event-stream mcp` 对外继续保持官方三件套，同时把字段、状态流转和错误语义尽量对齐官方。 + +- MCP surface 只保留 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三个无参数 tools。 +- 校准 start/status/stop 返回 JSON 的字段名、嵌套结构、active/completed 状态和 idempotent stop 行为；no-active status/stop 已按 Codex-hosted 官方观察校准为 `isRecording=false` / `maxDurationSeconds=1800` 的最小 text JSON。 +- 校准正常 stop、控制条 cancel、重复 start、无 active session stop、录制进程不可用等 endReason。 +- 已补 30 分钟最长录制限制；官方二进制字符串只确认 “up to 30 minutes” 和内部 timeout task，未确认对外 endReason，因此 OCU baseline 暂用 `recording_time_limit_reached` 并标记待 official golden 校准。 + +### P1.5：Start approval 与独立 UI + +目标是复刻官方 start 前 approval 的可观察语义，同时满足 OCU 脱离 Codex.app 独立运行的要求。 + +- 在 `EventStreamService.start()` 前加内部 approval gate，沉淀 approved / denied / cancelled 三种结果。已完成 baseline。 +- denied / cancelled 使用官方已观察错误文案，且不创建 session。已完成 baseline。 +- 测试、smoke 和 headless 路径必须可 auto-approve，避免阻塞自动化。已完成 baseline，使用 `OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=approve`。 +- macOS interactive runtime 接 OCU 自有开始确认 UI；用户确认后再显示录制中控制条。已完成 NSAlert baseline，文案会告知当前录制时限，默认 30 分钟；后续可替换成更贴近官方的浮层样式。 +- MCP elicitation 作为 host-capability 优化，不作为独立 OCU 的硬依赖；baseline 已接入 `OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp|elicitation`，host 声明 `capabilities.elicitation` 时发送内部 `elicitation/create`，accept 才创建 session，decline / cancel / unsupported 均不创建 session。 +- 当前 MCP elicitation 已按 Codex.app asar 里的 host schema 校准到显式 `mode: "form"` + 空对象 `requestedSchema` baseline，action mapping 覆盖 accept / decline / cancel；官方 Record & Replay 真实 start request 的 message、`_meta` 和业务特化 schema 仍需 official golden / runtime 观察校准。 + +### P2：事件和 AX payload 校准 + +目标是让 skill 可以稳定从 `events.jsonl` 还原用户操作意图。 + +- 对齐 `session.started`、`mouse.click`、`mouse.context_menu`、`mouse.drag`、`keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut`、`terminal.value_changed`、`selection.changed`、`AX.focusedWindowChanged`、`debug.error`、`session.ended` 的字段、顺序和 timing。 +- 对齐 click / keyboard 目标摘要字段；OCU 当前的 `targetAccessibilityElement` / `focusedAccessibilityElement` 是可用 baseline，但字段名和内容仍要以官方样本为准。 +- 用官方样本校准 `accessibilityInspectorPayload`、`diffFromPrevious`、cumulative diff、full tree fallback 和 suppressed 记录。 +- AX compact diff baseline 已改成 LCS 风格行 diff，并对 AX tree 常见的 element index 重编号做归一化；已有单测覆盖 changed / added / removed / diff too large fallback。算法仍待官方 golden fixtures 校准到官方 `UIElementRenderDifference` 级别。 + +### P3：OCU 扩展层 + +这层是开源版为了脱离 Codex.app 必须新增的能力，不应污染官方兼容 MCP surface。 + +- `event-stream start/status/stop/cancel/wait --json` 继续作为 CLI/runtime 扩展；`wait --json` 用 `waitTimedOut` 标记 listener 是被结束事件唤醒还是 timeout 返回，用 `waitSessionMatched` 标记返回的状态是否属于调用方指定的 session id。 +- 自有控制条负责显示录制中、计时、Done / Discard、拖动和用户交互回传。 +- 第一版通知模型使用 `wait`：调用方启动录制后可挂起等待，录制结束时返回最终 status。后续如果需要更主动的回调，再在 CLI 或独立 runtime API 上加 callback/webhook，不改官方 3 个 MCP tools。 +- 独立薄插件/skill repo 只依赖 OCU runtime：启动 `event-stream mcp`，按官方 skill 流程开始/结束录制，结束后读取 `eventsPath` / `metadataPath` 生成后续资产。 +- Codex 接入不要求手写 TOML：`open-computer-use install-codex-record-and-replay-mcp` 会写入独立 `record_and_replay` server；普通 9-tool Computer Use 仍由 `install-codex-mcp` 安装。 +- 仓库内已先提供 `skills/open-computer-use-record-and-replay/SKILL.md` 作为 thin skill baseline：它只依赖 `open-computer-use event-stream mcp` 的官方三件套和 OCU CLI `validate` / `summarize` / `scaffold-skill` / `wait --notify-command` 扩展，不依赖 Codex.app 绘制控制条或私有回调。安装态 runtime CLI 可把录制摘要生成第一版 skill 草稿，源码 checkout 中的 `scripts/scaffold-event-stream-skill.py` 保留为交叉检查；两者仍要求 agent 人工检查 `events.jsonl`、补业务输入和确认高风险动作。 +- `scripts/scaffold-record-and-replay-skill-repo.py --output-dir ` 可把仓库内 thin skill baseline 导出成 standalone repo 骨架；npm 包也暴露同等安装态命令 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `，包内会携带脚本和 `skills/open-computer-use-record-and-replay/` 源，不要求使用方 checkout 本仓库。安装态 launcher 对该命令提供专用 help，说明它生成 standalone repo 而不是修改 MCP/plugin config；运行时会优先使用 `PYTHON`，否则在 POSIX 上尝试 `python3` / `python`，在 Windows 上尝试 `python`，缺失 Python 3 或候选版本不是 Python 3 时返回专用诊断。生成骨架包含 `skills/open-computer-use-record-and-replay/SKILL.md`、README、package script、`scripts/verify-runtime.py`、`scripts/verify-skill-workflow.py`、`scripts/recording-to-skill-smoke.py`、组合门禁 `scripts/check.sh`、可选 `scripts/recording-lifecycle-smoke.py`、`.github/workflows/ci.yml` 和 `record-and-replay-skill-repo.json` runtime contract；生成物不复制 OCU runtime 源码，安装说明要求使用方先安装 `open-computer-use` 并运行 `open-computer-use install-codex-record-and-replay-mcp`。生成态 `record-and-replay-skill-repo.json` 用 `checks` 声明默认 package/runtime/skill-workflow/recording-to-skill/self-check，用 `optionalChecks` 声明真实录制 lifecycle smoke，并写入 no-active status/stop response contract，避免独立 repo CI 默认启动录制。生成态 `scripts/package-skill.sh` 会扫描 `skills/*/SKILL.md`，要求 frontmatter `name` 匹配目录名且 `description` 非空，再打出 `.zip` / `.skill`；`scripts/verify-runtime.py` 不启动录制，只通过 JSON-RPC initialize / `notifications/initialized` / `tools/list` / no-active `tools/call` 验证 server name、protocol version、initialize capabilities、无 `instructions` initialize response、官方三件套 tool metadata（description、empty input schema、annotations）和 no-active status/stop response shape；`scripts/verify-skill-workflow.py` 不启动录制，只检查 thin skill 保留官方 handoff 语义：只用三件套 MCP、start 后结束 turn、不轮询、用户完成后 stop、Discard/cancel 后不再 stop、读取返回路径并先跑 validator/scaffold gate；`scripts/recording-to-skill-smoke.py` 不启动录制，而是用临时合成 completed recording 验证安装态 `validate --strict-ocu --require-skill-draft`、events-only validation gate 和 runtime `scaffold-skill` 草稿生成路径；`scripts/check.sh` 串起 skill 打包、runtime contract verification、skill workflow verification 和 synthetic recording-to-skill verification，作为生成 repo 发布或安装前的最小自检入口；生成 workflow 安装发布版 `open-computer-use` 后运行同一 `check.sh`，并继续 pin GitHub Actions 到 40 位 SHA。`scripts/recording-lifecycle-smoke.py` 不进入默认 `check.sh`，但可在具备本地桌面权限的环境中显式运行：它会通过同一 `event-stream mcp` 进程 start/repeat start/status/stop/repeat stop/final status 一个最小 recording，确认 one-active、idempotent stop 和 completed status 语义，并用安装态 `event-stream validate --strict-ocu` 校验 `metadata.json`、`session.json` 和 `events.jsonl`。 + +## 验收门槛 + +进入独立 repo / skill 集成前，至少满足： + +- `tools/list` 与官方当前 baseline 的 tool name、description、input schema、annotations 对齐。 +- OCU golden tests 能用官方样本约束 session 文件布局、MCP response 和关键事件 schema。 +- OCU smoke 能稳定覆盖 start/status/stop、session alias、AX payload、可选截图上下文和 `wait` listener。 +- `open-computer-use event-stream validate --json` 和 `scripts/validate-event-stream-recording.py` 能同时校验 OCU smoke 产物和脱敏后的官方 golden recording 基本结构，包括 JSONL 可解析、event count、suppressed count、`session.started` 是否为第一事件、`session.started` 是否只出现一次、final `session.ended`、`session.ended` 是否为最后事件、`session.ended` 是否只出现一次、`endReason`、`screenshotPath` 文件存在性且必须位于当前 session 目录内，以及 strict OCU 下 active recording 必须带可存在的 `currentSegmentEventsPath` / `currentSegmentMetadataPath`、final recording 不能残留这两个 current segment 字段、完整 session 必须声明并解析存在的 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath`；非 strict 模式也可直接接收 `events.jsonl`，用于只有主事件流时的官方样本或 handoff 校验，并会从 `session.ended.endReason=recording_controls_cancelled` 推断取消态；对准备进入 skill scaffold 的 recording，可额外使用 `--require-skill-draft` 要求高层动作证据、已完成、未取消、`session.started` 只出现一次且为首事件、`session.ended` 只出现一次并收尾且没有阻断性诊断。 +- `open-computer-use event-stream summarize --json ` 和 `scripts/summarize-event-stream-recording.py` 能从 OCU 或官方脱敏 recording 中提取事件类型、窗口、action sequence、target/focused element、selection、debug、redaction signals、`skillReadiness`、结构化 `runtimeInputs`、结构化 `safetySignals`、结构化 `summaryLimits`、完成态 `recordingIncomplete`、`sessionStartedNotFirst`、`sessionStartedCountInvalid`、`sessionEndedNotFinal`、`sessionEndedCountInvalid` 和阻断性 `blockingDiagnostics`，作为创建 skill 前的可消费性审计；默认不输出 AX `value` / `selectedText` 原文，只输出长度、目标摘要、确认原因、省略计数和敏感标记，`--include-text` 仅用于人工审查已脱敏样本。 +- `open-computer-use event-stream scaffold-skill --json --skill-name ... --output-dir ...` 能在安装态从可 summarize 的 recording 生成带 workflow readiness、runtime inputs、confirmation signals、summary limits、agent replay procedure、replay steps、verification checklist 和 skill finalization checklist 的 `SKILL.md` 草稿及脱敏 summary；Runtime Inputs 优先来自 summary 的结构化 `runtimeInputs` 字段，Confirmation Signals 优先来自 `safetySignals` 字段，Summary Limits 优先来自 `summaryLimits` 字段，旧 summary 才回退到 action sequence 推断。finalization checklist 明确要求读取并完成 `skill-creator` workflow 和 validation，最终产出实际可发现 skill 目录，而不是停在生成的 Markdown 草稿。未完成、取消录制、缺失 / 重复 `session.started`、`session.started` 非首事件、重复 `session.ended`、`session.ended` 后仍有事件、外部 `screenshotPath` 引用或存在阻断性诊断的 recording 必须被判定为 `insufficient` 并拒绝生成草稿。源码 `scripts/scaffold-event-stream-skill.py --skill-name ... --output-dir ...` 作为 checkout 内交叉检查。`make event-stream-skill-scaffold-smoke` 会同时跑这两条路径，并校验不写入本机绝对路径、确认信号渲染、summary truncation guard、events-only 输入、未完成录制拒绝、取消录制拒绝、缺失 / 重复 `session.started` 拒绝、`session.started` 非首事件拒绝、重复 `session.ended` 拒绝、`session.ended` 非最终事件拒绝、外部 `screenshotPath` 拒绝和阻断性诊断拒绝路径。 +- `scripts/scaffold-record-and-replay-skill-repo.py --output-dir ` 能生成 standalone thin skill repo 骨架，且生成物只包含 skill、README、package script、runtime verifier、skill workflow verifier、recording-to-skill smoke、self-check script、可选 lifecycle smoke、CI workflow 和 runtime contract manifest；`make record-and-replay-skill-repo-smoke` 会校验生成物可打包、不含本机源码绝对路径，并验证生成态 manifest 把 package/runtime/skill-workflow/recording-to-skill/self-check 放在 `checks`，把 lifecycle smoke 放在 `optionalChecks`，同时验证生成态 package script 会拒绝缺失 description 的 frontmatter；随后用当前构建出的 `OpenComputerUse` binary 运行生成 repo 内的 `scripts/verify-runtime.py`、`scripts/verify-skill-workflow.py`、`scripts/recording-to-skill-smoke.py` 和 `scripts/check.sh`，固定 MCP server args/tools、protocol、server name、initialize capabilities、无 `instructions` initialize response、三件套 tool description / schema / annotations、no-active status/stop response shape、thin skill 官方 handoff 语义、安装态 recording-to-skill scaffold 能力，以及 workflow 的 SHA-pinned actions / `check.sh` 入口；同一 smoke 还会显式运行生成态 `scripts/recording-lifecycle-smoke.py`，确认安装态 runtime 可以实际创建最小 session、重复 start 不创建第二个 active session、停止幂等、停止后的 status 指向同一个 completed session，且 session 文件可被安装态 validator 校验。`make npm-record-and-replay-skill-repo-smoke` 额外验证 npm staging 包里的安装态 launcher 命令能生成同一类 standalone repo，并把生成态 `scripts/check.sh` 的 `OPEN_COMPUTER_USE_CLI` 指向 staged launcher 运行，覆盖安装态 package/runtime/skill workflow/recording-to-skill/`skill-creator` handoff；该 smoke 会在安装态 scaffold 前先用 staged launcher 校验 strict `declaredPaths` evidence 和外部 `screenshotPath` 拒绝路径，避免旧 `dist/` native artifacts 晚到生成 repo 自检才失败；该检查依赖本地已有 `dist/` native artifacts，因此作为 opt-in smoke。 +- standalone thin skill repo 的 `scripts/recording-to-skill-smoke.py` 还会验证 `screenshotPath` containment：合成一个外部但存在的 PNG 路径，要求安装态 strict validator 拒绝并保留 `checkedScreenshotPathContainment=true` evidence；生成态 manifest 声明 `recordingToSkill.strictValidation.requiresScreenshotPathsInsideSession=true`，源码 standalone、npm staged 和 baseline smoke 都会读取该 contract / evidence。 +- `open-computer-use install-codex-record-and-replay-mcp` 能幂等写入 `~/.codex/config.toml`，且目标 server 使用 `args = ["event-stream","mcp"]`；`make codex-record-and-replay-installer-smoke` 和默认 CI 覆盖该路径。 +- `scripts/import-event-stream-fixture.py` 能把 OCU 或官方 recording 脱敏导入为 fixture 目录，并可用 `--mcp-transcript ` 额外写入脱敏 `mcp-transcript.json`；脱敏后的 AX `treeLines` / `cumulativeTreeLines` 只保留 `+` / `-` / `~` marker，不保留原始文本；metadata 中已知 path 会改成 fixture 相对路径，事件和 suppressed event 中的通用 `path` / 未知 `*Path` 字符串会改成 redacted marker。`make event-stream-fixture-smoke` 覆盖路径改写、MCP response text JSON 解码脱敏、AX diff marker 保留和敏感内容过滤，并会把导入后的 fixture 继续喂给 golden readiness response-shape / suppressed evidence gate 与 recording compare MCP response shape/schema gate，确认导入产物可被后续官方/OCU 校准链路直接消费。 +- `scripts/check-event-stream-golden-readiness.py` 能把“是否可作为 golden baseline”从人工判断变成 JSON gate：检查 fixture manifest/source/version、final state、metadata/session alias 一致性、metadata event/suppressed counts、declared handoff path evidence、`session.started` 首事件 / 唯一性、`session.ended` 最终事件 / 唯一性、action evidence、AX payload、expected `endReason`、full/diff/cumulative diff 可选要求、suppressed evidence、MCP transcript、指定 `event_stream_*` response 证据，以及 `repeatStartResponseShape` / `repeatStopResponseShape` / `finalStatusResponseShape` 这类 raw probe lifecycle response shape 证据，明确区分 timeout evidence 与 result/error response，以及 lifecycle 样本显式放宽语义；`make event-stream-golden-readiness-smoke` 覆盖 ready、失败、timeout、response shape timeout、缺失 start boundary、duplicate/non-final session end、metadata/session alias 缺失或漂移、metadata/manifest count 缺失或漂移、handoff path 缺失或漂移、endReason mismatch、suppressed evidence 和放宽路径。 +- `scripts/compare-event-stream-recordings.py` 能比较官方 fixture 与 OCU candidate 的 event type sequence、event type counts、metadata key diff、stable metadata value diff、declared handoff path evidence、start/end lifecycle evidence、per-event JSON schema path/type、suppressed event sequence/schema、replay-critical semantic field evidence、AX diff evidence 和可选 MCP response shape evidence；`--require-same-metadata-keys` 会要求 candidate metadata key set 与 baseline 一致，`--require-same-metadata-values` 会要求 candidate 保留 baseline 的稳定 metadata 状态值，`--require-handoff-paths` 会要求 baseline 与 candidate 都声明并保留可解析的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath`，`--require-final-session-evidence` 会要求 candidate 保留 baseline 的 `session.started` 起点与最终 `session.ended` 闭环，`--require-same-suppressed-event-sequence` / `--require-same-suppressed-schema` 会要求 candidate 保留 baseline 的 `suppressed.jsonl` 降级 / 压缩 / 错误证据，`--require-semantic-fields` 会要求 candidate 保留 baseline 中出现的关键 replay 证据字段，`--require-ax-diff-evidence` 会要求 candidate 至少保留 baseline 中出现的 diff/cumulative payload 和 diff marker 类型，`--require-same-ax-diff-markers` 可进一步禁止额外 marker 类型，`--require-mcp-response-shapes` 会要求 candidate 保留 baseline transcript 中的 start/repeat/status/stop/final response shape result/error/timeout 状态，`--require-same-mcp-response-schema` 会进一步要求 response shape schema 不丢 baseline path/type。`make event-stream-compare-smoke` 覆盖成功与失败路径。 +- `scripts/probe-event-stream-recording.py --target local --start-stop` 能对 OCU baseline 产出 start/repeat start/status/stop/repeat stop/final status transcript；`--target official` 能复核官方 non-recording surface，`--target official --start-stop` 用于宿主状态满足时采集 raw start/repeat start/status/stop/repeat stop/final status 证据，但当前不进入默认 CI。probe 现在按 Codex host schema 应答 server-initiated elicitation：accept 带 `content: {}`，decline / cancel 带 `content: null`；`--elicitation-action accept|decline|cancel` 可用于采集不同 approval 分支，`scripts/test-event-stream-recording-probe.py` 用 fake MCP server 覆盖该行为和 request schema 摘要提取。 +- `scripts/test-event-stream-local-probe.py` 能把 local raw probe 变成默认可跑的 smoke：它构建本地 runtime，采集 start/repeat start/status/stop/repeat stop/final status transcript，要求 repeat start 返回同一个 active session、repeat stop 和 final status 返回同一个 completed session、generated fixture 不含本机绝对路径、MCP text JSON 被转换成脱敏 `textJSON`，并检查 active response 的 current segment 字段与 final response 的 current segment 清理语义；同一脚本还会用源码 validator 和 runtime validator 复核最终 session 文件。 +- `scripts/test-event-stream-probe-fixtures.py` 能校验已入库的官方 raw start-timeout fixture：不含本机绝对路径，server/tool surface 正确，start/status/stop 三个 timeout shape 存在且 transcript 中恰好有三次 `tools/call` timeout；同一 `make event-stream-probe-fixture-smoke` 也会运行 fake server probe self-test，覆盖 elicitation response shape 和 summary 提取。 +- 控制条在普通窗口、多显示器、Space 切换和权限缺失场景下有明确行为或明确错误。 +- docs 中清楚标记“已对齐官方”的字段和“OCU baseline / 待官方校准”的字段。 + +## 验证方式 + +- MCP `tools/list` 与官方 1.0.857 对比：tool name、description、inputSchema 和 annotations。 +- 对同一段简单人工操作分别用官方和 OCU 录制，比较 `events.jsonl` 的事件序列、关键字段和文件布局。 +- 覆盖重复 start、无 active stop、正常 stop、cancel、wait timeout、wait after completed。 +- 用 `./scripts/run-event-stream-smoke-tests.sh` 回归 OCU baseline 的 `event-stream mcp`、重复 start、stop 幂等、session 文件、AX payload 和 `session.json` alias；脚本会调用 runtime CLI `open-computer-use event-stream validate --json --strict-ocu` 与源码脚本 `scripts/validate-event-stream-recording.py --strict-ocu` 统一校验生成的录制产物，并用源码脚本 `scripts/summarize-event-stream-recording.py` 与 runtime CLI `open-computer-use event-stream summarize` 验证产物可被后续 skill 生成前置分析读取。默认 lifecycle 会按官方 skill handoff 方式从 `event_stream_stop` 返回 text JSON 中读取 `eventsPath` / `metadataPath` / `sessionPath`，并用 `sessionPath` alias 再跑 runtime validate/summarize。默认跳过截图采集以降低本地环境噪音，需要验证截图上下文时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS=1`，需要验证无 active recording 的 status/stop 时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1`，需要验证 MCP 级自动超时时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT=1`，需要验证 CLI wait timeout 返回体时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1`，需要复核官方 surface 时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1`，该模式只调用官方 initialize/tools-list，默认 10 秒等待官方响应，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT` 覆盖；需要验证 approval 拒绝路径时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=deny|cancel`，需要验证 MCP elicitation approval 时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1`，需要验证 OCU 扩展层的 app-agent 共享状态、`wait` listener、`wait --notify-command` callback 和 `wait after completed` 时设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1`。 +- 用 `make event-stream-smoke-matrix` 跑稳定本地矩阵：默认 lifecycle、no-active、maximum-duration timeout、wait timeout、approval denied/cancelled、MCP elicitation approval、app-agent wait/notify、fixture import 和 recording compare；官方 surface 对比和截图上下文分别用 `OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_OFFICIAL=1`、`OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_SCREENSHOTS=1` 显式加入。 +- 用 `make ci` 跑仓库默认门禁；该入口现在包含 `npm run package:skill`、`scripts/compare-event-stream-surface.py`、`scripts/compare-event-stream-no-active.py`、`swift test` 和稳定 Record & Replay smoke matrix，保证 baseline 不只停留在手工执行单个 smoke。截图上下文和官方 bundle 对比仍然是 opt-in，不进入默认 CI。 +- 用 `swift test --filter testEventStreamServiceRecordsInputMonitorMouseAndKeyboardEvents` 验证 recorder 安装的 input monitor handler 可把构造的 mouse/key `NSEvent` 写入 `events.jsonl`；这覆盖服务级动作录制链路，但不把程序合成输入等价为真实用户硬件输入。 +- 用 `swift test --filter testEventStreamServiceWritesAXFullDiffAndCumulativePayloads` 验证 `EventStreamService` 在实际 session 中写入的 `AX.focusedWindowChanged` 事件可从 full payload 进入 previous diff，并同时带 cumulative diff marker；这覆盖服务级 AX payload 写盘链路，但算法级字段仍待官方 golden fixture 校准。 +- 用 `swift test --filter testEventStreamServicePersistsAXScreenshotContext` 验证 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 等价的服务配置下，可控 snapshot 的 PNG 会写入当前 session `screenshots/`，且 `accessibilityInspectorPayload.screenshotPath` 指向可解析图片文件。 +- 用 `make codex-record-and-replay-installer-smoke` 验证 Codex Record & Replay MCP 安装器会写入独立 `record_and_replay` server，并且重复执行保持幂等；该 smoke 已进入默认 CI。 +- 用 `swift test --filter EventStreamRecordingSummary` 覆盖 summary 的 `skillReadiness`、结构化 `runtimeInputs`、结构化 `safetySignals`、`summaryLimits`、`recordingIncomplete`、`sessionStartedNotFirst`、`sessionStartedCountInvalid`、`sessionEndedNotFinal`、`sessionEndedCountInvalid` 和 `blockingDiagnostics` 判定；用 `swift test --filter EventStreamSkillScaffold` 覆盖 runtime `event-stream scaffold-skill` helper 的单元行为；用 `make event-stream-skill-scaffold-smoke` 同时覆盖源码 Python scaffold 和真实 runtime CLI scaffold。默认 CI 已包含该 smoke，并断言生成草稿含 workflow readiness、runtime inputs、confirmation signals、summary limits、`get_app_state` / `element_index` 优先的 replay procedure、verification checklist、完成 `skill-creator` workflow / validation 后才能报告创建完成的 finalization 指引，以及路径脱敏，确保录制后 handoff 不只停留在文档说明。 +- 用 `make record-and-replay-skill-repo-smoke` 验证 standalone thin skill repo scaffold，确保生成的 repo 可通过自带 `scripts/package-skill.sh` 校验 frontmatter 并打出 `.skill`，manifest 只声明 `open-computer-use event-stream mcp`、initialize capabilities、三件套 tools / metadata 和 no-active response contract，并用 `checks` / `optionalChecks` 区分默认 skill workflow / recording-to-skill smoke 与可选 lifecycle smoke；`scripts/verify-runtime.py` 可握手验证实际 runtime contract 和 no-active status/stop shape，`scripts/verify-skill-workflow.py` 可静态验证 thin skill 仍遵守官方 handoff 语义，`scripts/recording-to-skill-smoke.py` 可用合成 completed recording 验证安装态 strict/events-only validator 与 runtime scaffold，`scripts/check.sh` 可作为发布/安装前组合自检入口，生成 workflow 会安装发布版 runtime 后运行同一自检，且不泄漏本机源码绝对路径；同一 smoke 会显式运行 `scripts/recording-lifecycle-smoke.py`，验证生成 repo 在安装态 runtime 上可以完成最小 recording lifecycle，覆盖 one-active / idempotent stop / completed status，并校验 session 文件。默认 CI 已包含该 smoke。 +- `npm run package:skill` 会验证 `skills/*/SKILL.md` 都能打包为 `.zip` / `.skill`，并检查 `dist/skills/package-manifest.json` 中同时包含通用 Computer Use skill 和 Record & Replay thin skill；它已经接入默认 CI。 +- `scripts/compare-event-stream-surface.py --use-default-official` 可在本机默认官方 plugin cache 存在时对官方 `SkyComputerUseClient event-stream mcp` 的 initialize / `tools/list` 做同一 fixture 对比；该路径不启动录制，适合采集 golden recording 前先确认官方 non-recording surface 未漂移。 +- `make event-stream-no-active-smoke` 或 `scripts/compare-event-stream-no-active.py` 会用 Codex-hosted 官方 no-active fixture 校验本地 OCU `event_stream_status` / `event_stream_stop` text JSON，并确认没有创建 session 文件；该路径不调用 `event_stream_start`。 +- 用 `scripts/probe-event-stream-recording.py --target local --start-stop` 验证 raw probe 脚手架可以完成 OCU start/repeat start/status/stop/repeat stop/final status；用 `scripts/probe-event-stream-recording.py --target official` 做官方只读 probe；显式 `scripts/probe-event-stream-recording.py --target official --start-stop` 可记录官方 raw start lifecycle transcript，但当前本机宿主外预期会在首个 start 超时。用 `make event-stream-probe-fixture-smoke` 校验已保存的官方 start-timeout fixture。 +- 用 `make event-stream-fixture-smoke` 验证脱敏导入工具，确保生成的 fixture 不包含原始 session id、app attribution、window title、AX value / selectedText、MCP text JSON 原文、screenshot filename 或事件/suppressed event 中的本机 path，并且相对路径 metadata 能继续被 validator / summary 读取；AX diff line 会只保留 marker，以便后续 compare gate 校准 diff 结构。同一 smoke 还会用导入后的 `mcp-transcript.json` 跑 `check-event-stream-golden-readiness.py --require-mcp-response-shape repeatStartResponseShape|repeatStopResponseShape|finalStatusResponseShape`，并要求 suppressed evidence，再用 `compare-event-stream-recordings.py --require-mcp-response-shapes --require-same-mcp-response-schema` 自比，固定官方样本导入后可以直接进入 readiness/compare 门禁。 +- 用 `make event-stream-golden-readiness-smoke` 验证 golden readiness gate，确保动作类样本缺 action / AX payload、缺/重复/非首 `session.started`、缺/重复/非最终 `session.ended`、metadata/session alias 漂移、metadata 或 manifest count 漂移、handoff path 缺失或指向不存在文件会失败，lifecycle/status 样本只有显式放宽 action / AX 后才能通过；校准 stop / cancel / timeout 样本时，按官方观测值加 `--require-end-reason recording_controls_stopped|recording_controls_cancelled|`；需要 handoff alias 完整证据时，要求 `--require-session-alias`；需要 metadata count 完整证据时，要求 `--require-metadata-counts`；需要 declared path 完整证据时，要求 `--require-handoff-paths`；需要 suppressed fallback 证据时，要求 `--require-suppressed-events` 和对应 `--require-suppressed-event-type`；需要 MCP 对外行为证据时，要求 `--require-mcp-transcript` 和对应 `--require-mcp-tool-response event_stream_*`；需要 raw probe completed lifecycle 证据时，额外要求 `--require-mcp-response-shape repeatStartResponseShape|repeatStopResponseShape|finalStatusResponseShape`。required tool timeout 会以 `MCP tool timed out` 失败，required response shape timeout 会以 `MCP response shape timed out` 失败。 +- 用 `make event-stream-compare-smoke` 验证 recording 对比工具，确保相同样本通过，缺字段、handoff path 缺失或不可解析、事件顺序漂移、缺 AX diff/cumulative evidence 或缺 diff marker 会在 require 模式下失败。 +- 在多显示器、不同 Space、全屏 app、权限缺失场景下手动 smoke 录制控制条。 +- 对 AX compact diff 的单元与 fixture compare gate 已覆盖 changed / added / removed / diff too large fallback、diff/cumulative evidence 和 marker 保留；后续仍需真实官方 golden recording 校准字段与算法。 + +## 决策记录 + +- 2026-06-26：目标改为直接复刻官方 Record & Replay 行为，而不是只实现一个近似事件记录器。 +- 2026-06-26：MCP surface 保持官方兼容；OCU 扩展能力放在 CLI 和 runtime 内部，不增加官方 tools 参数。 +- 2026-06-26：录制控制条由 OCU 自己绘制和接收交互，避免与 Codex.app 或任何特定宿主耦合。 +- 2026-06-26：AX compact diff 纳入第一阶段目标，因为它是官方事件流的重要组成部分，不能长期用 full tree 替代。 +- 2026-06-26：Record & Replay MCP initialize 按官方当前行为返回 `protocolVersion` = `2025-11-25`；普通 Computer Use MCP 暂不跟随修改。 +- 2026-06-26：复核官方 `event-stream mcp` initialize，官方 result 不包含 `instructions`；OCU event-stream initialize 已去掉 instructions，普通 Computer Use MCP 不受影响。 +- 2026-06-26：`cancel` 作为 OCU CLI 扩展保留，不暴露到官方兼容 MCP surface；官方兼容 MCP 仍只有 `event_stream_start/status/stop`。 +- 2026-06-26：AX diff baseline 先复用现有 snapshot renderer，并实现轻量 `~` / `+` / `-` diff；是否完全等价官方 `UIElementRenderDifference` 需要后续 golden fixtures 校准。 +- 2026-06-26：截图上下文先按官方已观察到的 `screenshotNeededForContext` 线索做最小化策略，不默认每个事件截图；OCU 增加 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never` 作为可验证开关，后续再用官方 recordings 校准触发条件和字段细节。 +- 2026-06-26：按官方 “up to 30 minutes” 行为实现自动结束；官方 timeout endReason 尚未从当前字符串样本确认，OCU 暂用 `recording_time_limit_reached` 并在后续 golden 校准时复核。 +- 2026-06-26：start approval 作为 runtime 内部 gate 推进；官方兼容 MCP surface 仍保持三件套无参数，OCU 独立使用方通过 OCU 自有开始确认 UI / policy 承载用户确认，不依赖 Codex.app。 +- 2026-06-26：MCP elicitation 作为 start approval 的 host-capability 通道接入；它只在 event-stream MCP 内部发送 `elicitation/create`，不改变 `event_stream_start/status/stop` 的 tools/list 或 input schema。 +- 2026-06-26：官方 golden recording 采集脚手架独立成 `scripts/probe-event-stream-recording.py`；默认 CI 不跑官方 start/stop，因为官方 raw client 在宿主外会超时，后续应在正常 Codex 宿主链路或满足官方 runtime 状态的环境里采样；当前 raw start-timeout 观测以 sanitized fixture 形式入库。 +- 2026-06-26：Codex Record & Replay MCP 安装器作为 OCU 扩展层接入；它只写独立 `record_and_replay` server，不改变普通 Computer Use MCP server，也不改变官方三件套 tool surface。 +- 2026-06-26：程序合成的 OCU Computer Use 动作不会稳定回放给 macOS 全局 NSEvent monitor，不能作为端到端动作录制断言；服务级验证改为捕获 recorder 注册的 input monitor handlers 并喂入 `NSEvent`，覆盖 `recordMouseDown/up/keyDown` 到 `events.jsonl` 的链路。 +- 2026-06-27：生产 recorder 输入监听从单纯 `NSEvent.addGlobalMonitorForEvents` 升级为优先安装 listen-only `CGEvent` tap，失败时再回退到 NSEvent monitor；新增 opt-in `make event-stream-action-smoke` 用外部 HID click、scroll wheel 和 Shift-F20 keyboard event 验证 app-agent recorder 能真实写出 `mouse.click`、best-effort scroll raw-only `experimentalRawEvents` 与 `keyboard.shortcut`，并用同一份真实 recording 验证 runtime scaffold 产出的 skill 草稿 / summary 已路径脱敏且 `skillReadiness.canCreateSkillDraft=true`。 + +## 进度记录 + +- 2026-06-26：完成官方插件、Codex.app asar、Computer Use 二进制字符串和 MCP tools/list 的初步分析，并将方案落到文档。 +- 2026-06-26：完成 baseline 实现：`event-stream mcp`、`event-stream start/status/stop/wait --json`、长驻 app-agent 共享 session、`events.jsonl` / `metadata.json` / `session.json` / `suppressed.jsonl` 文件产物、基础鼠标/键盘/app 切换事件记录，并用 `swift test` 和本地 MCP smoke 验证。 +- 2026-06-26：新增 OCU 自有 Record & Replay 控制条，支持 Done、Discard、计时和拖动；新增 `event-stream cancel --json` 作为 OCU CLI 扩展，并验证 `recording_controls_cancelled` 写入 session endReason。 +- 2026-06-26：为 `AX.focusedWindowChanged` 接入 `accessibilityInspectorPayload`，支持首帧 full tree、后续同 app/window 轻量 compact diff，以及 snapshot 失败写入 `suppressed.jsonl`；本地 MCP smoke 已验证可生成 full AX payload。 +- 2026-06-26：为 diff AX payload 补轻量 cumulative diff baseline，保留 previous diff 为主 `renderedText/treeLines`，额外输出 `cumulativeRenderedText/cumulativeTreeLines`；算法和字段仍待 official golden recording 校准。 +- 2026-06-26：补齐截图上下文策略，支持 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never`、payload 级 `screenshotNeededForContext` / `screenshotAvailable` / `screenshotPath` 字段，以及 session 内 `screenshots/` PNG 产物；已用 `swift test` 和强制 `always` 的 MCP smoke 验证。 +- 2026-06-26：新增 `scripts/run-event-stream-smoke-tests.sh` 和 `make event-stream-smoke`，把 baseline MCP smoke 固化为仓库验证入口；默认检查 tools/list、start/status/stop、session 文件、AX payload 和 `session.json` alias，可选截图模式检查截图上下文，可选 timeout 模式检查自动结束。 +- 2026-06-26：为 `mouse.click` 增加 `targetAccessibilityElement`，为 `keyboard.text_input` 增加 `focusedAccessibilityElement`，并把 click 坐标明确为 screen-state `location` + AppKit `appKitLocation`；后续仍需 official golden fixtures 校准字段名和目标摘要细节。 +- 2026-06-26：修复并发 `wait` / `stop` race：`finish()` 先写最终 stopped/cancelled metadata 再清理 active session；app-agent 对 `status/stop/cancel/wait` 不再持有环境覆盖全局锁，避免上层 listener 阻塞结束命令。 +- 2026-06-26：补 stale active session 恢复：如果新 recorder 进程只看到旧的 active `latest-session.json`，会持久化改写为 `state=stopped`、`endReason=recording_process_unavailable`，同步 `metadata.json` / `session.json`，并清理 stale `active-session.json`。 +- 2026-06-26：补齐最长录制时长：默认 30 分钟后自动停止并写入 `session.ended`，测试可用 `OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS` 缩短时长；timeout endReason 仍待官方 golden recording 校准。 +- 2026-06-26：按官方字符串新增 `mouse.context_menu`、`keyboard.submit`、`keyboard.shortcut` 事件分类；scroll schema 留待 official golden recording 校准。 +- 2026-06-26：补齐 `mouse.drag` 可用 baseline，鼠标事件改为 down / dragged / up 后再判定 click/context menu/drag，避免拖拽被误记为点击；drag payload 字段结构仍待 official golden recording 校准。 +- 2026-06-26：补齐 `experimentalRawEvents` 可用 baseline，点击、拖拽、键盘事件会带低层 NSEvent 摘要；scroll wheel 在官方独立事件名未知时只写 raw-only 事件。 +- 2026-06-26：补齐 input 前 focused window context 检测，同一 app 内窗口变化后会在下一次输入事件前先补窗口上下文；当前已按官方字符串写入顶层 `window.changed`，并继续补 `AX.focusedWindowChanged`，字段与顺序仍待 golden recording 校准。 +- 2026-06-26：补齐轻量 `selection.changed` baseline；输入事件后如果 focused AX element 暴露非空 `selectedText` 且 selection signature 变化，会写入 selection 事件；如果此前有选择而当前选择为空，会写 `selectionCleared=true` 的清空事件，字段和触发语义仍待 official golden recording 校准。 +- 2026-06-26：补齐 `secureInput` 安全 baseline；secure/protected focused element 下键盘文本事件和 raw key event 不再写明文，只保留长度和 redaction 标记，字段仍待 official golden recording 校准。 +- 2026-06-26：补齐轻量 `terminal.value_changed` baseline；明显终端上下文下如果 focused AX value 变化，会写入 value hash / length 和脱敏 focused element，同时键盘 raw characters 与 focused value 不写终端缓冲区明文；字段与触发时机仍待 official golden recording 校准。 +- 2026-06-26：补齐轻量 `debug.error` baseline；AX snapshot、cumulative AX diff fallback 和 screenshot 写入失败时会在主事件流写入低敏 debug metadata，完整错误细节继续进入 `suppressed.jsonl`；字段仍待 official golden recording 校准。 +- 2026-06-26：补充 start approval 推进方案：先在 service 层沉淀 approved / denied / cancelled 语义和官方错误文案，再接 OCU 自有开始确认 UI，最后按 host 能力评估 MCP elicitation。 +- 2026-06-26：实现 start approval baseline：`event_stream_start` 已有 active session 时直接返回；新 session 创建前先走 approval gate；denied / cancelled 返回官方风格错误且不写 session；交互式 runtime 使用 OCU 自有开始确认弹窗，smoke 显式 auto-approve。 +- 2026-06-26：补 CLI wait listener 覆盖，验证 `event-stream wait --json --session-id ...` 能在另一路 stop 后返回同一个 stopped session；补 official initialize 无 `instructions` 的单测。 +- 2026-06-26：修正 app-agent event-stream MCP server 初始化时机，并让 proxied `event-stream mcp` 请求转发外层 `OPEN_COMPUTER_USE_*` 环境，避免 `event_stream_start` 的环境覆盖还没生效就提前初始化共享 recorder；新增 app-agent wait smoke，验证 MCP `start` 写入指定 recordings root，CLI `wait` 能被另一个 socket 连接上的 `stop` 唤醒。 +- 2026-06-26：为 event-stream smoke 增加可选官方 surface 对比模式，只调用官方 initialize/tools-list 并与 OCU 三件套 schema 对比，不启动官方录制。 +- 2026-06-26：扩展 event-stream MCP smoke，覆盖 active recording 时重复 start 返回同一 session、stop 后重复 stop/status 仍返回最终 stopped session，以及 timeout 后 stop 的幂等行为。 +- 2026-06-26：新增 approval smoke 模式，覆盖 `event_stream_start` 被 denied / cancelled 时返回官方错误文案且不创建 `latest-session.json` / `active-session.json`。 +- 2026-06-26：补 Input Monitoring 不可用时的显式降级：如果本地 NSEvent monitor 全部安装失败，录制仍会启动并写入低敏 `debug.error` 主事件，`subsystem=inputMonitoring`、`reason=inputMonitorsUnavailable`，避免上层把无输入事件误判为正常空录制；字段仍待 official golden recording 校准。 +- 2026-06-26：增强 OCU `event-stream wait --json` listener 返回体，新增 `waitTimedOut` 扩展字段；官方兼容 MCP 三件套不暴露该字段。 +- 2026-06-27：增强 OCU `event-stream wait --session-id` 语义：wait 会优先读取目标 session 目录里的完成态 metadata，支持返回非 latest 的历史 completed session；未知或不匹配 session id 会快速返回 `waitTimedOut=true` / `waitSessionMatched=false` 并跳过 notify callback，避免独立集成方无限等待错误 id。 +- 2026-06-26：新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1`，通过真实 app-agent CLI start/wait/cancel 路径验证 `waitTimedOut=true`。 +- 2026-06-26：新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1`，通过真实 MCP initialize/tools-list/status/stop 路径验证无 active recording 时不会写 session 文件；2026-06-27 该 smoke 的 response 断言已按官方 no-active shape 更新。 +- 2026-06-27：通过 Codex-hosted 官方 `event_stream_status` / `event_stream_stop` 观察 no-active response shape，并将 OCU MCP no-active status/stop 从 OCU `state=idle` payload 校准为官方最小 `isRecording=false` / `maxDurationSeconds=1800` payload；CLI `event-stream status/stop --json` 仍保留 OCU 扩展字段用于 wait 和本地诊断。 +- 2026-06-26：新增 `scripts/validate-event-stream-recording.py`，用于校验 Record & Replay session 目录或 metadata/session 文件；OCU smoke 已接入 strict 模式，后续官方 golden recording 可先用非 strict 模式做结构检查再推进 schema 对齐。 +- 2026-06-26：新增 runtime CLI `open-computer-use event-stream summarize --json` 和源码脚本 `scripts/summarize-event-stream-recording.py`，把 `events.jsonl` 摘成 event type、window、action sequence、target/focused element、selection、debug、redaction evidence 和结构化 `runtimeInputs`；默认不输出 AX `value` / `selectedText` 原文，只输出长度、目标摘要和敏感标记，默认 smoke 已接入读取验证，为后续录制后生成 skill 提供前置审计入口。 +- 2026-06-26：为 summary / scaffold 增加 `skillReadiness` baseline，输出 `ready` / `needsReview` / `insufficient`，把“信息不足不要猜”的官方 skill 语义沉淀成可检查字段；生成的 `SKILL.md` 会展示 workflow readiness 和建议下一步。 +- 2026-06-26：新增 runtime CLI `open-computer-use event-stream validate --json`，在安装态校验 metadata/session alias、event/suppressed counts、final `session.ended`、`endReason`、required event type 和 `screenshotPath`，并接入默认、timeout、wait-timeout、app-agent wait smoke；源码 Python validator 保留为仓库内交叉检查。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 增加 strict OCU current segment path gate；`state=recording` 必须带存在的 `currentSegmentEventsPath` / `currentSegmentMetadataPath`，`state=stopped|cancelled` 不能残留这两个字段。Swift 单测和 `make event-stream-skill-scaffold-smoke` 覆盖源码脚本与 runtime CLI 的缺失 / 残留拒绝路径。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 增加 strict OCU declared handoff path gate；输出 `declaredPaths`，并在 `--strict-ocu` 下要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个字段都存在且解析后文件存在。Swift 单测和 skill scaffold smoke 覆盖完整路径、缺失四字段和坏 `sessionPath`；skill scaffold 测试夹具同步补齐 `sessionPath` / `session.json`。 +- 2026-06-27：raw probe fixture 脱敏补强：`scripts/probe-event-stream-recording.py --fixture-output` 现在会把 MCP text content 中可解析的 JSON 转成脱敏 `textJSON`，避免 start/status/stop response text 里保留本机绝对路径；新增 `scripts/test-event-stream-local-probe.py` / `make event-stream-local-probe-smoke`，用真实 OCU start/status/stop probe 固定 active current segment 与 final cleanup response shape，并接入默认 CI。 +- 2026-06-27：raw probe start/stop 采样扩展为 start/repeat start/status/stop；首次 start 成功后 probe 会立即再调用一次 `event_stream_start`，用于采集官方 one-active 语义证据。local probe smoke 断言两次 start 指向同一个 active session，并继续验证脱敏 fixture 与 final session 文件。 +- 2026-06-27：raw probe completed lifecycle 采样补齐 repeat stop 和 final status；首次 stop 成功后 probe 会再调用一次 `event_stream_stop` 和一次 `event_stream_status`，用于采集官方 idempotent stop 与 completed status 证据。local probe smoke 断言 repeat stop / final status 仍指向同一个 completed session。 +- 2026-06-27：runtime CLI 与源码 Python validator 增加 `--require-skill-draft`，在结构校验之外要求至少一个高层动作且 recording 未取消；action smoke 和 scaffold smoke 已覆盖成功与 cancelled 拒绝路径。 +- 2026-06-27：runtime Swift validator 与源码 Python validator 增加非 strict `events.jsonl` 输入支持；只有主事件流时也能验证 JSONL、required event type、完成态、skill draft gate、从 `session.ended.endReason` 推断取消态和阻断性诊断，strict OCU 仍要求 metadata/session alias。runtime/source summary 与 scaffold 同步支持 `` 输入，scaffold smoke 覆盖成功和 cancelled 拒绝路径。 +- 2026-06-26：新增 `scripts/import-event-stream-fixture.py` 和 `make event-stream-fixture-smoke`，用于把 OCU 或官方 recording 脱敏导入 `docs/references/.../fixtures/recordings/`，并修正 validator / summary 对 fixture 相对路径的读取。 +- 2026-06-26：新增 `scripts/check-event-stream-golden-readiness.py` 和 `make event-stream-golden-readiness-smoke`，把官方/OCU recording fixture 是否足够作为 golden baseline 的判定机械化；默认要求 final state、`session.started` 首事件 / 唯一性、`session.ended` 最终事件 / 唯一性、动作事件和 AX payload，full/diff/cumulative AX 证据可按校准阶段显式要求。 +- 2026-06-26：扩展 official golden fixture 证据链：`scripts/probe-event-stream-recording.py --start-stop` 现在在 stop 前补一次 `event_stream_status`；`scripts/import-event-stream-fixture.py --mcp-transcript` 会把 probe output 脱敏成 `mcp-transcript.json`；`scripts/check-event-stream-golden-readiness.py` 可要求 `event_stream_start/status/stop` response 证据。 +- 2026-06-27：golden readiness gate 明确识别 required MCP tool timeout；当 `mcp-transcript.json` 只有 `startResponseShape.timeout` / `stopResponseShape.timeout` 这类宿主外边界证据时，会报 `MCP tool timed out`，避免把 timeout 样本误判成成功 response golden。 +- 2026-06-27：golden readiness gate 新增 `--require-mcp-response-shape`，可显式要求 `repeatStartResponseShape`、`repeatStopResponseShape` 和 `finalStatusResponseShape` 等 probe shape；shape 缺失、timeout 或没有 result/error 会独立失败，避免 one-active、idempotent stop 和 completed status 证据采到了但后续 golden gate 不认识。tool 级 evidence 聚合以“至少一次 result/error”为通过条件；同 tool 的 repeat timeout 不会污染已成功的 start/status/stop tool response，具体 repeat timeout 由 shape gate 负责报告。 +- 2026-06-27：golden readiness gate 新增 suppressed evidence gate；`--require-suppressed-events` 要求 `suppressed.jsonl` 非空，`--require-suppressed-event-type ` 要求指定 suppressed event type 存在，便于后续官方样本校准 AX / screenshot / 压缩 fallback 行为。 +- 2026-06-27：golden readiness gate 新增 `--require-end-reason `，会从 metadata/session alias 和 `session.ended.endReason` 汇总 `endReasons`，用于官方 stop / cancel / timeout golden fixture 入库时显式固定 expected control completion reason。 +- 2026-06-27：golden readiness gate 补齐 final session evidence，默认拒绝多个 `session.ended` 或 `session.ended` 后继续写入事件的 fixture，并在 JSON 输出里暴露 `lastEventType`、`sessionEndedCount` 和 `sessionEndedIsFinal`，避免 malformed completed recording 进入官方/OCU golden baseline。 +- 2026-06-27：golden readiness gate 新增 metadata/session alias evidence，默认拒绝同时存在但内容不同的 `metadata.json` / `session.json`，输出 `metadataSessionAliasComplete` 和 `metadataSessionAliasMatches`，并提供 `--require-session-alias` 要求两份 alias 文件都存在，避免官方 handoff fixture 只验证其中一个 JSON。 +- 2026-06-27：golden readiness gate 新增 metadata count evidence，默认拒绝已声明但与文件行数不一致的 `eventCount` / `suppressedEventCount`，输出 `metadataEventCountMatches` 和 `metadataSuppressedEventCountMatches`，并提供 `--require-metadata-counts` 要求两项 count 字段都存在；fixture manifest 也会校验 `suppressedEventCount`，避免 fallback evidence 计数漂移。 +- 2026-06-27：golden readiness gate 新增 declared handoff path evidence，输出 `declaredPaths`，默认拒绝已声明但不存在的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath`，并提供 `--require-handoff-paths` 要求四个字段都存在且可解析，避免官方 skill handoff fixture 只证明文件内容、不证明返回路径可消费。 +- 2026-06-26：新增 `scripts/compare-event-stream-recordings.py` 和 `make event-stream-compare-smoke`，用于把官方 fixture 与 OCU candidate 的事件序列和 JSON schema 差异变成可复跑报告。 +- 2026-06-27：recording compare 新增可选 MCP response shape evidence gate；当 official fixture 和 OCU candidate 都带 `mcp-transcript.json` 时，`--require-mcp-response-shapes` 会比较 baseline 中出现的 `startResponseShape` / `repeatStartResponseShape` / `statusResponseShape` / `stopResponseShape` / `repeatStopResponseShape` / `finalStatusResponseShape` 是否在 candidate 中存在且保持 result/error/timeout 状态，`--require-same-mcp-response-schema` 会进一步检查 response schema path/type 不丢失。 +- 2026-06-27：recording compare 新增 suppressed stream gate；`--require-same-suppressed-event-sequence` 比较 `suppressed.jsonl` 事件类型顺序，`--require-same-suppressed-schema` 比较 suppressed event schema path/type，避免官方样本中的 AX / screenshot / 压缩降级证据只靠 metadata `suppressedEventCount` 间接覆盖。 +- 2026-06-27：recording compare 新增 declared handoff path evidence 和 `--require-handoff-paths`，比较 baseline 与 candidate 是否都声明 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 且解析后文件存在，避免 candidate 只保留 metadata key set 却丢掉 skill 可消费路径。 +- 2026-06-26：新增 OCU `event-stream wait --notify-command ''` callback 扩展，结束唤醒时把最终 status JSON 通过 stdin/env 交给集成方，超时 wait 不触发 callback，callback 非零退出或超时会让 CLI 返回 error；新增单元测试覆盖 stop 后通知和 wait timeout 跳过通知。 +- 2026-06-26：扩展 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1`，在真实 app-agent socket 路径下验证 `wait --notify-command` 能收到 stop 后最终 status JSON、session/state env 和成功 `notification` 结果。 +- 2026-06-26：补 `event-stream wait` 生命周期回归：已结束 session 再 wait 会立即返回最终状态，cancel/Discard 也会唤醒 listener 并把 `recording_controls_cancelled` 传给 notify callback；app-agent wait smoke 覆盖 completed wait。 +- 2026-06-26：新增 `scripts/run-event-stream-smoke-matrix.sh` 和 `make event-stream-smoke-matrix`，把 Record & Replay baseline 的稳定本地验证模式收敛成一条命令；截图和官方 surface 对比保持 opt-in。 +- 2026-06-26：将 `npm run package:skill`、`swift test` 和稳定 Record & Replay smoke matrix 接入 `scripts/ci.sh` / `make ci`，并补齐最小 GitHub workflow、issue/PR template 和 dependency review 配置，使仓库默认 CI 入口可以守住当前 event-stream baseline 与 thin skill 制品。 +- 2026-06-26：增强默认 event-stream smoke 的官方 skill handoff 覆盖：从 `event_stream_stop` 返回 text JSON 中读取 `eventsPath` / `metadataPath` / `sessionPath`，并对 `sessionPath` alias 执行 runtime validate/summarize，固定独立 skill/plugin 后续可直接消费的路径语义。 +- 2026-06-26：新增 `open-computer-use-record-and-replay` thin skill，并将 `scripts/package-skill.sh` 改为自动扫描 `skills/*/SKILL.md` 打包多 skill；独立 repo 尚未创建，但仓库内已有可安装的 recording-to-skill workflow baseline。 +- 2026-06-26：新增 `scripts/scaffold-event-stream-skill.py` 和 `make event-stream-skill-scaffold-smoke`，把录制摘要转成 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`；默认 CI 覆盖该路径，作为独立 repo 前的最小 recording-to-skill handoff。 +- 2026-06-26：新增 runtime `open-computer-use event-stream scaffold-skill --json`，复用 Swift summary helper 生成 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`，让安装态 OCU / 独立 thin skill 不必依赖源码 checkout。 +- 2026-06-26：扩展 `make event-stream-skill-scaffold-smoke`，同一个临时 recording 会分别经过源码 Python helper 和 runtime CLI `event-stream scaffold-skill --json`，固定安装态 handoff 也能真实生成可脱敏草稿。 +- 2026-06-26：增强源码 Python scaffold 与 runtime Swift scaffold 的 `SKILL.md` 模板，生成物现在包含 runtime inputs、先检查 connector/API/专用工具再使用 Computer Use、`get_app_state` / semantic `element_index` 优先的 agent replay procedure、observed replay steps 和 verification checklist；focused 单测与 scaffold smoke 已覆盖这些章节。 +- 2026-06-27：summary / scaffold 的 runtime inputs 从 Markdown 临时推断升级为结构化 `runtimeInputs` 字段；Swift runtime summary 和源码 Python summary 都会从文本输入与 selection 事件提取运行时值候选，scaffold 优先使用该字段渲染 Runtime Inputs，并在单测与 scaffold smoke 中覆盖路径脱敏、敏感输入提示和 selection 输入候选。 +- 2026-06-27：summary / scaffold 增加结构化 `safetySignals` 字段；Swift runtime summary 和源码 Python summary 会从 `keyboard.submit` 以及 target/focused element 的 send/delete/save/upload/publish 等低敏标题关键词提取显式确认线索，scaffold 在 Safety 区块渲染 Confirmation Signals，并把 `hasSafetySignals` 纳入 readiness review reason。单测与 scaffold smoke 覆盖 Swift/Python 两条生成路径。 +- 2026-06-27：thin Record & Replay skill 和通用 `open-computer-use` Record & Replay reference 已同步要求 agent 消费 `runtimeInputs` / `safetySignals`：用前者抽运行时输入，用后者作为 submit/send/delete/save/upload/publish 等步骤的第一确认清单，并继续以 `events.jsonl` 作为最终证据。standalone thin skill repo scaffold smoke 已固定导出 repo 会保留这些指引。 +- 2026-06-27：summary / scaffold 增加结构化 `summaryLimits` 字段；Swift runtime summary 和源码 Python summary 会报告 action sequence、runtime inputs、safety signals、target/focused elements、diagnostics 和截图路径等高容量字段的上限、源计数、保留计数和省略计数，scaffold 在 Summary Limits 区块提示被省略字段并要求回读原始 `events.jsonl`。单测和 scaffold smoke 覆盖长录制截断场景。 +- 2026-06-27：summary / validator / scaffold 增加阻断性 `blockingDiagnostics` gate；当前把 `debug.error` 的 `reason=inputMonitorsUnavailable` 判定为 demonstration 不可信，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 和 Swift/Python scaffold 都会拒绝生成草稿并提示修权限后重录。单测和 scaffold smoke 覆盖 Swift runtime、源码 Python helper 和 runtime CLI 路径。 +- 2026-06-27：summary / validator / scaffold 增加完成态 gate；`state=recording`、`active=true` 或缺少 `session.ended` 会输出 `recordingIncomplete=true`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并提示先 stop 录制。单测和 scaffold smoke 覆盖 Swift runtime、源码 Python helper 和 runtime CLI 路径。 +- 2026-06-27：summary / validator / scaffold 增加 start event gate；缺失 `session.started`、多个 `session.started` 或 `session.started` 不是第一条事件时，会输出 `sessionStartedCountInvalid` / `sessionStartedNotFirst`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录，避免消费没有清晰 start boundary 的 malformed recording。 +- 2026-06-27：summary / validator / scaffold 增加 final event gate；如果事件流里出现 `session.ended` 但最后一条事件不是 `session.ended`，会输出 `sessionEndedNotFinal=true` / `sessionEndedIsFinal=false`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录，避免消费 stop/cancel 后仍继续写入事件的 malformed recording。 +- 2026-06-27：summary / validator / scaffold 增加 duplicate final event gate；如果事件流里出现多个 `session.ended`，会输出 `sessionEndedCountInvalid=true` / `sessionEndedCount>1`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录，避免重复 stop/cancel 写入的 malformed recording 进入 skill handoff。 +- 2026-06-27：runtime Swift scaffold 和源码 Python scaffold 现在会在 summary / 写文件前先跑 recording validator 的 skill draft gate；event count、metadata/session alias、路径或截图引用等结构校验失败时直接返回 `recordingValidationFailed`，不再允许只凭可摘要事件生成草稿。Swift 单测和 `make event-stream-skill-scaffold-smoke` 覆盖 event count mismatch 的源码与 runtime 拒绝路径。 +- 2026-06-27：golden readiness 和 recording compare 同步补 start boundary gate；动作类 golden 默认要求 `session.started` 恰好一次且为首事件，recording compare 的 `--require-final-session-evidence` 保留兼容 flag 名但同时比较 `session.started` 个数 / 首事件位置与最终 `session.ended` 闭环。 +- 2026-06-26：新增 `scripts/compare-event-stream-surface.py` 和 `make event-stream-surface-smoke`，把 OCU vs 官方 1.0.857 non-recording fixture 的 surface drift check 独立成可复跑入口；本机默认官方 bundle 的 opt-in 对比已通过，不触发真实录制。 +- 2026-06-27：新增 `scripts/compare-event-stream-no-active.py` 和 `make event-stream-no-active-smoke`，把 Codex-hosted 官方 no-active `event_stream_status` / `event_stream_stop` fixture 接成独立 drift check；脚本只调用本地 OCU status/stop，不调用 start，并断言不会创建 session 文件。该脚本已接入默认 `scripts/ci.sh`。 +- 2026-06-26：新增 MCP elicitation approval baseline：`OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp|elicitation` 时，支持 elicitation 的 host 会收到 `elicitation/create`，accept 后才创建 session，decline / cancel / unsupported 不创建 session；新增单元测试和 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1` smoke，默认 matrix 已覆盖该路径。 +- 2026-06-26：复核 Codex.app asar 中通用 MCP elicitation schema，确认 host form path 依赖 `mode: "form"`、object `requestedSchema` 和 `accept|decline|cancel` response；OCU start approval request 已校准为显式 form mode + 空对象 schema，probe fixture 摘要也开始记录 elicitation request shape。官方 Record & Replay 真实业务 message / `_meta` 仍待 golden 样本。 +- 2026-06-26：新增 `scripts/probe-event-stream-recording.py` 和 `make event-stream-probe` / `make event-stream-official-probe` / `make event-stream-official-start-probe`；local start/status/stop probe 已通过，官方只读 probe 已通过，官方 raw start/status/stop 当前超时且未产出 recording paths；新增官方 raw start-timeout fixture 和 `make event-stream-probe-fixture-smoke`。 +- 2026-06-27：补强 raw probe elicitation 覆盖：probe 对 server-initiated `elicitation/create` 的 accept response 保持 `content: {}`，decline / cancel response 使用 Codex host 观察到的 `content: null`；新增 `--elicitation-action accept|decline|cancel` 和 fake MCP server self-test，默认 CI 覆盖 request schema 摘要、三种 action 应答形状和旧 `--decline-elicitation` 兼容别名。 +- 2026-06-26：新增 `scripts/install-codex-record-and-replay-mcp.sh` 和 `make codex-record-and-replay-installer-smoke`，通过临时 `CODEX_HOME` 验证 `record_and_replay` TOML 写入和幂等性；npm launcher / package build 已同步转发该命令。 +- 2026-06-26：新增 `testEventStreamServiceRecordsInputMonitorMouseAndKeyboardEvents`，验证 input monitor handler 级 mouse/key 事件写入；同时修正 raw event summary 只在适用事件类型上读取 `phase` / `clickCount` / key 坐标，避免 AppKit 对不支持属性抛 exception。 +- 2026-06-26：新增 `scripts/scaffold-record-and-replay-skill-repo.py` 和 `make record-and-replay-skill-repo-smoke`，把仓库内 thin skill baseline 导出成 standalone repo 骨架并校验可打包、路径脱敏和 MCP runtime contract；真实外部 repo 创建与发布验证仍作为后续项。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/verify-runtime.py`，并让 `make record-and-replay-skill-repo-smoke` 用当前构建出的 `OpenComputerUse` binary 实际探测 `event-stream mcp` initialize / `notifications/initialized` / `tools/list`，验证官方兼容三件套 surface 不只是 manifest 声明。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/check.sh`，串起 skill package 和 runtime contract verifier；`make record-and-replay-skill-repo-smoke` 已覆盖该组合自检入口。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `.github/workflows/ci.yml`，在干净 macOS runner 上安装发布版 `open-computer-use` 并运行 `scripts/check.sh`；smoke 会校验 workflow 使用 SHA-pinned actions 且调用同一组合自检入口。 +- 2026-06-27：standalone thin skill repo 的 `scripts/verify-runtime.py` 升级为校验官方 initialize capabilities 和三件套完整 tool metadata（description、empty input schema、annotations），`record-and-replay-skill-repo.json` 同步写入 `capabilities` / `toolMetadata`，避免独立 repo 只检查工具名称而漏掉 surface 漂移。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/recording-lifecycle-smoke.py` 作为可选真实录制闭环检查；`make record-and-replay-skill-repo-smoke` 会用当前构建出的 `OpenComputerUse` binary 运行它，覆盖 start/repeat start/status/stop/repeat stop/final status、one-active、idempotent stop、`metadata.json` / `session.json` / `events.jsonl` 存在性和安装态 `event-stream validate --strict-ocu`。 +- 2026-06-27：standalone thin skill repo 生成态 `scripts/package-skill.sh` 新增 frontmatter gate，要求 `SKILL.md` 具备匹配目录名的 `name` 和非空 `description`；`make record-and-replay-skill-repo-smoke` 覆盖缺失 description 会失败的路径。 +- 2026-06-27：standalone thin skill repo 生成态 README 新增 recording-to-skill handoff 指引，明确完整 OCU session 先跑 `validate --strict-ocu --require-skill-draft `，只有 `eventsPath` 时跑非 strict events-only gate，并说明 cancelled / incomplete recording 不能用于创建或更新 skill；`make record-and-replay-skill-repo-smoke` 已断言这些指引会随独立 repo 导出。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/recording-to-skill-smoke.py` 并接入默认 `scripts/check.sh`；该 smoke 不启动真实录制，而是创建临时 completed recording，验证安装态 runtime 的 strict skill-draft validation、events-only validation 和 `event-stream scaffold-skill --json` 草稿生成能力，确保独立 repo 默认自检覆盖 recording-to-skill handoff。 +- 2026-06-27：standalone thin skill repo 的生成态 `scripts/recording-to-skill-smoke.py` 现在会检查 strict validator 输出的 `declaredPaths`，要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个 handoff path 都解析存在；生成态 README 和 thin skill/reference 同步说明 strict OCU validation 能证明 declared handoff paths，而 events-only gate 不能证明。 +- 2026-06-27：standalone thin skill repo manifest 新增 `recordingToSkill` contract，机器可读声明 strict validation 命令、必须存在的 declared handoff path、events-only gate 不能证明 metadata/session alias 与 declared paths，以及 scaffold-skill 会先跑 skill draft validation gate；`scripts/test-record-and-replay-skill-repo-scaffold.py` 已固定该 contract。 +- 2026-06-27：standalone thin skill repo manifest 固化 `checks` / `optionalChecks` 分层：package、runtime contract、synthetic recording-to-skill 和组合 self-check 是默认发布/安装前门禁；真实创建 session 的 lifecycle smoke 只作为可选本地检查，避免独立 repo CI 默认启动桌面录制。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/verify-skill-workflow.py` 并接入默认 `scripts/check.sh` / manifest `checks`；该 verifier 不启动录制，只检查 thin skill 保留官方 handoff 约束和 OCU validator/scaffold 前置门禁，避免独立 repo 后续改文案时漂移到轮询、扩 MCP surface 或取消后再 stop。 +- 2026-06-27：npm staging 包新增安装态 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir ` launcher 命令，并把 scaffold 脚本和 thin skill 源打包进去；使用方全局安装 `open-computer-use` 后即可生成 standalone Record & Replay skill repo 骨架，不必依赖源码 checkout。 +- 2026-06-27：新增 `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 和 `make npm-record-and-replay-skill-repo-smoke`，用本地 `dist/` artifacts stage npm 包后，从 staged launcher 执行安装态 scaffold 命令，并让生成 repo 的默认 `scripts/check.sh` 反向使用同一个 staged launcher，覆盖 package、runtime verifier、skill workflow verifier、合成 recording-to-skill scaffold 和 `skill-creator` handoff；该 smoke 会先用 staged launcher 校验合成 recording，确认 strict validator 已输出 `declaredPaths`，避免旧 `dist` native artifacts 进入后续 scaffold 自检；随后读取安装态生成的 `record-and-replay-skill-repo.json`，固定 `recordingToSkill` manifest contract 中 strict validation、events-only validation、scaffold-skill 和 `rejectsCancelledRecordings` 的机器可读边界；该 smoke 作为 release/staging 前 opt-in 检查，不进入默认 CI。 +- 2026-06-27：复跑 `scripts/compare-event-stream-surface.py --use-default-official`，确认本机 OCU `event-stream mcp` 和官方 `record-and-replay/1.0.857` 默认 cache 的 initialize / `tools/list` 仍同时匹配已入库 fixture；本次不调用 `event_stream_start`，不产生官方 recording。 +- 2026-06-26：增强 AX compact diff baseline：从相邻行启发式改为 LCS 风格 diff，比较时归一化 AX tree leading element index，避免插入元素后把后续稳定行误判为变更；新增 changed / added / removed / over-budget fallback 单测，官方算法级行为仍待 golden fixtures 校准。 +- 2026-06-26：扩展官方/OCU recording 校准链路：fixture import 会在脱敏 `treeLines` / `cumulativeTreeLines` 时保留 `+` / `-` / `~` marker；`scripts/compare-event-stream-recordings.py --require-ax-diff-evidence` 可要求 candidate 保留 baseline 的 AX diff/cumulative payload 和 marker 类型，`make event-stream-compare-smoke` 已覆盖缺 evidence / 缺 marker 失败路径。 +- 2026-06-27：recording compare 新增 `--require-semantic-fields`，按事件类型比较官方 baseline 中出现的 replay-critical 字段是否也出现在 OCU candidate 中；覆盖 target AX、keyboard focused AX、drag 起止点、selection / terminal 摘要字段、window attribution 和 AX treeLines / cumulativeTreeLines，避免只看完整 schema 或事件序列时漏掉可复用 skill 所需的关键证据。 +- 2026-06-27：recording compare 新增 metadata key diff 和 `--require-same-metadata-keys`，把官方 / OCU candidate 的 metadata/session 文件布局键漂移变成可失败的 gate；`make event-stream-compare-smoke` 覆盖 candidate 缺 metadata key 的失败路径。 +- 2026-06-27：recording compare 新增 stable metadata value diff 和 `--require-same-metadata-values`，比较 `state`、`active`、`endReason`、`eventCount`、`suppressedEventCount`，避免 metadata key 一致但 final state / endReason / count 漂移时仍通过；`make event-stream-compare-smoke` 覆盖 candidate endReason 漂移失败路径。 +- 2026-06-27：recording compare 新增 final session evidence 和 `--require-final-session-evidence`，比较 `session.ended` 个数、是否为最后事件以及 endReason 集合；`make event-stream-compare-smoke` 覆盖 candidate 最后事件不是 `session.ended` 的失败路径。 +- 2026-06-26：补 AX payload 服务级覆盖：把 snapshot-to-payload 的 diff 状态机抽成生产与测试共用 helper，并新增测试确认 full -> previous diff -> cumulative diff 会落到真实 session `events.jsonl` 的 `AX.focusedWindowChanged` 事件里。 +- 2026-06-27：补截图上下文服务级覆盖：用可控 PNG snapshot 验证 AX payload 在 `always` 策略下会把截图写入 session `screenshots/`，并在实际 `events.jsonl` 的 `screenshotPath` 引用该文件。 +- 2026-06-27：新增 `make event-stream-action-smoke`，启动 app-agent recorder 后由外部 Swift 进程发出低影响 CGEvent click 和 best-effort scroll wheel，最终用 validator / summary 要求 `mouse.click`、`AX.focusedWindowChanged` 和 `session.ended` 入流，并继续用同一份真实录制产物调用 runtime `event-stream scaffold-skill --json` 生成 `SKILL.md` 草稿；捕获到的 `experimentalRawEvents` / `reason=scrollWheel` 会纳入 `actionSequence` 并渲染为 Scroll replay step，但不作为默认 hard gate。该 smoke 也断言生成的 `SKILL.md` / `references/recording-summary.json` 不含本机绝对路径，summary 路径字段使用 `` 占位，且 `skillReadiness.canCreateSkillDraft=true`。 +- 2026-06-27：在当前 worktree 复跑默认 `make event-stream-smoke-matrix`、`make event-stream-action-smoke` 和 `scripts/compare-event-stream-surface.py --use-default-official` 均通过；其中默认 matrix 覆盖 lifecycle / no-active / timeout / wait / approval / MCP elicitation / app-agent wait-notify / fixture import / recording compare，action smoke 真实录到 click、scroll raw-only、keyboard shortcut 并生成 skill 草稿，官方 surface 对比确认 OCU 与官方 1.0.857 仍匹配 `Record & Replay` / `2025-11-25` / 三件套。 +- 2026-06-27：新增 opt-in `make record-and-replay-baseline-smoke`，作为本机 baseline 候选一键验证入口，串起默认 `event-stream` matrix、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、standalone thin skill repo smoke 和 npm standalone repo smoke；该入口依赖桌面输入权限、官方 bundled cache 和当前 `dist/` artifacts，不进入默认 CI。本机已复跑通过，确认组合入口本身可用。 +- 2026-06-27：将 baseline 候选验证命令收敛到 `scripts/run-record-and-replay-baseline-smoke.sh`，`make record-and-replay-baseline-smoke` 只负责转调；脚本成功时输出 `{"ok":true,"baseline":"record-and-replay",...}` 摘要，便于后续独立 repo、release 或人工验收复用同一入口，而不是复制 Makefile 命令串。 +- 2026-06-27：`scripts/run-record-and-replay-baseline-smoke.sh` 现在也串入 official fixture set gate smoke，让本机 opt-in baseline 候选验证同时守住后续 official successful recording 入库后的集合级校准门禁；成功 JSON 的 `checks` 同步包含 `official-fixture-set-smoke`。 +- 2026-06-27：复跑新的 `make record-and-replay-baseline-smoke` 通过，最终摘要为 `{"ok":true,"baseline":"record-and-replay","checks":[...]}`；该次覆盖默认 matrix、真实输入 action smoke、官方 1.0.857 surface compare、official fixture set gate smoke、standalone repo smoke 和 npm staged repo smoke。 +- 2026-06-27：`scripts/run-record-and-replay-baseline-smoke.sh` 现在会捕获并解析 standalone repo smoke 与 npm staged repo smoke 的 JSON 输出，要求 runtime contract、no-active shape、no-arg malformed request gate、wait/notify contract、recording-to-skill handoff、declared handoff paths、scaffold failure exit、cancelled recording contract / rejection、真实 lifecycle smoke、npm manifest contract、npm 生成 repo 默认 `check.sh` evidence 和 `skill-creator` handoff evidence 都为 true；最终 baseline 摘要新增 `evidence.standaloneSkillRepo` / `evidence.npmStagedSkillRepo`，避免组合 smoke 只看子命令退出码。随后 standalone lifecycle evidence 进一步细化为 `checkedLifecycleOneActive`、`checkedLifecycleIdempotentStop` 和 `checkedLifecycleFinalStatus` 三个必需项，确认可选真实录制闭环不只是脚本跑过，而是实际验证了重复 start、stop 幂等和 completed final status。 +- 2026-06-27：`scripts/run-record-and-replay-baseline-smoke.sh` 现在也会捕获并解析官方 1.0.857 non-recording surface compare JSON，显式要求 `local-open-computer-use` 与 `official-record-and-replay` 两个 label 都通过；最终 baseline 摘要新增脱敏 `evidence.officialSurfaceCompare`，只保留 fixture 名、protocol、server name、tool names 和 ok 状态，不写本机官方 plugin 路径。该 evidence 只证明 non-recording surface 对齐，不替代 official successful recording fixture。 +- 2026-06-27：baseline 聚合脚本现在显式运行截图上下文 smoke，并把 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 下的检查写入 `evidence.screenshotContextSmoke`；该 evidence 要求 AX payload 出现 `screenshotNeededForContext`，当前环境实际返回截图数据时也要求 `screenshotPath` 存在。 +- 2026-06-27：baseline 聚合脚本现在显式运行 `scripts/compare-event-stream-no-active.py`，要求本地 OCU MCP no-active `event_stream_status` / `event_stream_stop` 与 Codex-hosted 官方 fixture 的最小 text JSON 对齐，并确认不创建 session 文件；最终摘要新增 `evidence.officialNoActiveResponse`，只保留 fixture 名、checked tools、status/stop shape 和 no-session-files evidence。 +- 2026-06-27：baseline 聚合脚本现在也会捕获并解析默认 matrix 与真实输入 action smoke 的 JSON 输出，要求 matrix 至少覆盖 no-active、timeout、wait-timeout、approval、MCP elicitation、app-agent wait/notify 和 default lifecycle handoff，要求 action smoke 至少录到 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.click` 和 `session.ended` 并生成 skill 草稿；scroll raw-only `experimentalRawEvents` 作为 best-effort evidence 保留给 summary/scaffold；最终摘要新增脱敏 `evidence.eventStreamMatrix` / `evidence.realInputActionSmoke`，不写临时 recording 或 skill 文件路径。 +- 2026-06-27：真实输入 action smoke 的最终 JSON 继续细化为 concrete evidence；同一份 action recording 必须证明 MCP start/repeat start/status/stop/repeat stop/final status response shapes 已捕获、`skillReadiness.canCreateSkillDraft=true`、生成草稿保留 `skill-creator` finalization handoff、`SKILL.md` / summary 已脱敏本机路径。baseline summary 和 runner 都把 `checkedMcpResponseShapesCaptured`、`checkedSkillReadinessCanCreateDraft`、`checkedSkillCreatorFinalizationHandoff` 和 `checkedGeneratedSkillPathRedaction` 纳入 `realInputActionSmoke` 必需项。 +- 2026-06-28：完整 baseline audit 现在会额外运行 `simple-action-stop` 和 `drag-stop` action candidate smoke，并把 `checkedSimpleActionStopCandidate` / `checkedDragStopCandidate` 纳入 `usableBaseline` 必需 evidence；runner summary-json smoke 已按场景拆分 fake action fixture，确认 shell runner 真实传递 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO`。当前落盘 `dist/record-and-replay-baseline-summary.json` 已通过 artifact audit，显示两个字段都为 true，同时仍保持 required `simple-action-stop` official successful recording 缺口。 +- 2026-06-28：standalone / npm staged runtime contract evidence 继续拆细。生成态 `scripts/verify-runtime.py` 现在输出 `checkedInitializeSurfaceContract`、`checkedToolMetadataContract` 和 `checkedToolInputSchemaNoArguments`；shared baseline contract、source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费这些字段，避免独立 repo 后续只用宽泛 `checkedRuntimeContract` 证明官方三件套 surface。 +- 2026-06-28：standalone thin skill repo scaffold 新增生成态 `scripts/verify-readme-handoff.py` 并接入默认 `scripts/check.sh` / manifest `checks.readmeHandoffContract`。该 verifier 不启动录制，只检查 README 仍保留官方 evidence 文件、baseline / strict audit 命令、strict expected-failure audit、required / recommended successful recording 场景、recording-to-skill 命令和 wait/notify 边界；source scaffold smoke、npm staged scaffold smoke、shared baseline contract 和 summary builder 都消费 `checkedReadmeHandoffContract`、`checkedReadmeOfficialEvidenceHandoff`、`checkedReadmeOfficialGoldenGap` 和 `checkedReadmeWaitNotifyBoundary`，并补了 README 漂移负例。 +- 2026-06-28：official fixture set gate 的 same-scenario compare 改成默认强策略，除原有 event sequence、schema、metadata、handoff path、final session、semantic fields 和 MCP response shape 外，也要求 AX diff evidence / marker 与 suppressed event sequence / schema 对齐。`make event-stream-official-fixture-set-smoke` 新增 candidate 缺 AX diff / cumulative diff 的失败覆盖，baseline summary / artifact audit 消费对应 evidence，确保后续 official golden 入库后能直接暴露 OCU compact diff 或 suppressed fallback 漂移。 +- 2026-06-28：standalone manifest 进一步声明 official fixture set gate 的强对比策略。`officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy` 固定 AX diff evidence、AX diff marker、suppressed event sequence 和 suppressed schema 边界；生成态 manifest verifier、source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费 `checkedOfficialFixtureSetComparePolicyManifest`，防止独立 repo handoff 丢失 same-scenario 强 compare 合同。 +- 2026-06-28：standalone scaffold 现在会把源仓 `dist/record-and-replay-baseline-summary.json` 脱敏投影复制到生成 repo 的 `evidence/source-baseline-summary.json`，并新增 `scripts/verify-source-baseline-summary.py` / `checks.sourceBaselineSummaryEvidence` 让默认 `scripts/check.sh` 直接审计该 evidence。source scaffold smoke、npm staged scaffold smoke、baseline summary 和 artifact audit 都消费 `checkedSourceBaselineSummaryEvidence`，确保独立 repo handoff 不只保留 source audit 命令文案，也消费已落盘的 baseline audit 结果。 +- 2026-06-27：baseline 聚合脚本现在也会捕获并解析 official fixture set gate smoke JSON，要求集合级 gate、`simple-action-stop` / `cancel` scenario policy、candidate pairing、missing scenario/candidate 负例和导入 scenario manifest 自检都为 true；最终摘要新增 `evidence.officialFixtureSetGate`，同时扫描仓库真实 official recording fixture scenario 覆盖并输出 `repoHasRequiredOfficialSuccessfulFixture` / `missingRepoOfficialScenarios`，避免把“gate 自身通过”误读成“官方 successful recording 已经入库”。 +- 2026-06-27：新增 hosted official recording ingest 入口：`scripts/ingest-official-record-and-replay-fixture.py` 可从官方 `event_stream_stop/status` 返回 JSON、JSON-RPC `result.content[].text` 或 `--status-json -` stdin 里递归提取 `metadataPath` / `sessionPath` / `eventsPath`，并可通过 `--status-json-base-dir` 为相对 handoff path 指定解析基准，随后调用 `scripts/import-event-stream-fixture.py` 脱敏生成 official fixture，并按 scenario 运行 golden readiness；提供 `--mcp-transcript --require-mcp-transcript-evidence --check-fixture-set` 时会继续要求 response-shape 证据并跑集合级 gate，`--mcp-transcript -` 可在 status JSON 来自文件时从 stdin 读取单独 transcript，`--use-status-json-as-transcript` 可在同一 stdin/status JSON 含 response-shape evidence 时临时复用。该入口现在也支持 `--check-coverage` / `--require-coverage`，导入后可直接输出 official scenario coverage + required readiness report，或在 required scenario 覆盖 / readiness 不足时返回非零。新增 `make event-stream-official-fixture-ingest-smoke` 覆盖 nested hosted response、stdin 输入、相对路径 base dir、transcript stdin、双 stdin 拒绝、脱敏、readiness、fixture set gate、post-ingest coverage/readiness report 和 require-coverage 失败路径，并接入默认 CI。 +- 2026-06-27：official hosted ingest 入口新增 `--inspect-only`，用于刚拿到 Codex hosted `event_stream_stop/status` JSON 后先做无写入诊断；输出 `stage=inspect`、`recordingInputInspection`、`mcpTranscriptInspection` 和 `wouldImport=false`，确认 handoff paths、recording input 和可选 MCP transcript 证据可解析后再正式导入。`make event-stream-official-fixture-ingest-smoke` 已覆盖 inspect-only 成功、stdin status JSON 复用 transcript 成功、要求 transcript 但未提供时失败，baseline summary 也新增 `checkedOfficialFixtureInspectOnly` evidence。 +- 2026-06-27：新增 OCU candidate ingest 入口：`scripts/ingest-ocu-record-and-replay-candidate.py` 可从已有 OCU recording、action smoke JSONL 或 opt-in `--run-action-smoke` 生成 `source=ocu` candidate fixture；脚本先跑 strict OCU validator，再调用 fixture import 脱敏入库，并按 scenario 跑 readiness。提供 `--official-root --check-fixture-set` 时会直接跑 same-scenario official-vs-candidate gate。新增 `make event-stream-ocu-candidate-ingest-smoke` 覆盖 direct recording、smoke JSONL、脱敏和 official pairing，并接入默认 CI。 +- 2026-06-27：OCU candidate ingest 的场景级 gate 补齐 `keyboard-input-stop` 和 `drag-stop`。`scripts/ingest-ocu-record-and-replay-candidate.py` 现在会在 validation 阶段要求对应的 `keyboard.text_input` / `mouse.drag` 与 AX context，在 readiness 阶段要求对应 action event、`recording_controls_stopped`、full AX payload，以及启用 `--require-mcp-transcript-evidence` 时要求 repeat start / repeat stop / final status response-shape evidence。`make event-stream-ocu-candidate-ingest-smoke` 已覆盖 keyboard / drag candidate readiness。 +- 2026-06-27：OCU candidate ingest 输出继续补齐后续 handoff；成功导入后会返回 `commands.pairingPreflightShell` 和 `commands.fixtureSetGateShell`，指向 same-scenario pairing preflight 与集合级 official-vs-candidate gate。`make event-stream-ocu-candidate-ingest-smoke` 输出 `checkedCandidateIngestHandoffCommands=true`，baseline runner、summary 和 artifact audit 都消费 `checkedOcuCandidateIngestHandoffCommands`，避免 candidate 采样后仍靠人工拼接后续校准命令。 +- 2026-06-27：baseline 聚合脚本现在把 `make event-stream-official-fixture-ingest-smoke` 和 `make event-stream-ocu-candidate-ingest-smoke` 纳入同一条 opt-in `make record-and-replay-baseline-smoke`，并在最终 JSON 写入 `evidence.fixtureIngestPipelines`；该 evidence 要求 official fixture ingest、official `sessionDirectoryPath` handoff、OCU candidate ingest、candidate ingest 后续 handoff commands、smoke JSON import、official/candidate pairing、candidate redaction、keyboard readiness 和 drag readiness 均为 true，确保后续采到 official successful recording 后可以直接走已验证的导入/配对/候选校准链路。 +- 2026-06-27：baseline 聚合脚本的最终 JSON 新增顶层 `status` 区块，直接汇总 `usableBaseline`、`officialNonRecordingBaselineVerified`、`standaloneRepoScaffoldBaselineVerified`、`standaloneRepoBaselineReady`、`officialSuccessfulRecordingGoldenComplete`、`officialSuccessfulRecordingEquivalenceReady`、`requiresOfficialGoldenCapture` 和缺失的 required / recommended official successful recording scenarios。`usableBaseline` 现在由 event-stream matrix、截图上下文、真实输入 action、官方 non-recording surface、官方 no-active response、official fixture set gate、fixture ingest、standalone repo 和 npm staged repo 的具体 evidence 推导；若必需 evidence 缺失，摘要会输出 `status.missingUsableBaselineEvidence` 并返回非零。默认模式仍允许 official successful recording golden 缺失，但不允许其它 baseline 证据缺失；`standaloneRepoBaselineReady` 表示 standalone scaffold baseline 与 npm staged scaffold baseline 已可交付，`officialSuccessfulRecordingEquivalenceReady` 只有在 usable baseline 与 required official successful recording golden 都成立时才为 true；`officialSuccessfulRecordingGoldenComplete` 只有在 required scenario 覆盖和 required readiness 都通过时才为 true。 +- 2026-06-27：baseline 聚合脚本的最终 JSON 继续细化 official golden 缺口：`status.notReadyRequiredOfficialSuccessfulRecordingScenarios` 和 `evidence.officialFixtureSetGate.notReadyRepoOfficialScenarios` 会列出 scenario 已覆盖但 readiness 未通过的官方 fixture；严格 gate 失败时 stderr 分别输出 `missing=...` 与 `notReady=...`，避免把“缺官方样本”和“已有样本不合格”混在一起。 +- 2026-06-27：baseline 聚合脚本新增严格 official golden gate：`scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden` / `make record-and-replay-official-golden-gate` 复用完整 baseline 检查，但如果 required official successful recording fixture 缺失或 required readiness 未通过，最终摘要写入 `ok=false`、`strictOfficialGoldenRequired=true`、`officialGoldenRequirementSatisfied=false`、`officialGoldenGatePassed=false` 并返回非零。默认模式允许缺 official golden 时只会让 `officialGoldenRequirementSatisfied=true`，不会把 `officialGoldenGatePassed` 误标为 true。当前缺 `simple-action-stop` 时该 gate 预期失败，用于后续 official golden 入库后的 release / 拆 repo 验收。 +- 2026-06-27：baseline 最终摘要生成逻辑从 shell heredoc 抽到 `scripts/build-record-and-replay-baseline-summary.py`，并新增 `scripts/test-record-and-replay-baseline-summary.py` / `make record-and-replay-baseline-summary-smoke`，用合成证据覆盖默认允许缺 official golden、严格缺 required official fixture 失败、严格已覆盖 required official fixture 通过，以及 standalone/npm evidence 缺失时默认模式失败四类分支；该快测已接入默认 CI。 +- 2026-06-27：真实输入 action smoke 改为通过 proxied `event-stream mcp` 三件套录制同一 session，并输出 `mcpTranscriptPath`；`scripts/ingest-ocu-record-and-replay-candidate.py --smoke-json` 会自动使用该 transcript，`--run-action-smoke` 则用短 `/tmp/ocu-rnr-*` 目录避免 app-agent Unix socket path 过长。已实测 `make event-stream-action-smoke` 和 `ingest-ocu-record-and-replay-candidate.py --run-action-smoke --require-mcp-transcript-evidence` 通过。 +- 2026-06-27：真实输入 action smoke 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop`，分别采样 click 和 drag，并保持默认 `mixed-action-stop` 覆盖 click，同时 best-effort 发送 scroll wheel。`scripts/ingest-ocu-record-and-replay-candidate.py --run-action-smoke --scenario ` 会把 simple / drag 映射到同名 action smoke 场景，其它非 keyboard 场景回退到 mixed，用于在 official golden 入库前先产出同 scenario OCU candidate。`keyboard-input-stop` 仍由 candidate ingest 的 direct recording / smoke JSON 路径和合成 smoke readiness 覆盖，暂不通过 synthetic keyboard `--run-action-smoke` 采样。 +- 2026-06-27：新增 official golden fixture set gate：`scripts/import-event-stream-fixture.py` 支持 `--scenario` 写入 manifest，`scripts/check-event-stream-official-fixture-set.py` 默认要求 official fixture set 至少有 `simple-action-stop` successful recording 并跑 strict readiness；传入 `--candidate-root` 后按同 scenario 对 OCU candidate 运行强 compare，覆盖 event sequence、schema、metadata keys/values、handoff paths、final session evidence、semantic fields 和 MCP response shape。新增 `make event-stream-official-fixture-set-smoke` 覆盖 gate 成功、缺 candidate、缺 scenario 和导入 scenario manifest。 +- 2026-06-27：official fixture set gate 现在也会对传入的同 scenario OCU candidate 先跑 `source=ocu` readiness，再进入 official-vs-candidate compare；`make event-stream-official-fixture-set-smoke` 新增 candidate `metadata.json` / `session.json` alias 漂移负例，避免手动传入结构不完整的 candidate fixture 时 compare 独自通过。 +- 2026-06-27：新增 official fixture coverage report：`scripts/check-event-stream-official-fixture-coverage.py` 单独扫描仓库 recording fixtures 的 official scenario 覆盖，默认缺 `simple-action-stop` 时失败；`--allow-missing` 用于当前默认 CI 和 baseline smoke，只报告 `coverageOk=false` / `missingOfficialScenarios`，避免把 synthetic fixture set smoke 误解成真实 official successful recording 已入库。新增 `--check-readiness` / `--require-readiness` 后，required scenario 存在时会继续跑 official fixture set readiness，并用 `notReadyOfficialScenarios` 暴露已覆盖但不合格的场景；baseline smoke 使用 `--allow-missing --check-readiness`，strict gate 依赖 required coverage + readiness。新增 `make event-stream-official-fixture-coverage-smoke` 覆盖缺失、allow-missing、candidate ignored、覆盖成功、readiness report、readiness 失败 / 成功、重复 scenario 和缺 scenario 负例，并接入默认 CI。 +- 2026-06-27：official fixture coverage report 现在区分 required 与 recommended 覆盖。required 默认仍是 `simple-action-stop`，影响非 `--allow-missing` 退出码；recommended 默认报告 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`，只作为后续 official golden 采样清单。`scripts/run-record-and-replay-baseline-smoke.sh` 的最终 JSON 同步输出 `recommendedOfficialScenarios`、`repoHasRecommendedOfficialScenarioCoverage` 和 `missingRepoRecommendedOfficialScenarios`。 +- 2026-06-27：强化 official fixture set 的 scenario 策略：`simple-action-stop` 现在要求 `mouse.click` 与 `recording_controls_stopped` endReason，`keyboard-input-stop` 要求 `keyboard.text_input` 与 `recording_controls_stopped` endReason，`drag-stop` 要求 `mouse.drag` 与 `recording_controls_stopped` endReason，`cancel` 要求 `recording_controls_cancelled` endReason；`timeout` 因官方 endReason 仍未观测，继续只跑 lifecycle readiness 而不固定具体值，并允许 no action / no AX payload。`make event-stream-official-fixture-set-smoke` 覆盖 stop endReason mismatch、keyboard/drag 成功录制策略、keyboard 缺失必需事件失败、cancel 场景成功和 timeout lifecycle-only 样本通过。 +- 2026-06-27：复跑 `make event-stream-official-fixture-set-smoke`、`make event-stream-fixture-smoke`、`make event-stream-golden-readiness-smoke` 和 `make event-stream-compare-smoke` 通过，确认新增 scenario manifest 不破坏既有导入/readiness/compare 链路，集合级 gate 自身也可复跑。 +- 2026-06-27：复跑 `make event-stream-official-start-probe`，官方 1.0.857 bundled raw MCP client 仍只返回 initialize / tools-list；`event_stream_start`、后续 status 和 stop 均 10 秒超时，没有返回 recording paths。结论不变：official successful recording fixture 仍需通过正常 Codex 宿主流程或补齐官方 runtime 所需宿主状态后采集。 +- 2026-06-27：刷新官方 raw start-timeout sanitized fixture，入库的 `record-and-replay-official-raw-start-timeout-1.0.857.json` 现在显式包含 `startResponseShape`、`statusResponseShape` 和 `stopResponseShape` 三个 10 秒 timeout，并且 `scripts/test-event-stream-probe-fixtures.py` 会要求 transcript 中恰好有三次 `tools/call` timeout，避免 fixture 证据退回旧的 start/stop-only 形态。 +- 2026-06-27：baseline 聚合脚本现在显式运行 `scripts/test-event-stream-probe-fixtures.py`，并把官方宿主外 raw start/status/stop timeout 写入最终摘要的 `evidence.officialRawStartTimeout`。该 evidence 要求三次 raw `tools/call` timeout、没有 `metadataPath` / `eventsPath` / `sessionPath` 等 recording handoff paths、官方 surface 存在且 sanitized fixture 不含本机路径；`usableBaseline` 会消费这些字段，避免 hostless timeout 边界只停留在 standalone manifest 声明里。 +- 2026-06-27：复跑默认 `make ci` 通过，覆盖文档骨架、repo hygiene、GitHub Actions pinning、shell / Node / Python 语法检查、skill 打包、R&R surface/no-active/probe/golden readiness/skill scaffold/standalone repo/Codex installer smoke、`swift test` 228 个测试、默认 `event-stream` matrix，以及 Windows / Linux Go test；真实输入 action smoke 和 npm staging baseline 仍由 opt-in `make record-and-replay-baseline-smoke` 覆盖。 +- 2026-06-27：在刷新官方 raw start-timeout fixture 后再次复跑 `make ci` 通过；同次复跑 `make record-and-replay-baseline-smoke` 也通过，覆盖默认 event-stream matrix、真实输入 action smoke、官方 1.0.857 surface compare、official fixture set gate smoke、standalone repo smoke 和 npm staged repo smoke，最终输出 `{"ok":true,"baseline":"record-and-replay",...}`。 +- 2026-06-27：补齐 README / README.zh-CN 的 Record & Replay 快速入口，明确 dedicated R&R MCP 安装、官方三件套 surface、OCU CLI wait/notify 扩展、validate/summarize/scaffold-skill 录制到 skill 草稿流程，以及 `open-computer-use scaffold-record-and-replay-skill-repo` 生成独立 thin skill repo 的默认自检。 +- 2026-06-27：强化 runtime Swift scaffold 和源码 Python scaffold 的 finalization checklist：生成的 `SKILL.md` 草稿现在明确要求读取并完成 `skill-creator` workflow、包含 validation，并且只有最终成为实际可发现 skill 目录后才能报告已创建；`make event-stream-skill-scaffold-smoke` 已固定该文案。 +- 2026-06-27：固定取消录制的 recording-to-skill guard：summary 在 cancelled / `recording_controls_cancelled` 时返回 `insufficient` 和 `canCreateSkillDraft=false`，runtime Swift scaffold 与源码 Python scaffold 都拒绝生成草稿；Swift 单测和 `make event-stream-skill-scaffold-smoke` 覆盖该路径。 +- 2026-06-27：同步 thin skill / reference 的官方 handoff 语义：`event_stream_start` 成功后 agent 应结束当前 turn、请用户完成后回来，并告知录制最长 30 分钟；不改变 MCP tool schema 或 status response。 +- 2026-06-27：同步 official concurrent recording handoff：重复 start 返回 active session 是 runtime one-active 语义，agent 不应静默复用，而应询问用户要使用该 active recording 还是等待它停止。 +- 2026-06-27：OCU 自有开始确认弹窗同步显示录制时限，默认文案为 `up to 30 minutes`；该提示只属于独立 UI，不改变官方兼容 MCP surface。 +- 2026-06-27：standalone thin skill repo scaffold 的生成态 contract 增加 `extensionLayer`，机器可读声明 `wait --notify-command`、`waitTimedOut` / `waitSessionMatched`、notify stdin/env 和 listener 不进入官方 MCP surface；生成态 `verify-skill-workflow.py` 与 smoke 也会检查 Independent Wait / Notify Integration 文案，避免独立 repo 后续只保留官方三件套 handoff 而丢失 OCU 独立回调边界。 +- 2026-06-27：standalone thin skill repo scaffold 的生成态 manifest 增加 `officialEvidence`，机器可读声明当前 contract 基于官方 `record-and-replay/1.0.857` 的 surface fixture、Codex-hosted no-active status/stop fixture、hostless raw start/status/stop timeout fixture，以及从 copied source baseline summary 派生的 `hasSuccessfulRecordingGolden` 状态；manifest 也固化 required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景清单，并在 `officialEvidence.scenarioRecipes` 中写入每个推荐场景的采集合同。源码 standalone smoke、npm staged smoke 和 baseline smoke 都会读取 `checkedOfficialEvidenceManifest=true` 与 `checkedOfficialEvidenceScenarioManifest=true`。 +- 2026-06-27：Record & Replay MCP `tools/call` 运行时 contract 继续收紧：`params.name` 现在必须是非空 string，缺失或非 string 会返回 tool error，且不触发 start approval、不创建 session 索引文件；standalone thin skill repo manifest / `verify-runtime.py` / smoke 同步验证该 malformed request 无副作用。 +- 2026-06-27：Record & Replay MCP `tools/call` 运行时 contract 继续补齐 `params` object gate；缺失或非 object `params` 会返回 tool error，且不触发 start approval、不创建 session 索引文件；standalone thin skill repo manifest / `verify-runtime.py` / smoke 同步验证该 malformed request 无副作用。 +- 2026-06-27：standalone thin skill repo scaffold 新增生成态 `scripts/wait-notify-contract-smoke.py`，并接入默认 `scripts/check.sh` 与 manifest `checks.waitNotifyContractSmoke`；该 smoke 不启动录制，只等待一个不存在的 session，验证 `waitTimedOut=true`、`waitSessionMatched=false`、notify callback 被跳过且不会创建 `latest-session.json` / `active-session.json`，把 OCU listener/callback 边界从静态文案推进到安装态 CLI 行为检查。后续补强为也用合成 completed session 验证失败 callback：CLI 必须非零并在 notification 中返回 `reason=nonZeroExit`；生成态 manifest 声明 `extensionLayer.waitNotify.callbackFailureMakesCliFail=true`，standalone / npm staged / baseline summary 都消费 `checkedNotifyCallbackFailureExit` 与 `checkedNotifyCallbackFailureReason`。 +- 2026-06-27：official golden capture preflight 新增 `--capture-packet-dir `；该参数仍不启动官方录制、不写 fixture，只生成 scenario recipe、preflight JSON、hosted status/transcript placeholder 和 verify/inspect/import/check wrapper，便于后续拿到 Codex hosted official JSON 后直接替换输入、先跑 `verify-inputs.sh`，再跑 inspect-only。消费 hosted JSON 的 `inspect-only.sh` / `import-fixture.sh` wrapper 会在运行命令前分别拒绝 status response 或 MCP transcript 中仍带 `_placeholder=true` 的 input 文件，并提示替换对应 hosted JSON。`--capture-packet-recommended-scenarios` 会为 required + recommended 场景生成 per-scenario 子目录、根级 `capture-packets.json` 和根级 verify/inspect/import/check/ingest-ocu-candidates 批量 wrapper；根级 `verify-all.sh` 会逐个检查输入文件存在、JSON 可解析且不再是占位，根级 `inspect-all.sh` / `import-all.sh` 会在执行任何子 wrapper 前先全量检查所有场景输入，根级 `ingest-ocu-candidates.sh` 会批量运行可自动生成 OCU candidate 的子场景。单场景 `capturePacket`、根级 `capturePacketSet` 和 `capture-packets.json` 会用 `includeTranscript` / `requiresMcpTranscriptInput` 机器可读声明当前采集包是否要求 `inputs/mcp-transcript.json`。使用 `--no-include-transcript` 生成单场景 packet 或批量 packet set 时，这些字段为 `false`，wrapper 只检查 status response，不要求未生成的 `inputs/mcp-transcript.json`。preflight smoke 已覆盖 capture packet 文件结构、placeholder JSON、wrapper 脚本引用、verify-inputs、inspect/import 两条单场景 placeholder guard、根级 verify-all、单场景 no-transcript packet、no-transcript packet set、manifest transcript requirement 字段、根级批量 wrapper 对 placeholder 失败的传播，以及后续场景仍是占位时不先执行前置场景 wrapper 的半批次导入防护；baseline summary 现在也消费 `checkedOfficialCapturePacketNoTranscript` 和 `checkedOfficialCapturePacketTranscriptManifest`，缺失时 `usableBaseline=false`。 +- 2026-06-27：official capture packet 继续补齐采样后 handoff scripts。单场景 packet 的 README 和 `capturePacket` JSON 现在显式暴露 `check-fixture-set.sh`、`strict-golden-gate.sh` 和可用时的 `ingest-ocu-candidate.sh`，并返回 `checkFixtureSetShell`、`strictGoldenGateShell`、`ingestOcuCandidateShell` 字段；official capture preflight smoke 输出 `checkedCapturePacketHandoffScripts=true`，baseline runner、summary 和 artifact audit 都消费 `checkedOfficialCapturePacketHandoffScripts`，避免 official hosted JSON 导入后缺少 fixture-set gate、strict gate 或 same-scenario OCU candidate 导入命令。后续又把 `ingest-ocu-candidate.sh` 的输出目录固定为当前 official fixture root 下的 `ocu-candidates/`，并通过 `checkedCapturePacketOcuCandidateOutputDir` / `checkedOfficialCapturePacketOcuCandidateOutputDir` 纳入 baseline evidence，避免自定义 fixture root 时 official fixture 与 OCU candidate 分散到不同根目录。 +- 2026-06-27：recommended capture packet set 继续补齐批量 OCU candidate handoff。根目录现在生成 `ingest-ocu-candidates.sh`，只运行存在 `ingest-ocu-candidate.sh` 的子场景并跳过 recording-required / no-action 场景；official capture preflight smoke 输出 `checkedCapturePacketSetOcuCandidateHandoff=true`，baseline runner、summary 和 artifact audit 都消费 `checkedOfficialCapturePacketSetOcuCandidateHandoff`，避免 recommended set 采样后还要人工逐个进入子目录找可用 candidate wrapper。 +- 2026-06-27：Makefile 新增 `record-and-replay-official-golden-capture-packet` 与 `record-and-replay-official-golden-capture-packet-set`，把单场景 packet 和 required+recommended packet set 固化为短命令；默认仍写系统临时目录且不启动官方录制，可用 `RNR_SCENARIO` / `RNR_PACKET_DIR` 覆盖。后续补充了 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`、`RNR_FIXTURE_ROOT` 和 `RNR_OFFICIAL_PLUGIN_ROOT`，并把两个 Make target 纳入 official capture preflight smoke。 +- 2026-06-27：baseline 聚合脚本现在显式运行 official golden capture preflight smoke 与 OCU candidate pairing preflight smoke，并在最终摘要写入 `evidence.preflightPipelines`。该 evidence 要求 missing official scenario、capture packet Make targets、capture packet、single-scenario no-transcript packet、recommended capture packet set、no-transcript recommended packet set、单场景 verify-inputs、单场景 inspect/import placeholder guard、packet set 根级 verify-all、packet set 根级 OCU candidate handoff、packet set 根级 placeholder guard、packet set 根级 preflight-all placeholder guard、`--require-ready` 失败、missing official plugin、keyboard recording-required、coverage error、missing candidate、paired candidate 和 candidate compare 成功路径都被覆盖；`usableBaseline` 会消费这些字段，确保后续 official golden 采样与 OCU candidate 配对入口也是完整 baseline 的一部分。 +- 2026-06-27：standalone thin skill repo manifest 的 `officialEvidence` 新增 `sourceRepoBaselineChecks` 和 `standaloneRepoBoundary`；前者声明源仓 baseline 会验证 hostless raw timeout、official golden capture preflight 与 OCU candidate pairing preflight，后者声明独立 repo 默认自检不启动官方录制、不复制 preflight 脚本、不复制 OCU runtime 源码。源码 scaffold、npm staged scaffold 和 baseline summary 都新增 `checkedOfficialEvidencePreflightManifest=true` evidence，并带缺失负例，避免后续拆 repo 时丢失 preflight/standalone 边界。 +- 2026-06-27：baseline summary 新增 `standaloneRepoBaselineReady` 与 `officialSuccessfulRecordingEquivalenceReady` 两个派生状态；前者让发版 / 拆 repo 判断不用从 standalone/npm evidence 间接推断，后者在缺 official successful recording golden 时保持 false，避免 `usableBaseline=true` 被误解成可以宣称官方事件 schema / AX / screenshot 等价。 +- 2026-06-27：baseline summary 新增机器可读 `nextActions`。默认缺 official golden 时会给出 `make record-and-replay-official-golden-capture-packet RNR_SCENARIO= RNR_PACKET_DIR=`、packet 内 `verify-inputs.sh`、`inspect-only.sh` / `import-fixture.sh`、strict golden gate 和 `make record-and-replay-official-golden-capture-packet-set RNR_PACKET_DIR=` / `verify-all.sh` / `inspect-all.sh` / `import-all.sh` / `ingest-ocu-candidates.sh`；standalone baseline ready 时会给出 `open-computer-use scaffold-record-and-replay-skill-repo`、生成 repo `check.sh` 与 optional lifecycle smoke 命令；baseline evidence 缺失时会优先给出修复 usable baseline 的动作。summary smoke 已覆盖默认缺 golden、baseline evidence 缺失和 strict covered 三类 `nextActions` 分支，并会实际执行 summary 里的 packet Make 命令生成临时单场景 packet 与 recommended packet set，验证 `preflight.json`、`scenario-recipe.json`、`capture-packets.json` 和 verify/inspect/import/ingest-ocu-candidates wrapper 都存在。 +- 2026-06-27：新增 `scripts/check-record-and-replay-baseline-summary.py` 和 `make record-and-replay-baseline-summary-audit-smoke`，并把 audit 接入 `scripts/run-record-and-replay-baseline-smoke.sh` 的最终 summary 生成路径；runner 保持 stdout 最终 summary / `--summary-json` 文件合同不变，把 audit 输出写到 stderr。默认审计要求顶层 `baseline=record-and-replay`、顶层 `ok` 与 `usableBaseline && officialGoldenRequirementSatisfied` 一致、usable baseline、standalone repo baseline、official non-recording baseline 和 raw timeout boundary 成立，并直接检查 event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、official surface、no-active response、raw-timeout boundary、official capture / OCU pairing preflight、baseline audit Make target、source standalone repo 和 npm staged repo evidence；严格审计要求 official golden gate、required scenario readiness、无 coverage errors 和 equivalence ready 全部成立。 +- 2026-06-27：baseline summary audit 继续补强为命令级 handoff 审计。默认缺 official golden 时，落盘 artifact 必须保留 required capture packet Make、`verify-inputs.sh`、inspect/import、strict gate，以及 recommended packet set Make、`verify-all.sh`、inspect-all/import-all 和 `ingest-ocu-candidates.sh`；standalone ready 时必须保留 scaffold repo、生成 repo `scripts/check.sh` 和 optional lifecycle smoke 命令。`scripts/test-record-and-replay-baseline-summary-audit.py` 新增 `nextActions` command 篡改负例，确保 release / standalone audit 不只看 action kind。 +- 2026-06-27：baseline summary audit 进一步补强派生状态审计。落盘 artifact 的 `requiresOfficialGoldenCapture` 必须等于 official golden 未完成，`officialSuccessfulRecordingEquivalenceReady` 必须等于 usable baseline + official golden complete，`standaloneRepoBaselineReady=true` 必须依赖 `usableBaseline=true`，且 official golden complete 时不能仍带 required scenario gaps 或 coverage errors。summary audit smoke 新增派生状态篡改负例和 golden-complete-with-gaps 负例，避免 release / standalone audit 误消费状态组合不一致的 summary。 +- 2026-06-27：baseline summary audit 现在也审计顶层 `checks` 声明。落盘 artifact 必须声明 event-stream matrix、screenshot context、real-input action、official surface / no-active / raw timeout、official fixture set / coverage / ingest、OCU candidate ingest、official capture / OCU pairing preflight、baseline audit target、source standalone 和 npm staged standalone 等必需检查，并且不能带未知或重复 check 名称；summary audit smoke 新增删除、追加未知项和重复项的负例，避免 release / standalone audit 消费检查清单漂移或被污染的摘要。 +- 2026-06-27：baseline summary 顶层 `checks` 的 canonical list 已抽到 `scripts/record_and_replay_baseline_contract.py`；builder 按该 tuple 顺序输出，audit 按同一合同复审，summary audit smoke 断言 builder 输出顺序与 shared contract 完全一致，避免后续 baseline 检查清单只更新生成器或审计器一侧。 +- 2026-06-27：Makefile 新增 `record-and-replay-baseline-audit` 和 `record-and-replay-official-golden-gate-audit`，把 release / standalone audit 的 summary artifact 入口固化为短命令。默认 audit 复用 `scripts/run-record-and-replay-baseline-smoke.sh --summary-json`，默认写 `dist/record-and-replay-baseline-summary.json`，可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖；strict audit 默认写 `dist/record-and-replay-official-golden-gate-summary.json`，可用 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=` 覆盖。默认 audit 保持缺 official golden 时可通过的 baseline 语义，strict audit 保持缺 required official successful recording 时非零退出的 gate 语义，并且默认不会覆盖 baseline 可用 artifact。 +- 2026-06-27：新增 `scripts/test-record-and-replay-baseline-audit-make-targets.py` 和 `make record-and-replay-baseline-audit-targets-smoke`，用 `make -n` 验证 audit Make target 不漂移：baseline 默认/自定义 `RNR_BASELINE_SUMMARY_JSON` 会传给 `--summary-json`，strict 默认/自定义 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 会传给 `--summary-json`，strict audit 会同时传 `--require-official-golden`，且 strict 默认 artifact 不复用 baseline 默认 artifact。该 smoke 接入默认 CI，但不运行完整 baseline、不启动官方录制；默认 CI 会带外层 `RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-summary.json` 和 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-golden-summary.json` 运行它，持续覆盖脚本内部默认路径检查的环境隔离；后续又补充命令行反向变量隔离断言，确认 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON` 不影响 baseline audit、`RNR_BASELINE_SUMMARY_JSON` 不影响 strict audit。完整 baseline runner 也会运行它，并把 baseline 默认/自定义 summary path、baseline 忽略 strict summary 变量、strict 默认/自定义 summary path、strict 忽略 baseline summary 变量、strict 独立 artifact 和 strict audit target 写入 `evidence.preflightPipelines`，纳入 `usableBaseline` 必需 evidence。 +- 2026-06-27：standalone wait-notify contract smoke 扩展为用合成 completed session 验证 notify callback 非零退出路径；同时修正 `open-computer-use event-stream ...` CLI 入口，使 `runOpenComputerUseEventStream` 标记的 `hasToolError` 和普通 `call` 一样映射为非零进程退出码。生成态默认自检现在既验证 unmatched session 跳过 callback，也验证 callback 失败会通过 JSON `notification` 和 CLI exit code 传递给独立集成方。 +- 2026-06-27：standalone wait-notify contract smoke 继续补齐 callback 超时失败路径；生成态 manifest 声明 `extensionLayer.waitNotify.callbackTimeoutMakesCliFail=true`，合成 completed session 下的 timeout callback 必须输出 `reason=timeout` 且 CLI 非零。源码 standalone smoke、npm staged smoke 和 baseline summary 都消费 `checkedNotifyCallbackTimeoutFailureExit` / `checkedNotifyCallbackTimeoutReason`，避免独立 repo 只验证非零退出而漏掉超时失败语义。 +- 2026-06-27：standalone wait-notify contract smoke 继续补齐 suppressed stream handoff；OCU `event-stream wait --notify-command` 成功回调现在会收到 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH`，生成态 smoke 用合成 completed session 验证该变量指向 `suppressed.jsonl`。源码 standalone smoke、npm staged smoke、baseline summary 和 artifact audit 都消费 `checkedNotifySuppressedEventsPathEnv`，避免独立 repo 结束回调只能拿到 metadata/session/events 而拿不到 suppressed diagnostics 路径。 +- 2026-06-27：standalone recording-to-skill smoke 扩展无动作 recording 负例；生成态 `scripts/recording-to-skill-smoke.py` 现在不仅验证 completed recording 能通过 strict/events-only gate 并生成 skill 草稿，也验证失败的 `event-stream scaffold-skill --json` 会输出 `ok=false` 且进程非零，避免独立 repo 的 shell/CI 误把不能生成 skill 的 recording 当成成功。 +- 2026-06-27：standalone recording-to-skill smoke 扩展 cancelled recording 负例；生成态 `scripts/recording-to-skill-smoke.py` 现在要求 strict validation、events-only validation 和 `event-stream scaffold-skill --json` 都拒绝 `recording_controls_cancelled` 录制，manifest 的 `recordingToSkill.rejectsCancelledRecordings=true` 也让独立 repo contract 机器可读地表达该边界。 +- 2026-06-27：standalone recording-to-skill smoke 继续扩展截图路径边界负例；生成态 `scripts/recording-to-skill-smoke.py` 现在会创建 session 外部但存在的 screenshot 文件引用，要求安装态 strict validator 拒绝，并在自检 JSON 输出 `checkedScreenshotPathContainment=true`。源码 standalone smoke、npm staged smoke 和 baseline 聚合都强制读取该 evidence。 +- 2026-06-27：npm staged standalone smoke 把 screenshot containment 检查前置到安装态 scaffold 之前；`scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 会用 staged launcher 直接校验 session 外 `screenshotPath` 负例，如果当前 `dist/Open Computer Use.app` 里的 native binary 尚未拒绝，会明确提示重建 `dist` artifacts。 +- 2026-06-27：baseline summary 继续细化 source standalone recording-to-skill evidence；`checkedStrictValidation`、`checkedEventsOnlyValidation`、`checkedScaffoldSkill` 和 `checkedSkillCreatorHandoff` 现在都进入 `usableBaseline` 必需项，runner 也会在解析 standalone smoke JSON 时提前断言这些字段，避免独立 repo 只声明 recording-to-skill handoff 却缺少具体 validation、scaffold 或 skill-creator 交接证据。 +- 2026-06-27：同一 concrete evidence 继续扩展到 npm staged standalone 路径；`scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 会从安装态 launcher 生成 repo 的 `check.sh` 输出中断言 strict validation、events-only validation 和 scaffold-skill 正路径，并在最终 JSON 输出 `checkedStrictValidation`、`checkedEventsOnlyValidation` 和 `checkedScaffoldSkill`，baseline summary 与 runner 都把这些字段纳入 npm staged 必需 evidence。 +- 2026-06-27：npm staged standalone scaffold 的 Python launcher 诊断也进入 baseline evidence。安装态 smoke 已验证 `PYTHON` 指向不存在的可执行文件和 Python 2 都会以 command-specific 诊断失败，并输出 `checkedNpmPythonLauncherDiagnostics=true`；baseline runner 早期断言、summary 和 artifact audit 都会消费该字段，缺失时报告 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics`。 +- 2026-06-27：standalone thin skill repo 生成态 README 的 Python 3 前置条件也进入 baseline evidence。生成 README 现在声明自检脚本要求 `python3`，并提示 npm launcher 用户用 `PYTHON=/path/to/python3` 修复 shell 找不到 Python 3 的环境；source scaffold smoke、npm staged scaffold smoke、baseline runner、summary 和 artifact audit 都消费 `checkedGeneratedReadmePrerequisites`，缺失时报告 `standaloneSkillRepo.checkedGeneratedReadmePrerequisites` 或 `npmStagedSkillRepo.checkedGeneratedReadmePrerequisites`。 +- 2026-06-27：再次复跑完整 opt-in `make record-and-replay-baseline-smoke` 通过；本次覆盖默认 event-stream matrix、截图上下文、真实输入 action、官方 non-recording surface、official no-active response、hostless raw timeout fixture、official fixture set gate、official/OCU ingest、capture/pairing preflight、source standalone repo smoke 和 npm staged standalone repo smoke。最终摘要继续证明 `usableBaseline=true` 与 `standaloneRepoBaselineReady=true`，并继续报告 required `simple-action-stop` 与 recommended official successful recording fixtures 缺失，因此 `officialSuccessfulRecordingEquivalenceReady=false` 的边界不变。 +- 2026-06-27：strict official golden gate audit 复跑时发现 audit Make target dry-run smoke 会继承外层 `RNR_BASELINE_SUMMARY_JSON`，从而把“默认 summary path”误判为自定义路径；`scripts/test-record-and-replay-baseline-audit-make-targets.py` 已在内部 `make -n` 调用前移除继承环境。后续又将 strict audit 默认 artifact 拆到 `dist/record-and-replay-official-golden-gate-summary.json`，并新增 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`，避免缺 required official sample 时的预期失败摘要覆盖 `dist/record-and-replay-baseline-summary.json`。`RNR_BASELINE_SUMMARY_JSON=/tmp/outer-should-not-affect-default.json RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/outer-golden-should-not-affect-default.json make record-and-replay-baseline-audit-targets-smoke` 已验证通过；默认 `scripts/ci.sh` 也改为同时带两个外层变量运行同一 smoke，把该回归纳入默认 CI。 +- 2026-06-27：official capture packet README 的 repo-root 说明已对齐真实 wrapper 行为：`inspect-only.sh` / `import-fixture.sh` 等脚本会写入创建 packet 时的仓库路径，并允许通过 `REPO_ROOT=/path/to/repo` 覆盖；不再误称通过 `git rev-parse` 推断仓库。official capture preflight smoke 已断言 README 包含 embedded repo path / `REPO_ROOT` 覆盖说明且不再出现 `git rev-parse`。 +- 2026-06-28：npm package build 现在要求 `dist/record-and-replay-baseline-summary.json` 已存在并复制进 `open-computer-use` npm 包；缺失时 `scripts/npm/build-packages.mjs` 会以 `Missing Record & Replay baseline summary artifact` 失败并提示先运行 `make record-and-replay-baseline-audit`。npm staged scaffold smoke 会临时移走该 artifact 覆盖负例，确保安装态 standalone scaffold 不能发布成缺 `evidence/source-baseline-summary.json` 的半成品。 +- 2026-06-28：baseline summary 的 missing official golden `nextActions` 现在把采样后的最终命令指向 `make record-and-replay-official-golden-gate-audit`，并由 artifact audit 的 `checkedMissingGoldenNextActionStrictAuditStep` 复审。这样后续导入 required `simple-action-stop` official successful recording 后，默认下一步会直接刷新 strict summary artifact，而不是只跑一次不留证的 strict gate。 +- 2026-06-28：official capture packet 新增 `capture-contract.json`，记录 scenario、fixture name、预期 action event、预期 `endReason`、status handoff path evidence 和 MCP transcript evidence requirement。`verify-inputs.sh` 现在不只拒绝占位/非法 JSON，还会确认 hosted status JSON 至少包含 `eventsPath`，并且包含 `metadataPath` 或 `sessionPath` 这类 handoff 文件 evidence；`suppressedEventsPath` 和 `sessionDirectoryPath` 只作为 optional handoff evidence 记录，以兼容官方 hosted 最小 `events.jsonl + session.json` 包。transcript enabled 时还会确认 `inputs/mcp-transcript.json` 包含 response-shape 或 transcript evidence。`inspect-only.sh` / `import-fixture.sh` 在执行导入前会先跑同一语义校验；official capture preflight smoke 输出 `checkedCapturePacketInputSemanticGuard=true`，baseline runner、summary 和 artifact audit 通过 `checkedOfficialCapturePacketInputSemanticGuard` 消费该 evidence。 +- 2026-06-28：official capture packet 的 strict handoff 现在也作为 baseline evidence 固化。`strict-golden-gate.sh` 实际运行 `make record-and-replay-official-golden-gate-audit`，official capture preflight smoke 输出 `checkedCapturePacketStrictAuditHandoff=true`，baseline summary / artifact audit 消费 `checkedOfficialCapturePacketStrictAuditHandoff` / `checkedOfficialCapturePacketStrictAuditHandoffEvidence`，避免 official fixture 入库后只跑不落盘的 strict gate。 +- 2026-06-28：official capture packet 和 packet set 继续补 `strict-expected-failure-audit.sh`。该 wrapper 审计 strict summary artifact 的 `--allow-strict-official-golden-missing` 口径，official capture preflight smoke 输出 `checkedCapturePacketStrictExpectedFailureAuditHandoff=true`，baseline summary / artifact audit 消费 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff` / `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence`，避免 required official golden 尚缺时无法机器判断 strict 失败是否预期。 +- 2026-06-28：recommended capture packet set 的根级 `capture-packets.json` 继续补齐集合级合同，新增 `captureContracts` 和 `captureContractPaths`，直接内联每个子 packet 的 `capture-contract.json` 内容与路径。official capture preflight smoke 输出 `checkedCapturePacketSetContractManifest=true`，baseline runner、summary 和 artifact audit 通过 `checkedOfficialCapturePacketSetContractManifest` 消费该 evidence；后续独立 repo 或 skill 可以只读根 manifest 获取每个 scenario 的 expected action event、expected `endReason`、handoff path evidence 和 transcript requirement。 +- 2026-06-28：official capture packet / packet set 继续补齐机器可读 post-capture 顺序。单场景 `capture-contract.json`、返回 JSON 和根级 `capture-packets.json` 都写入 `postCaptureWorkflow`，覆盖替换 hosted status、按需替换 transcript、verify、inspect、import、coverage / fixture-set gate、可选 OCU candidate ingest、strict gate 和 strict expected-failure audit。official capture preflight smoke 输出 `checkedCapturePacketPostCaptureWorkflow=true` / `checkedCapturePacketSetPostCaptureWorkflow=true`，baseline summary / artifact audit 消费 `checkedOfficialCapturePacketPostCaptureWorkflow` / `checkedOfficialCapturePacketSetPostCaptureWorkflow` 与对应 evidence gate，避免采样包只提供 wrapper 文件但没有可机器消费的执行顺序。 +- 2026-06-28：standalone source summary projection 继续继承 official capture packet set 合同证据。生成态 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketSetContractManifest`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketSetContractManifest=true` 并带缺失负例；`record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 也声明 `checkedOfficialCapturePacketInputSemanticGuard` 与 `checkedOfficialCapturePacketSetContractManifest`。source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费该 evidence，避免后续拆独立 repo 时丢失 recommended packet set 的集合级 `captureContracts` 合同。 +- 2026-06-28：standalone source summary projection 继续继承 official capture packet post-capture workflow evidence。生成态 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow` 与 `preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true` 和 `checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true` 并带缺失负例；`record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 同步声明 `checkedOfficialCapturePacketPostCaptureWorkflow` 与 `checkedOfficialCapturePacketSetPostCaptureWorkflow`。source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费这些 evidence，避免后续拆独立 repo 时丢失 capture packet / packet set 录制后的机器可读 handoff 顺序。 +- 2026-06-28：standalone source summary projection 也继承 official capture packet strict audit handoff。生成态 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true` 并带缺失负例；`record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 同步声明 `checkedOfficialCapturePacketStrictAuditHandoff`。source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费该 evidence，避免后续拆独立 repo 时丢失 official fixture 入库后刷新 strict summary artifact 的 handoff。 +- 2026-06-28：standalone source summary projection 也继承 official capture packet strict expected-failure handoff。生成态 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，`scripts/verify-source-baseline-summary.py` 输出 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true` 并带缺失负例；`record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 同步声明 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`。source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费该 evidence，避免后续拆独立 repo 时丢失 required official golden 尚缺时的 strict expected-failure audit handoff。 +- 2026-06-28：standalone source summary verifier 不再把“缺 `simple-action-stop` required official golden”写成唯一合法状态。生成态 `scripts/verify-source-baseline-summary.py` 现在接受两种自洽状态：当前 required gap 明确且 equivalence 未声明，或未来 official golden gate / successful recording equivalence 已完成且 required gaps / readiness gaps / coverage errors 为空；如果 summary 仍缺 required scenario 却声明 `officialSuccessfulRecordingEquivalenceReady=true` 会失败。source scaffold smoke 和 npm staged scaffold smoke 都新增 completed-equivalence 正例与矛盾状态负例，避免将来 official golden 入库后独立 repo 自检被旧 gap 合同卡住。 +- 2026-06-28：baseline summary 现在把 standalone source summary official-state 自洽检查提升为 release / standalone audit evidence。source scaffold smoke 和 npm staged scaffold smoke 都输出 `checkedSourceBaselineSummaryOfficialGoldenState=true`，shared baseline contract、summary builder 和 artifact audit 都消费该字段；缺失时会报告 `standaloneSkillRepo.checkedSourceBaselineSummaryOfficialGoldenState` 或 `npmStagedSkillRepo.checkedSourceBaselineSummaryOfficialGoldenState`，避免 generated repo verifier 已经守住 official golden 状态，但最终 baseline artifact 没有审计这条证据。 +- 2026-06-28：standalone generated repo 默认自检继续补 package artifact gate。生成态 `scripts/check.sh` 现在在 `scripts/package-skill.sh` 后运行 `scripts/verify-package-artifact.py`，校验 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` frontmatter 和 Record & Replay handoff 片段仍存在；source scaffold smoke、npm staged smoke、baseline summary 和 artifact audit 都消费 `checkedPackageArtifact=true`，避免后续拆独立 repo 时只证明 `.skill` 文件存在而没有证明制品内容。 +- 2026-06-28：official capture packet / packet set 新增 `verify-workflow.sh`。单场景 verifier 会读取 `capture-contract.json` 并校验 `postCaptureWorkflow` 的 input / command、生成 wrapper、可执行 bit 和 transcript 开关一致；packet set verifier 会读取 `capture-packets.json`，确认根级 scripts manifest、每个子 `capture-contract.json` 和子 wrapper 一致。official capture preflight smoke 输出 `checkedCapturePacketWorkflowVerifier=true` / `checkedCapturePacketSetWorkflowVerifier=true`，baseline summary / artifact audit 消费 `checkedOfficialCapturePacketWorkflowVerifier` / `checkedOfficialCapturePacketSetWorkflowVerifier`，standalone source summary 继承为 `checkedSourceBaselineSummaryCapturePacketWorkflowVerifier` / `checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier`，避免采到 hosted official JSON 前 packet 自身的执行计划已经漂移。 +- 2026-06-28：official hosted capture transcript gate 对齐官方 skill handoff。官方 successful recording 不再要求额外 `repeatStartResponseShape`，而是要求 `event_stream_start`、active `event_stream_status`、`event_stream_stop` 和 final `event_stream_status` 对应的 `startResponseShape` / `statusResponseShape` / `stopResponseShape` / `finalStatusResponseShape`。新增 `scripts/finalize-record-and-replay-official-capture-packet.py` 和 `make record-and-replay-official-capture-finalizer-smoke`，用于把 Codex-hosted start/status/stop/final-status JSON 写入 capture packet inputs 并生成 `mcp-transcript.json`，随后自动跑 `verify-workflow.sh` / `verify-inputs.sh`，减少真实录完 official sample 后手工拼 transcript 的风险。 +- 2026-07-01：required `simple-action-stop` official successful recording fixture 已通过 Codex-hosted Record & Replay 采集并脱敏入库，`check-event-stream-golden-readiness.py`、coverage `--require-readiness` 和 `make record-and-replay-official-golden-fixture-gate` 均通过。当前状态从 required missing 推进到 required ready；recommended `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 仍待采样和校准。 +- 2026-07-01：standalone scaffold manifest 的 `officialEvidence.hasSuccessfulRecordingGolden` 改为从 source baseline summary 派生，生成态 verifier 校验其与 copied `evidence/source-baseline-summary.json` 一致；Record & Replay Python helper 补齐 future annotations，保证 macOS system Python 3.9 可直接运行 capture packet、ingest、readiness 和 scaffold 入口。 diff --git a/docs/histories/2026-06/20260626-1737-document-record-and-replay-plan.md b/docs/histories/2026-06/20260626-1737-document-record-and-replay-plan.md new file mode 100644 index 00000000..54ab507a --- /dev/null +++ b/docs/histories/2026-06/20260626-1737-document-record-and-replay-plan.md @@ -0,0 +1,608 @@ +## [2026-06-26 17:37] | Task: Record & Replay baseline + +### 🤖 Execution Context +* **Agent ID**: `Codex` +* **Base Model**: `GPT-5` +* **Runtime**: `Codex CLI` + +### 📥 User Query +> 把 Record & Replay 复刻方案和相关逆向上下文落到 docs，后续基于这个推进。 + +### 🛠 Changes Overview +**Scope:** docs, macOS runtime, OpenComputerUseKit tests + +**Key Actions:** +- **[Plan]**: 新增 active execution plan，明确 macOS event-stream 复刻目标、范围、里程碑、验证方式和关键决策。 +- **[Reference]**: 新增官方 Record & Replay 逆向参考，沉淀插件包装、MCP surface、Codex.app asar 分工、Computer Use 二进制字符串、AX diff 和截图上下文线索。 +- **[Navigation]**: 在 reverse-engineering README 中挂载新参考文档，方便后续 Agent 发现。 +- **[Baseline]**: 新增 Record & Replay 兼容 `event-stream mcp`，暴露 `event_stream_start`、`event_stream_status`、`event_stream_stop` 三个无参数 tools，并对齐官方 descriptions、annotations 和 Record & Replay MCP protocol version。 +- **[Session Files]**: 新增 event-stream session 文件模型，写入 `events.jsonl`、`metadata.json`、兼容 alias `session.json`、`suppressed.jsonl`、`latest-session.json` 和 active session 状态。 +- **[CLI]**: 新增 `open-computer-use event-stream start/status/stop/wait --json`，并通过 app-agent proxy 复用长驻进程里的 recorder 状态。 +- **[Wait Semantics]**: 修复并发 `wait` / `stop` race，确保最终 metadata 写入后再清理 active session；app-agent 对 `status/stop/cancel/wait` 不再持有环境覆盖全局锁，避免 listener 阻塞结束命令。 +- **[Events]**: baseline 记录 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.click`、`mouse.context_menu`、`mouse.drag`、`keyboard.text_input`、`keyboard.submit`、`keyboard.shortcut`、`terminal.value_changed`、`selection.changed`、`debug.error`、`session.ended`，并附带基础 app/window attribution。 +- **[Input Context]**: `mouse.click` / `mouse.context_menu` 现在会尽量附带 `targetAccessibilityElement`，键盘事件现在会尽量附带 `focusedAccessibilityElement`；click 坐标明确区分 screen-state `location` 和 `appKitLocation`；`mouse.drag` 当前记录起止 screen-state / AppKit 坐标、距离、时长和目标 AX 摘要，payload 字段结构仍待 official golden recording 校准；`experimentalRawEvents` 当前记录低层 NSEvent 摘要，scroll wheel 在官方独立事件名未知时只写 raw-only 事件。 +- **[Controls]**: 新增 OCU 自有 Record & Replay 控制条，显示录制中、计时、Done / Discard，并支持拖动；Discard 走 `recording_controls_cancelled`。 +- **[Cancel]**: 新增 `open-computer-use event-stream cancel --json` 作为 OCU CLI 扩展，MCP surface 仍保持官方三件套。 +- **[AX Payload]**: `AX.focusedWindowChanged` 事件现在会尽量带 `accessibilityInspectorPayload`；首帧为 full tree，同 app/window 的后续事件使用轻量 `~` / `+` / `-` compact diff，diff payload 同时带 previous diff 和 cumulative diff baseline，失败写入 `suppressed.jsonl`。 +- **[AX Diff Coverage]**: AX compact diff baseline 从相邻行启发式改为 LCS 风格行 diff，比较时归一化 AX tree leading element index，降低插入元素导致后续稳定行被误判为变更的噪音；单测覆盖 changed / added / removed / over-budget fallback。 +- **[AX Payload Service Coverage]**: `EventStreamService` 的 snapshot-to-payload diff 状态机已抽成生产和测试共用 helper；新增服务级测试确认 full payload、previous diff 和 cumulative diff 会写入真实 session `events.jsonl` 的 `AX.focusedWindowChanged` 事件。 +- **[Window Context]**: 新增 input 前 focused window context 检测；如果 app bundle + frontmost window title 变化，会在输入事件前先写顶层 `window.changed`，再补 `AX.focusedWindowChanged`，覆盖同一 app 内窗口切换后的上下文缺口；字段、初始帧语义和事件顺序待 golden recording 校准。 +- **[Screenshot Context]**: 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never`，在 AX payload 需要额外视觉上下文时写入 session `screenshots/` 并返回 `screenshotNeededForContext`、`screenshotAvailable`、`screenshotPath`。 +- **[Screenshot Context Coverage]**: 新增服务级测试，用可控 PNG snapshot 验证 `always` 策略会把截图写入当前 session `screenshots/`，且 `accessibilityInspectorPayload.screenshotPath` 指向可解析图片。 +- **[Time Limit]**: 按官方 “up to 30 minutes” 描述新增最长录制时长，默认 30 分钟自动 stop 并写入 `recording_time_limit_reached`；该 endReason 暂为 OCU baseline，后续用 official golden recording 校准。 +- **[Tests]**: 补 parser、proxy decision、MCP surface、session 文件生命周期、wait/stop 并发、最长录制时长、AX diff、控制条开关和截图策略单测；新增 `scripts/run-event-stream-smoke-tests.sh` 和 `make event-stream-smoke`，用可复跑 MCP smoke 验证 event-stream 文件产物；默认 smoke 跳过截图采集以避免锁屏/登录窗口环境的 ScreenCaptureKit 噪音；截图 smoke 在 snapshot 有截图数据时验证 `screenshotPath` 落盘，无截图数据时验证 payload 标记；timeout smoke 通过 MCP start/status 验证自动结束和 `recording_time_limit_reached`。 +- **[Docs Handoff]**: 补充后续推进入口、P0/P1/P2/P3 分层、验收门槛和官方兼容层 / OCU 扩展层边界，明确第一版通知模型是 `wait` listener，主动 callback/webhook 后续只放在 OCU 扩展层。 +- **[Design Note]**: 新增 Record & Replay 复刻设计入口，把“直接复刻官方 observable behavior + OCU 自有控制条/wait/通知扩展”的分层方案、推进顺序、验收门槛和当前缺口集中沉淀到 `docs/design-docs/`。 +- **[Drag Events]**: 鼠标录制从 mouse-down 立即写 click 改为 down / dragged / up 后判定，补齐官方已确认的 `mouse.drag` baseline，避免拖拽误记成点击。 +- **[Raw Events]**: 按官方字符串补齐 `experimentalRawEvents` baseline，点击、拖拽、键盘事件会附带 raw 摘要，并用 raw-only 事件保留 scroll wheel；新增 `OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS=0` 关闭开关。 +- **[Selection]**: 按官方字符串补齐轻量 `selection.changed` baseline；输入事件后如果 focused AX element 暴露非空 `selectedText` 且 selection signature 变化，会写入 selected text、focused element summary 和 app/window attribution；如果此前有选择而当前选择为空，会写 `selectionCleared=true` 清空事件，字段与触发时机仍待 official golden recording 校准。 +- **[Secure Input]**: 按官方 `secureInput` 字符串线索补键盘文本脱敏 baseline；secure/protected focused element 下不写明文 `text` 或 raw `characters`，只保留长度和 redaction 标记。 +- **[Terminal Events]**: 按官方 `terminal.value_changed` 字符串补齐轻量 baseline；明显终端上下文下 focused AX value 变化时记录 value hash / length 和脱敏 focused element，同时避免键盘 raw characters 与 focused value 写入终端缓冲区明文；字段与触发时机仍待 official golden recording 校准。 +- **[Debug Events]**: 按官方 `debug.error` 字符串补齐轻量 baseline；AX snapshot、cumulative AX diff fallback 和 screenshot 写入失败时会在主事件流写入低敏 debug metadata，完整错误细节继续进入 `suppressed.jsonl`。 +- **[Input Monitoring Diagnostics]**: 如果本地 input monitor 全部安装失败，录制仍会启动但会写入 `debug.error` 主事件，标记 `subsystem=inputMonitoring` / `reason=inputMonitorsUnavailable`，避免上层误把无输入事件的录制当成正常空录制。 +- **[Start Approval Handoff]**: 补充 start approval / MCP elicitation 推进方案：官方兼容 MCP surface 仍保持三件套无参数，OCU 独立模式通过自有开始确认 UI 或 policy 承载用户确认；后续先在 service 层沉淀 approved / denied / cancelled 语义，再接 UI 和可选 MCP elicitation。 +- **[Start Approval Baseline]**: 在 `EventStreamService.start()` 前接入 approval gate；已有 active session 时不重复确认，新 session 只有 approved 后才创建；denied / cancelled 返回官方已观察错误文案且不写 session；新增 `OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=auto|interactive|approve|deny|cancel|mcp|elicitation`，交互式 runtime 使用 OCU 自有开始确认弹窗，smoke 显式 auto-approve，MCP elicitation smoke 显式走 `mcp`。 +- **[MCP Elicitation Baseline]**: 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp|elicitation`；event-stream MCP server 会在 host `initialize` 声明 `capabilities.elicitation` 时发送内部 `elicitation/create` approval request，accept 后才创建 session，decline / cancel / unsupported 不创建 session；该路径不改变官方三件套 tools/list 或 input schema。后续已按 Codex.app host schema 校准 form request shape，官方业务 message / `_meta` 仍待 golden 样本。 +- **[Official Initialize Alignment]**: 复核官方 `SkyComputerUseClient event-stream mcp` 的 initialize/tools-list 输出；确认三件套 tools 与当前 schema 一致，并将 OCU event-stream initialize 调整为不返回 `instructions`，对齐官方当前 behavior。 +- **[Wait Listener Coverage]**: 补 `runOpenComputerUseEventStream(.wait...)` 的 CLI dispatcher 测试，验证 wait listener 能被同一 runtime 内的 stop 唤醒并返回最终 stopped session。 +- **[Wait Timeout Signal]**: `event-stream wait --json` 返回体新增 OCU 扩展字段 `waitTimedOut`，上层 plugin/skill 可区分 stop/cancel 唤醒和 timeout 返回；官方兼容 MCP surface 不暴露该字段。 +- **[Wait Session Match Signal]**: `event-stream wait --json --session-id ...` 返回体新增 OCU 扩展字段 `waitSessionMatched`；wait 会优先读取目标 session 目录里的完成态 metadata，因此 latest session 被后续录制覆盖后仍能返回指定历史 session。未知或不匹配 session id 会快速返回 `waitTimedOut=true` / `waitSessionMatched=false` 并跳过 notify callback，避免独立集成方无限等待错误 id。 +- **[Wait Timeout Smoke]**: `run-event-stream-smoke-tests.sh` 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1`，通过真实 app-agent CLI start/wait/cancel 路径验证 `waitTimedOut=true`。 +- **[No Active Smoke]**: `run-event-stream-smoke-tests.sh` 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1`，通过真实 MCP initialize/tools-list/status/stop 路径验证无 active recording 时返回 idle 且不写 session 文件。 +- **[Official Surface Smoke]**: `run-event-stream-smoke-tests.sh` 增加 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1`，可选启动官方 Record & Replay client 只做 initialize/tools-list 对比，避免触发真实录制。 +- **[Lifecycle Smoke]**: 扩展 event-stream MCP smoke，覆盖 active session 重复 start、stop 后重复 stop/status，以及 timeout 后 stop 的幂等行为，固定官方 one-active / idempotent 语义。 +- **[Approval Smoke]**: `run-event-stream-smoke-tests.sh` 增加 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=deny|cancel`，通过真实 MCP stdio 路径验证 approval 被拒绝时返回官方错误文案且不创建 session 文件。 +- **[App-Agent Wait Smoke]**: 修正 app-agent event-stream MCP server 初始化时机，并为 proxied `event-stream mcp` 请求转发外层 `OPEN_COMPUTER_USE_*` 环境，避免 `event_stream_start` 的环境覆盖生效前提前初始化共享 recorder；新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1`，通过 app-agent socket 验证 MCP start 写入指定 recordings root、共享 recorder 状态和 CLI `wait` 被并发 `stop` 唤醒。 +- **[Stale Session Recovery]**: 当新 recorder 进程只看到旧 active `latest-session.json` 时，会持久化写回 `state=stopped` / `endReason=recording_process_unavailable`，同步 `metadata.json` / `session.json` 并清理 stale `active-session.json`，避免独立集成方持续看到幽灵录制。 +- **[No Active Unit Coverage]**: 在单元层补 `event_stream_status` / `event_stream_stop` 未 start 时的 idle 语义断言，固定无 session id、无路径、且不创建 `latest-session.json` / `active-session.json` 的行为；同时复跑官方 surface smoke，只做 initialize/tools-list 对比，不触发官方录制。 +- **[Official Surface Fixture]**: 新增官方 `record-and-replay` 1.0.857 的 normalized `initialize` / `tools/list` fixture，作为非录制 surface 漂移对比证据；该 fixture 不包含 `event_stream_start` 输出或用户录制内容。 +- **[Official Fixture Regression]**: 将官方 non-recording surface fixture 接入单元测试；本地 `event-stream mcp` 的 protocol/name/capabilities/no-instructions 和三件套 tools 完整 JSON 必须持续匹配 fixture，server version 单独忽略以适配 OCU 自身版本。 +- **[Skill Integration Reference]**: 在 `skills/open-computer-use` 中新增 Record & Replay reference，说明 `event-stream mcp` 官方三件套、session 文件、OCU `wait/cancel` 扩展和安全边界，为后续独立薄 skill/repo 提供可复用蓝本。 +- **[Official Smoke Timeout]**: 为 official surface smoke 的官方 MCP 响应读取增加默认 10 秒超时和 `OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT` 覆盖；补充逆向观察，裸官方 `event_stream_status` 可能依赖宿主/runtime 状态并挂起，因此自动化官方对比继续只覆盖 initialize/tools-list。 +- **[Recording Validator]**: 新增 `scripts/validate-event-stream-recording.py`，校验 session 目录或 metadata/session 文件的 JSONL 可解析性、metadata/session alias、event/suppressed counts、final `session.ended`、`endReason` 和 `screenshotPath`；OCU smoke 已接入 strict 模式，官方 golden recording 后续可先用非 strict 模式做结构检查。 +- **[Recording Summary]**: 新增 runtime CLI `open-computer-use event-stream summarize --json` 和源码脚本 `scripts/summarize-event-stream-recording.py`，把录制事件流提取为 event types、window、action sequence、target/focused element、selection、debug 和 redaction evidence；默认只输出 AX `value` / `selectedText` 长度、不输出原文，event-stream smoke 已接入读取验证，为后续根据 recording 创建 skill 提供前置审计入口。 +- **[Runtime Validator]**: 新增 runtime CLI `open-computer-use event-stream validate --json`，让安装态 OCU 可以直接校验 metadata/session alias、event/suppressed counts、final `session.ended`、`endReason`、required event type 和 `screenshotPath`，独立 skill/plugin 不再依赖源码 checkout；默认、timeout、wait-timeout 和 app-agent wait smoke 同时调用 runtime validator 与源码 Python validator 做交叉检查。 +- **[Runtime Declared Path Gate]**: runtime Swift validator 与源码 Python validator 在 `--strict-ocu` 下新增 declared handoff path gate，输出 `declaredPaths`，并要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个字段都存在且解析后文件存在；Swift 单测和 skill scaffold smoke 覆盖完整路径、缺失四字段和坏 `sessionPath`，skill scaffold 测试夹具同步补齐 `sessionPath` / `session.json`。 +- **[Events-Only Validation]**: runtime Swift validator 与源码 Python validator 现在支持非 strict `events.jsonl` 输入；只有主事件流时也能检查 JSONL、required event type、完成态、skill draft gate、从 `session.ended.endReason` 推断取消态和阻断性诊断。strict OCU 仍要求 metadata/session alias，runtime/source summary 和 scaffold 也同步支持 `` 输入并拒绝 cancelled events-only stream。 +- **[Skill Draft Validation Gate]**: runtime CLI 与源码 Python validator 新增 `--require-skill-draft`，在结构校验之外要求至少一个高层动作且 recording 未取消；action smoke 和 scaffold smoke 覆盖可生成草稿与 cancelled 拒绝路径，方便独立 skill/plugin 在 scaffold 前先做硬 gate。 +- **[Fixture Import]**: 新增 `scripts/import-event-stream-fixture.py` 和 `make event-stream-fixture-smoke`，为后续官方 golden recording 入库提供脱敏导入路径；默认改写本机路径、session id、timestamp、app attribution、窗口标题、AX value / selectedText，并移除 screenshot path；runtime validator / summary 和 Python helper 同步支持 fixture metadata 中的相对路径。`--mcp-transcript` 会把 probe output 脱敏成 `mcp-transcript.json`，用于保存官方 `event_stream_*` response shape。 +- **[Golden Readiness Gate]**: 新增 `scripts/check-event-stream-golden-readiness.py` 和 `make event-stream-golden-readiness-smoke`，把“这个 fixture 是否足够作为 official/OCU golden baseline”从人工判断变成可复跑 JSON gate；默认要求 fixture manifest/source/version、final state、`session.started` 恰好一次且为首事件、`session.ended`、动作事件和 AX payload，full/diff/cumulative AX 证据可按校准阶段显式要求，也可要求 `mcp-transcript.json` 和指定 `event_stream_start/status/stop` response 证据。 +- **[MCP Timeout Evidence]**: golden readiness gate 对 required MCP tool timeout 给出明确 `MCP tool timed out` 错误；`make event-stream-golden-readiness-smoke` 覆盖只有 timeout shape 的 transcript 不会被误判为成功 response golden。 +- **[Fixture Compare]**: 新增 `scripts/compare-event-stream-recordings.py` 和 `make event-stream-compare-smoke`，把官方 fixture 与 OCU candidate 的 event type sequence、event counts、metadata keys 和 per-event JSON schema path/type 差异变成可复跑 JSON 报告，并覆盖相同样本通过、缺字段失败和事件顺序漂移失败。 +- **[AX Diff Compare Gate]**: fixture import 在脱敏 `treeLines` / `cumulativeTreeLines` 时保留 `+` / `-` / `~` marker；recording compare 新增 `--require-ax-diff-evidence` 和 `--require-same-ax-diff-markers`，可要求 OCU candidate 保留官方 fixture 中出现的 AX diff/cumulative payload 和 marker 类型；`make event-stream-compare-smoke` 覆盖缺 evidence 与缺 marker 失败路径。 +- **[Semantic Field Compare Gate]**: recording compare 新增 `--require-semantic-fields`，按事件类型比较官方 baseline 中出现的 replay-critical 字段是否也出现在 OCU candidate 中；覆盖 target AX、keyboard focused AX、drag 起止点、selection / terminal 摘要字段、window attribution 和 AX treeLines / cumulativeTreeLines，避免只看事件序列或完整 schema 时漏掉可复用 skill 所需的关键证据。 +- **[Metadata Key Compare Gate]**: recording compare 新增 metadata key diff 和 `--require-same-metadata-keys`，把 metadata/session 文件布局键漂移变成可失败的官方校准 gate；`make event-stream-compare-smoke` 覆盖 candidate 缺 metadata key 的失败路径。 +- **[Metadata Value Compare Gate]**: recording compare 新增 stable metadata value diff 和 `--require-same-metadata-values`，比较 `state`、`active`、`endReason`、`eventCount`、`suppressedEventCount`，避免 metadata key 一致但 stop/cancel/final count 状态漂移时仍通过；`make event-stream-compare-smoke` 覆盖 candidate endReason 漂移失败路径。 +- **[Lifecycle Compare Gate]**: recording compare 的 `--require-final-session-evidence` 保留兼容 flag 名，但 evidence 已扩展为 start/end lifecycle：比较 `session.started` 个数、是否为首事件、`session.ended` 个数、是否为最后事件以及 endReason 集合；`make event-stream-compare-smoke` 覆盖 candidate 缺失 start boundary 和最后事件不是 `session.ended` 的失败路径。 +- **[Wait Notify Callback]**: 新增 `open-computer-use event-stream wait --notify-command ''` 作为 OCU 独立集成 callback 扩展；结束唤醒时通过 stdin/env 传最终 status JSON，wait timeout 时跳过 callback，callback 非零退出或超时会让 CLI 返回 error；单元测试覆盖成功通知和 timeout 跳过。 +- **[Wait Notify Smoke]**: 扩展 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1`，在真实 app-agent socket 路径下验证 `wait --notify-command` 能收到 stop 后最终 status JSON、session/state env 和成功 `notification` 结果。 +- **[Wait Lifecycle Coverage]**: 补 `event-stream wait` 生命周期覆盖，固定已结束 session 再 wait 会立即返回最终状态，cancel/Discard 会唤醒 listener，并把 `recording_controls_cancelled` 传给 notify callback；app-agent wait smoke 也覆盖 completed wait。 +- **[Session Alias Handoff]**: status / metadata JSON 增加 OCU 便利字段 `sessionPath`，指向 `session.json` alias；`wait --notify-command` 同步暴露 `OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH`，validator 和 fixture import 会校验 / 相对化该字段，方便独立 skill 在取消路径直接读取 session alias。 +- **[Skill Handoff Smoke]**: 默认 event-stream smoke 现在按官方 skill 消费路径从 `event_stream_stop` 的 text JSON 读取 `eventsPath` / `metadataPath` / `sessionPath`，并使用 `sessionPath` alias 再跑 runtime validate/summarize，固定独立 skill/plugin 的路径消费语义。 +- **[Smoke Matrix]**: 新增 `scripts/run-event-stream-smoke-matrix.sh` 和 `make event-stream-smoke-matrix`，默认跑稳定本地 baseline 矩阵：lifecycle、no-active、timeout、wait timeout、approval denied/cancelled、MCP elicitation approval、app-agent wait/notify、fixture import 和 recording compare；截图与官方 surface 对比保持 opt-in。 +- **[CI Gate]**: 将 shell / Node / Python 脚本语法检查、`npm run package:skill`、Record & Replay surface drift smoke、`swift test` 和稳定 Record & Replay smoke matrix 接入 `scripts/ci.sh`，并补齐最小 `.github` workflow / issue / PR / dependency-review 骨架，使仓库默认 CI 入口能守住 event-stream baseline 与 thin skill 制品；截图和官方 bundle 对比仍保持 opt-in。 +- **[Thin Skill]**: 新增 `open-computer-use-record-and-replay` 独立 skill，聚焦 macOS 录制、停止、读取 `eventsPath` / `metadataPath` / `sessionPath`、validate / summarize 和生成可复用 skill 的流程；它不依赖 Codex.app 私有 UI，后续可迁到独立 repo。 +- **[Skill Packaging]**: `scripts/package-skill.sh` 从 hardcoded 单 skill 改为扫描 `skills/*/SKILL.md`，逐个校验 frontmatter、打包 `.zip` / `.skill` 并写入多 skill manifest；该打包路径已接入默认 CI。 +- **[Surface Drift Smoke]**: 新增 `scripts/compare-event-stream-surface.py` 和 `make event-stream-surface-smoke`，只调用 initialize / `tools/list`，用官方 1.0.857 non-recording fixture 校验 OCU `event-stream mcp`；脚本支持 `--use-default-official` opt-in 复核本机官方 bundle，当前本机官方对比通过且不启动录制。 +- **[MCP Elicitation Smoke]**: 新增 event-stream MCP elicitation approval 单测与 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1` smoke；默认 smoke matrix 已覆盖该路径，验证真实 stdio MCP server 发出 `elicitation/create`、接收 accept 后开始录制，并继续通过 stop / validate / summarize 收敛。 +- **[Official Recording Probe]**: 新增 `scripts/probe-event-stream-recording.py`，支持 local OCU 和官方 bundled Record & Replay client 的 raw MCP transcript 探测；local `--start-stop` 可完成 start/status/stop，官方只读 initialize/tools-list 可返回，官方 raw start/status/stop 当前在宿主外超时且未产出 recording paths。新增 `make event-stream-probe`、`make event-stream-official-probe` 和 `make event-stream-official-start-probe`。 +- **[Official Probe Fixture]**: 将官方 1.0.857 raw start/status/stop 宿主外 timeout 观测脱敏保存为 `record-and-replay-official-raw-start-timeout-1.0.857.json`；新增 `scripts/test-event-stream-probe-fixtures.py` 和 `make event-stream-probe-fixture-smoke`，默认 CI 校验该 fixture 不含本机绝对路径且仍表达三件套 surface 与 start/status/stop timeout 边界。 +- **[MCP Elicitation Host Schema]**: 复核 Codex.app asar 的通用 MCP elicitation schema 和 pending request panel，确认 host form path 使用 `mode: "form"`、object `requestedSchema` 和 `accept|decline|cancel` response；OCU start approval request 改为显式 `mode: "form"` + 空对象 schema，去掉 request schema 内的 `additionalProperties`，并让 probe fixture 摘要记录 server-initiated elicitation request shape。 +- **[Probe Elicitation Coverage]**: `scripts/probe-event-stream-recording.py` 的 server-initiated elicitation response 已按 host schema 校准：accept 发送 `content: {}`，decline / cancel 发送 `content: null`；新增 `--elicitation-action accept|decline|cancel`，保留旧 `--decline-elicitation` 兼容别名；新增 `scripts/test-event-stream-recording-probe.py` fake MCP server smoke，覆盖 request schema 摘要提取、三种 action 应答形状和兼容别名，并接入 `make event-stream-probe-fixture-smoke` 与默认 CI。 +- **[Recording-To-Skill Scaffold]**: 新增 `scripts/scaffold-event-stream-skill.py`，从 validated / summarized recording 生成第一版 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`；新增 `scripts/test-event-stream-skill-scaffold.py`、`make event-stream-skill-scaffold-smoke` 并接入默认 CI，固定录制后 handoff 不只停留在文档流程。 +- **[Runtime Skill Scaffold]**: 新增 `open-computer-use event-stream scaffold-skill --json`，复用 Swift summary helper 在安装态生成第一版 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`；独立 thin skill / repo 不再需要 checkout 本仓库源码才能完成 recording-to-skill handoff。 +- **[Runtime Scaffold Smoke]**: 扩展 `scripts/test-event-stream-skill-scaffold.py`，同一临时 recording 同时经过源码 Python helper 和 runtime CLI `event-stream scaffold-skill --json`，并检查两条生成路径都不泄漏本机绝对路径。 +- **[Replay-Oriented Scaffold]**: 扩展 Swift runtime scaffold 和源码 Python scaffold 的生成模板，`SKILL.md` 草稿现在会写出 runtime inputs、`get_app_state` / semantic `element_index` 优先的 agent replay procedure、observed replay steps 和 verification checklist；单测与 smoke 同步断言这些章节，避免录制到 skill 的 handoff 只剩事件摘要。 +- **[Skill Finalization Scaffold]**: 继续对齐官方 Record & Replay skill 的“创建实际可发现 skill”要求，生成的 `SKILL.md` 草稿新增 finalization checklist，明确不能停在 runbook / replay plan，最终化前应读取并完成 `skill-creator` workflow、包含 validation，最终产物应是实际可发现 skill 目录；thin skill 和通用 reference 同步说明这一边界。 +- **[Skill Creation Strategy]**: 对齐官方 Record & Replay skill 的生成策略，Swift runtime scaffold、源码 Python scaffold、thin skill 和 reference 都明确生成 skill 前应先检查 connector / API / 专用工具，Computer Use 只用于 UI 依赖、视觉验证或缺少语义工具覆盖的部分。 +- **[Skill Readiness]**: 为 Swift runtime summary 和源码 Python summary 增加 `skillReadiness`，输出 `ready` / `needsReview` / `insufficient`、原因和建议下一步；scaffold 生成的 `SKILL.md` 同步展示 workflow readiness，避免录制信息不足时生成看似完整但需要猜测的 skill。 +- **[Structured Runtime Inputs]**: Swift runtime summary 和源码 Python summary 现在输出结构化 `runtimeInputs`，从文本输入和 selection 事件提取运行时值候选、目标摘要、长度和敏感标记；runtime Swift scaffold 与源码 Python scaffold 优先用该字段渲染 Runtime Inputs，并保留 action sequence fallback。 +- **[Structured Safety Signals]**: Swift runtime summary 和源码 Python summary 现在输出结构化 `safetySignals`，从 `keyboard.submit` 和 target/focused element 的 send/delete/save/upload/publish 等低敏标题关键词提取显式确认线索；runtime Swift scaffold 与源码 Python scaffold 会在 Safety 区块渲染 Confirmation Signals，并把 `hasSafetySignals` 纳入 workflow readiness 的 human review reason。 +- **[Thin Skill Safety Handoff]**: `open-computer-use-record-and-replay` thin skill 和通用 `open-computer-use` Record & Replay reference 现在要求 agent 显式消费 `runtimeInputs` / `safetySignals`；standalone thin skill repo scaffold smoke 也断言导出 repo 保留 `safetySignals`、`skillEvidence.hasSafetySignals=true` 和 replay 前显式确认指引。 +- **[Summary Truncation Guard]**: Swift runtime summary 和源码 Python summary 现在输出结构化 `summaryLimits`，记录 action sequence、runtime inputs、safety signals、target/focused elements、diagnostics 和截图路径等高容量字段的上限、源计数、保留计数和省略计数；scaffold 会渲染 Summary Limits 并在截断时要求回读原始 `events.jsonl`，避免长录制草稿静默漏步骤。 +- **[Blocking Diagnostics Gate]**: Swift runtime summary / validator、源码 Python summary / validator 和 scaffold smoke 现在识别阻断性 `blockingDiagnostics`；当前把 `debug.error` 的 `reason=inputMonitorsUnavailable` 判定为不可用于 skill creation，`skillReadiness.canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求修权限后重录。 +- **[Recording Completion Gate]**: Swift runtime summary / validator、源码 Python summary / validator 和 scaffold smoke 现在识别未完成录制；`state=recording`、`active=true` 或缺少 `session.ended` 都会输出 `recordingIncomplete=true`，`skillReadiness.canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求先 stop 录制。 +- **[Start Event Gate]**: Swift runtime summary / validator、源码 Python summary / validator 和 scaffold smoke 现在识别 start boundary 异常；缺失 `session.started`、重复 `session.started` 或 `session.started` 不是第一条事件都会输出 `sessionStartedCountInvalid` / `sessionStartedNotFirst`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录。 +- **[Final Event Gate]**: Swift runtime summary / validator、源码 Python summary / validator 和 scaffold smoke 现在识别 `session.ended` 后仍继续写事件的 malformed recording；summary 输出 `sessionEndedNotFinal=true` / `sessionEndedIsFinal=false`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录。 +- **[Duplicate Final Event Gate]**: Swift runtime summary / validator、源码 Python summary / validator 和 scaffold smoke 现在识别多个 `session.ended` 的 malformed recording；summary 输出 `sessionEndedCountInvalid=true` / `sessionEndedCount>1`，`skillReadiness.status=insufficient` / `canCreateSkillDraft=false`，`--require-skill-draft` 与 Swift/Python scaffold 都会拒绝生成草稿并要求重录。 +- **[Scaffold Validation Gate]**: Swift runtime scaffold 与源码 Python scaffold 现在会在 summary / 写文件前先跑 recording validator 的 skill draft gate；event count、metadata/session alias、路径或截图引用等结构不一致时返回 `recordingValidationFailed` 并拒绝生成草稿，避免坏 recording 只因可摘要而进入 skill draft。 +- **[Cancelled Recording Guard]**: 取消录制的 session 会被 summary 标为 `skillReadiness.status=insufficient` / `canCreateSkillDraft=false`；runtime Swift scaffold 与源码 Python scaffold 都拒绝从 cancelled recording 生成 skill 草稿，并由单测和 scaffold smoke 覆盖。 +- **[Recording Time Limit Handoff]**: thin skill 与通用 Record & Replay reference 同步官方 handoff：start 成功后结束当前 turn、请用户完成后回来，并告知录制最长 30 分钟；不改变官方兼容 MCP schema。 +- **[Start Approval Time Limit]**: OCU 自有开始确认弹窗会显示当前录制时限，默认 `up to 30 minutes`；新增 helper 单测覆盖文案格式，避免独立 UI 漏掉官方 time-limit handoff。 +- **[Concurrent Recording Handoff]**: thin skill、通用 reference、设计 handoff 与执行计划同步官方 one-active 语义：重复 start 返回 active session 时，agent 应询问用户要使用当前录制还是等待它结束，不能静默当作本次新录制。 +- **[Codex R&R Installer]**: 新增 `open-computer-use install-codex-record-and-replay-mcp`，写入独立 `record_and_replay` MCP server，启动 `open-computer-use event-stream mcp`；普通 9-tool Computer Use 仍由 `install-codex-mcp` 安装，避免两个 surface 混在一起。 +- **[Installer Smoke]**: 新增 `scripts/test-codex-record-and-replay-installer.mjs` 和 `make codex-record-and-replay-installer-smoke`，用临时 `CODEX_HOME` 验证 TOML 写入 `args = ["event-stream","mcp"]` 和重复执行幂等；该 smoke 已接入默认 CI。 +- **[Input Handler Recording]**: 新增 `testEventStreamServiceRecordsInputMonitorMouseAndKeyboardEvents`，通过自定义 input monitor installer 捕获 recorder 注册的 mouse/key handlers，再喂入构造的 `NSEvent`，验证 `mouse.click` 和 `keyboard.text_input` 经真实 handler 路径写入 `events.jsonl`。 +- **[Raw Event Safety]**: 修正 raw event summary 对 AppKit event 属性的读取边界：`phase` / `momentumPhase` 只读 scroll event，`clickCount` 只读 pointer event，keyDown 不再附带无意义 AppKit 坐标，避免不支持属性的 `NSEvent` 抛 exception。 +- **[Standalone Thin Repo Scaffold]**: 新增 `scripts/scaffold-record-and-replay-skill-repo.py`，把仓库内 `open-computer-use-record-and-replay` thin skill 导出成只依赖 `open-computer-use` runtime 的独立 repo 骨架；新增 `scripts/test-record-and-replay-skill-repo-scaffold.py` 和 `make record-and-replay-skill-repo-smoke`，校验生成物可打包 `.skill`、不泄漏本机源码路径，并固定 `event-stream mcp` 三件套 runtime contract。 +- **[Standalone Runtime Verification]**: standalone thin skill repo scaffold 现在会生成 `scripts/verify-runtime.py`；smoke 会构建当前 `OpenComputerUse` binary，并在生成态 repo 内运行 verifier，按 initialize / `notifications/initialized` / `tools/list` 顺序实际检查 `Record & Replay` server name、`2025-11-25` protocol、无 `instructions` initialize response，以及 `event_stream_start/status/stop` 三个无参数 tools。 +- **[Standalone Repo Self Check]**: standalone thin skill repo scaffold 现在会生成 `scripts/check.sh`，串起 `scripts/package-skill.sh` 和 `scripts/verify-runtime.py`；smoke 会在生成态 repo 内运行该组合门禁，固定发布或安装前的最小自检入口。 +- **[Standalone Repo CI]**: standalone thin skill repo scaffold 现在会生成 `.github/workflows/ci.yml`，使用 SHA-pinned `actions/checkout` 和 `actions/setup-node`，在 macOS runner 中安装发布版 `open-computer-use` 后运行 `scripts/check.sh`；smoke 会检查 workflow 调用同一自检入口且所有 `uses:` 都 pin 到 40 位 SHA。 +- **[Standalone Package Frontmatter Gate]**: standalone thin skill repo 生成态 `scripts/package-skill.sh` 现在会校验 `skills/*/SKILL.md` 的 frontmatter，要求 `name` 匹配目录名且 `description` 非空；smoke 会临时清空 description 并确认 package script 失败，再恢复后继续打包。 +- **[Standalone Surface Contract]**: standalone thin skill repo 生成态 `scripts/verify-runtime.py` 现在不只检查三件套 tool 名称，还校验官方兼容的 initialize capabilities、tool description、empty input schema 和 MCP annotations；`record-and-replay-skill-repo.json` 同步写入 `capabilities` / `toolMetadata`，smoke 覆盖 manifest 和 verifier 内容。 +- **[Standalone Lifecycle Smoke]**: standalone thin skill repo scaffold 现在会生成可选 `scripts/recording-lifecycle-smoke.py`；它通过同一 `event-stream mcp` 进程执行 start/repeat start/status/stop/repeat stop/final status，检查 one-active、idempotent stop、completed status、`metadata.json` / `session.json` / `events.jsonl`，并用安装态 runtime `event-stream validate --strict-ocu` 校验 session 文件。`make record-and-replay-skill-repo-smoke` 会用当前构建出的 `OpenComputerUse` binary 实际运行该脚本，但生成 repo 默认 `scripts/check.sh` 和 CI workflow 仍保持 non-recording contract check。 +- **[Standalone Recording Handoff README]**: standalone thin skill repo 生成态 README 现在固定 recording-to-skill handoff 指引：完整 OCU session 先跑 `validate --strict-ocu --require-skill-draft `，只有 `eventsPath` 时跑非 strict events-only gate；取消或未完成 recording 不能用于创建或更新 skill，且 generated scaffold 也会先跑 skill-draft validation gate。smoke 已断言这些指引随独立 repo 导出。 +- **[Standalone Recording-To-Skill Smoke]**: standalone thin skill repo scaffold 现在会生成 `scripts/recording-to-skill-smoke.py` 并接入默认 `scripts/check.sh`；该 smoke 使用临时合成 completed recording 验证安装态 runtime 的 strict skill-draft validation、events-only validation 和 `event-stream scaffold-skill --json` 草稿生成路径，不需要启动真实桌面录制。`make record-and-replay-skill-repo-smoke` 会单独运行该脚本，也会通过生成态 `check.sh` 再跑一次。 +- **[Standalone Declared Path Self-Check]**: 生成态 `scripts/recording-to-skill-smoke.py` 现在会检查 strict validator 输出的 `declaredPaths`，要求 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 四个 handoff path 都解析存在；README、thin skill 和通用 reference 同步说明 strict OCU validation 能证明 declared handoff paths，而 events-only gate 不能证明。 +- **[Standalone Recording-To-Skill Contract]**: standalone thin skill repo manifest 新增 `recordingToSkill` 区块，机器可读声明 strict validation、events-only validation 和 scaffold-skill 的 handoff 边界，包括 strict 必须证明 declared handoff paths、events-only 不能证明 alias/path，以及 scaffold 会先跑 skill draft validation gate。 +- **[Standalone Skill Workflow Verification]**: standalone thin skill repo scaffold 新增生成态 `scripts/verify-skill-workflow.py`，并接入默认 `scripts/check.sh` 和 manifest `checks.skillWorkflow`；该 verifier 不启动录制，只检查 thin skill 保留官方 handoff 语义：只用 `event_stream_start/status/stop` 三件套、start 后结束 turn、不轮询、用户完成后 stop、取消后不再 stop，以及 validate/summarize/scaffold 前置门禁。smoke 会验证正常通过和移除关键文案后的失败路径。 +- **[Standalone Wait Notify Contract]**: standalone thin skill repo scaffold 的生成态 manifest 新增 `extensionLayer`，显式声明 `wait --notify-command`、`waitTimedOut` / `waitSessionMatched`、notify stdin/env 和 listener 不属于官方兼容 MCP surface；生成态 README 与 `scripts/verify-skill-workflow.py` 同步守住 Independent Wait / Notify Integration，smoke 覆盖 manifest、正向校验和删除关键 env 文案后的失败路径。 +- **[Standalone Wait Notify Smoke]**: standalone thin skill repo scaffold 新增生成态 `scripts/wait-notify-contract-smoke.py`，并接入默认 `scripts/check.sh` 与 manifest `checks.waitNotifyContractSmoke`；该 smoke 不启动真实录制，只等待不存在的 session，断言安装态 CLI 返回 `waitTimedOut=true` / `waitSessionMatched=false`、notify callback 被跳过，且不会创建 `latest-session.json` / `active-session.json`。 +- **[NPM Standalone Repo Scaffold]**: npm staging 包现在携带 `scripts/scaffold-record-and-replay-skill-repo.py` 和 `skills/open-computer-use-record-and-replay/`，launcher 新增 `open-computer-use scaffold-record-and-replay-skill-repo --output-dir `；安装态无需源码 checkout 即可生成 standalone Record & Replay thin skill repo 骨架，并继续通过生成态 `scripts/verify-skill-workflow.py` 自检官方 handoff 语义。 +- **[NPM Standalone Repo Smoke]**: 新增 `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 和 `make npm-record-and-replay-skill-repo-smoke`；该 opt-in smoke 用本地 `dist/` artifacts stage `open-computer-use` npm 包，再通过 staged launcher 执行安装态 scaffold 命令，验证生成 repo 包含 workflow verifier 且 package script 可产出 `.skill`。 +- **[NPM Scaffold UX]**: 安装态 launcher 为 `scaffold-record-and-replay-skill-repo` 补专用 help，明确它生成 standalone repo 而不是更新 MCP/plugin config；同时增加 Python 3 解析和缺失诊断，优先使用 `PYTHON`，再按平台尝试系统 Python，并在 npm staging smoke 中覆盖 help 文案和 missing Python failure。 +- **[NPM Python Gate]**: 安装态 launcher 现在会解析 `python --version` 并只接受 Python 3，避免旧环境中 `python` 指向 Python 2 时进入 scaffold 脚本后才失败；npm staging smoke 增加 fake Python 2 executable，覆盖非 Python 3 候选会被拒绝。 +- **[Action Smoke Skill Handoff]**: 真实输入 action smoke 现在会检查 runtime scaffold 生成的 skill 草稿保留 `skill-creator` finalization 指引，并明确不能停在 standalone runbook / replay plan，防止“能录制、能生成文件”但不满足官方 recording-to-skill handoff。 +- **[Standalone Skill Creator Handoff]**: standalone thin skill repo scaffold 生成的 `scripts/recording-to-skill-smoke.py` 也会检查 runtime scaffold 草稿保留 `skill-creator` finalization 指引和非 runbook/replay-plan 文案，并在 smoke JSON 中输出 `checkedSkillCreatorHandoff=true`，让独立 repo 默认 `scripts/check.sh` 守住同一 handoff 语义。 +- **[NPM Standalone Repo Self-Check]**: npm staging smoke 现在会让 staged launcher 生成 standalone repo 后运行生成态 `scripts/check.sh`，并把 `OPEN_COMPUTER_USE_CLI` 指向同一个 staged launcher；这会在安装态路径下同时验证 package、runtime contract、skill workflow、合成 recording-to-skill scaffold 和 `skill-creator` handoff。 +- **[NPM Recording-To-Skill Contract]**: npm staging smoke 现在会先用 staged launcher 校验合成 recording，确认 strict validator 已输出 `declaredPaths`，再读取安装态生成 repo 的 `record-and-replay-skill-repo.json`，断言 `recordingToSkill` contract 保留 strict validation、events-only validation 和 scaffold-skill 的机器可读边界，避免 npm scaffold 路径丢失 standalone recording-to-skill 合同。 +- **[NPM Smoke Verification]**: 本地刷新 `dist/Open Computer Use.app` 后，`make npm-record-and-replay-skill-repo-smoke` 已通过；该结果证明当前 staged npm launcher 可以携带 strict `declaredPaths` validator evidence 走完整 standalone repo 默认自检。 +- **[Baseline Verification]**: 当前 worktree 复跑 `make event-stream-smoke-matrix`、`make event-stream-action-smoke` 和 `scripts/compare-event-stream-surface.py --use-default-official` 均通过，覆盖无真实输入的 lifecycle 矩阵、真实 CGEvent 输入录制到 skill 草稿链路，以及 OCU / 官方 1.0.857 non-recording surface drift。 +- **[Baseline Smoke Target]**: 新增 opt-in `make record-and-replay-baseline-smoke`，把默认 matrix、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、standalone thin skill repo smoke 和 npm standalone repo smoke 串成一个本机候选验证入口；该入口不进默认 CI，因为它依赖桌面输入权限、官方 bundled cache 和当前 `dist/` artifacts。本机已复跑通过。 +- **[Default CI Verification]**: 复跑 `make ci` 通过，覆盖文档/仓库卫生、脚本语法、skill 打包、Record & Replay 默认 smoke、Swift 单测、默认 matrix 和 Windows / Linux Go test；真实输入 action 与 npm staging baseline 继续由 opt-in baseline smoke 覆盖。 +- **[Official No-Active Response]**: 通过 Codex-hosted 官方 Record & Replay `event_stream_status` / `event_stream_stop` 观察到 no-active text JSON 只有 `isRecording=false` 和 `maxDurationSeconds=1800`；OCU MCP no-active status/stop 已按该 shape 校准，CLI 扩展仍保留本地诊断字段。 +- **[Standalone No-Active Runtime Contract]**: standalone thin skill repo 生成态 `scripts/verify-runtime.py` 现在也会在临时 recording root 下调用 no-active `event_stream_status` / `event_stream_stop`，确认返回官方 `isRecording=false` / `maxDurationSeconds=1800` shape 且不会创建 session 文件；生成态 manifest 同步写入 `mcpServer.noActiveResponse`。 +- **[Official Surface Recheck]**: 复跑 `scripts/compare-event-stream-surface.py --use-default-official`，确认 OCU 本地 `event-stream mcp` 与官方 `record-and-replay/1.0.857` 默认 cache 的 initialize / `tools/list` 仍匹配已入库 fixture；该检查不启动录制，只验证 non-recording surface。 +- **[CGEvent Tap Input]**: EventStream 生产录制路径从单纯 `NSEvent.addGlobalMonitorForEvents` 升级为优先安装 listen-only `CGEvent` tap，失败时再回退到 NSEvent monitor；这更接近 Record & Replay 对 session 输入流的需求，也能捕获外部进程发出的 HID click 和 keyboard event。 +- **[Action Smoke]**: 新增 `make event-stream-action-smoke` / `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1`，启动 app-agent recorder 后由外部 Swift 进程发出低影响 CGEvent click 和 Shift-F20 keyboard event，停止后用 validator / summary 要求 `mouse.click`、`keyboard.shortcut`、`AX.focusedWindowChanged`、`session.ended` 入流；该 smoke 现在还会用同一份真实录制产物调用 runtime `event-stream scaffold-skill --json` 生成 `SKILL.md` 草稿，并断言草稿 / `recording-summary.json` 不含本机绝对路径、summary 路径字段使用 `` 占位且 `skillReadiness.canCreateSkillDraft=true`，覆盖真实输入录制到路径脱敏 skill 草稿的端到端链路。该 smoke 为 opt-in，不进入默认 CI。 + +### 🔁 Follow-up (2026-06-27, fixture-backed no-active drift check) + +- **[No-Active Fixture Check]**: 新增 `scripts/compare-event-stream-no-active.py` 和 `make event-stream-no-active-smoke`，用 Codex-hosted 官方 1.0.857 no-active fixture 校验本地 OCU MCP `event_stream_status` / `event_stream_stop` text JSON，固定 `isRecording=false` / `maxDurationSeconds=1800` shape。 +- **[No Session Side Effect]**: 新检查使用临时 recording root，断言 no-active status/stop 不会创建 `latest-session.json`、`active-session.json` 或 session 目录，避免官方 idle response 校准引入隐式状态写入。 +- **[CI Sync]**: 将 no-active drift check 接入 `scripts/ci.sh`，并同步 Makefile、architecture、CI/CD、execution plan、reverse-engineering reference、handoff 和 quality score。 + +### 🔁 Follow-up (2026-06-27, strict OCU current segment gate) + +- **[Current Segment Validation]**: runtime Swift validator 与源码 Python validator 新增 strict OCU current segment path gate；`state=recording` 必须带存在的 `currentSegmentEventsPath` / `currentSegmentMetadataPath`，`state=stopped|cancelled` 不能残留这两个字段。 +- **[Runtime/Source Parity]**: `scripts/test-event-stream-skill-scaffold.py` 同时覆盖源码脚本和安装态 runtime CLI 的缺失 / 残留拒绝路径，避免独立 skill scaffold 只在源码 checkout 下有 gate。 +- **[Scope Note]**: 该 gate 固定 OCU 自身 active/final session 文件一致性，不把 current segment 字段视为已完成的官方 successful recording schema；最终字段语义仍待 official golden fixtures 校准。 + +### 🔁 Follow-up (2026-06-27, local probe sanitization gate) + +- **[Probe Fixture Redaction]**: `scripts/probe-event-stream-recording.py --fixture-output` 现在会把 MCP text content 中可解析的 JSON 转成脱敏 `textJSON`，避免 start/status/stop response text 里保留本机绝对路径。 +- **[Local Probe Smoke]**: 新增 `scripts/test-event-stream-local-probe.py` 和 `make event-stream-local-probe-smoke`，用真实本地 OCU `event-stream mcp` 采集 start/status/stop transcript，固定 active response 带 current segment 字段、final response 不残留 current segment 字段、fixture 不含本机路径，并用源码 validator 与 runtime validator 校验最终 session 文件。 +- **[CI Sync]**: 将 local probe smoke 接入默认 `scripts/ci.sh`，并同步 Makefile、CI/CD、architecture、execution plan、reverse-engineering reference 和 quality score。 + +### 🔁 Follow-up (2026-06-27, repeat-start probe evidence) + +- **[Probe One-Active Evidence]**: `scripts/probe-event-stream-recording.py --start-stop` 的采样顺序扩展为 start / repeat start / status / stop；首次 start 成功后立即再次调用 `event_stream_start`，用于保存 one-active 语义证据。 +- **[Local Probe Assertion]**: `scripts/test-event-stream-local-probe.py` 现在断言两次 start 返回同一个 active session，并继续校验 current segment 脱敏和 final cleanup。 +- **[Fixture Shape]**: probe fixture 新增 `repeatStartResponseShape`，由 fake MCP server test 覆盖脱敏后的 repeat start response；首个 start 超时时仍保持官方 raw start-timeout fixture 的旧边界，不强行采集 repeat start。 + +### 🔁 Follow-up (2026-06-27, completed lifecycle probe evidence) + +- **[Probe Completed Evidence]**: `scripts/probe-event-stream-recording.py --start-stop` 现在在首次 stop 成功后继续调用 repeat stop 和 final status，采样顺序变为 start / repeat start / status / stop / repeat stop / final status。 +- **[Local Probe Assertion]**: `scripts/test-event-stream-local-probe.py` 断言 repeat stop 和 final status 仍返回同一个 completed session，覆盖 idempotent stop 与 completed status 证据。 +- **[Fixture Shape]**: probe fixture 新增 `repeatStopResponseShape` 和 `finalStatusResponseShape`；首个 start 超时时仍保持官方 raw start-timeout fixture 的旧边界，不强行采集 completed lifecycle。 + +### 🔁 Follow-up (2026-06-27, golden readiness lifecycle shapes) + +- **[Readiness Shape Gate]**: `scripts/check-event-stream-golden-readiness.py` 新增 `--require-mcp-response-shape`，可显式要求 `repeatStartResponseShape`、`repeatStopResponseShape` 和 `finalStatusResponseShape` 这类 raw probe completed lifecycle 证据。 +- **[Timeout Semantics]**: required response shape 缺失、timeout 或没有 result/error 会独立失败；timeout 错误使用 `MCP response shape timed out: `，避免宿主外边界样本被误判成 successful response golden。 +- **[Tool/Shape Split]**: tool 级 `--require-mcp-tool-response event_stream_*` 现在按“该 tool 至少有一次 result/error”聚合；同一 tool 的 repeat shape timeout 不会污染已成功的 tool response，具体 repeat lifecycle 仍由 `--require-mcp-response-shape` 单独要求。 +- **[Fixture Import Coverage]**: fixture import smoke 增加 repeat start / repeat stop / final status shape 脱敏断言，确认 MCP text JSON 仍会被解码成相对路径和脱敏 session id。 + +### 🔁 Follow-up (2026-06-27, compare MCP response shapes) + +- **[Compare Shape Gate]**: `scripts/compare-event-stream-recordings.py` 新增可选 `--require-mcp-response-shapes`，当 baseline 与 candidate 都带 `mcp-transcript.json` 时，比较 start/repeat/status/stop/final response shape 是否存在并保持 result/error/timeout 状态。 +- **[Schema Gate]**: 新增 `--require-same-mcp-response-schema`，在 shape 存在和状态一致之外要求 candidate response shape 保留 baseline 的 schema path/type，避免 response text JSON 字段在 OCU candidate 中静默丢失。 +- **[Smoke Coverage]**: `scripts/test-event-stream-recording-compare.py` 覆盖 matching transcript 通过、缺 repeat start shape、repeat start timeout 状态漂移和 repeat start response schema 缺失失败路径。 + +### 🔁 Follow-up (2026-06-27, fixture import calibration chain) + +- **[Import-To-Gate Smoke]**: `scripts/test-event-stream-fixture-import.py` 现在会把脱敏导入后的 fixture 继续交给 golden readiness response-shape gate 和 recording compare MCP response shape/schema gate，证明 `mcp-transcript.json` 的 sanitized response shape 可以被后续官方/OCU 校准链路直接消费。 +- **[Readiness Chain]**: 同一 smoke 要求导入产物通过 `repeatStartResponseShape`、`repeatStopResponseShape` 和 `finalStatusResponseShape` 的 readiness shape gate，避免 fixture import 只验证文件脱敏、不验证 lifecycle response evidence 可用性。 +- **[Compare Chain]**: 同一 smoke 用导入 fixture 自比并打开 `--require-mcp-response-shapes --require-same-mcp-response-schema`，固定 response shape evidence 与 schema gate 的输入合同。 + +### 🔁 Follow-up (2026-06-27, suppressed stream compare) + +- **[Suppressed Compare Gate]**: `scripts/compare-event-stream-recordings.py` 现在读取 `suppressed.jsonl`，输出 suppressed event sequence、count diff 和 schema diff，并新增 `--require-same-suppressed-event-sequence` / `--require-same-suppressed-schema` 两个 gate。 +- **[Fallback Evidence Coverage]**: `scripts/test-event-stream-recording-compare.py` 增加 candidate 缺 suppressed event 和 suppressed schema 缺字段失败路径，避免后续官方样本中的 AX / screenshot / 压缩降级证据只通过 `suppressedEventCount` 间接覆盖。 + +### 🔁 Follow-up (2026-06-27, readiness suppressed evidence) + +- **[Readiness Suppressed Gate]**: `scripts/check-event-stream-golden-readiness.py` 新增 `--require-suppressed-events` 和 `--require-suppressed-event-type`，动作类或 fallback 校准样本可以显式要求 `suppressed.jsonl` 中存在指定降级证据。 +- **[Fixture Path Redaction]**: `scripts/import-event-stream-fixture.py` 现在会脱敏事件和 suppressed event 中的通用 `path` / 未知 `*Path` 字段，只保留 redacted marker 和长度，避免非标准字段泄漏本机路径。 +- **[Smoke Coverage]**: `scripts/test-event-stream-fixture-import.py` 和 `scripts/test-event-stream-golden-readiness.py` 覆盖 suppressed event 脱敏、suppressed evidence readiness gate，以及导入 fixture 后继续通过 readiness / compare 校准链路。 + +### 🔁 Follow-up (2026-06-27, readiness endReason gate) + +- **[EndReason Readiness Gate]**: `scripts/check-event-stream-golden-readiness.py` 新增 `--require-end-reason `，从 metadata/session alias 和 `session.ended.endReason` 汇总 `endReasons`，用于官方 stop / cancel / timeout golden fixture 入库时显式固定 expected completion reason。 +- **[Smoke Coverage]**: `scripts/test-event-stream-golden-readiness.py` 覆盖 required `recording_controls_stopped` 通过和 required `recording_controls_cancelled` mismatch 失败路径，避免 control completion reason 只靠人工审查。 + +### 🔁 Follow-up (2026-06-27, readiness final session evidence) + +- **[Session End Readiness Gate]**: `scripts/check-event-stream-golden-readiness.py` 现在默认拒绝多个 `session.ended` 或 `session.ended` 后仍继续写入事件的 malformed fixture，并输出 `lastEventType`、`sessionEndedCount` 和 `sessionEndedIsFinal` 供 JSON 报告审查。 +- **[Smoke Coverage]**: `scripts/test-event-stream-golden-readiness.py` 覆盖 duplicate `session.ended` 和 non-final `session.ended` 失败路径，避免坏 lifecycle recording 被误判为 official/OCU golden baseline。 + +### 🔁 Follow-up (2026-06-27, readiness session alias evidence) + +- **[Session Alias Readiness Gate]**: `scripts/check-event-stream-golden-readiness.py` 现在读取并报告 `metadata.json` / `session.json` alias 状态，输出 `metadataSessionAliasComplete` 和 `metadataSessionAliasMatches`，默认拒绝两者同时存在但内容不同的 fixture。 +- **[Required Alias Coverage]**: readiness 新增 `--require-session-alias`，用于官方 handoff golden fixture 入库时强制要求 `metadata.json` 和 `session.json` 两份 alias 都存在且一致。 +- **[Smoke Coverage]**: `scripts/test-event-stream-golden-readiness.py` 覆盖 ready fixture alias 通过、缺失 `session.json` alias 失败和 alias 内容漂移失败路径。 + +### 🔁 Follow-up (2026-06-27, readiness metadata counts) + +- **[Metadata Count Readiness Gate]**: `scripts/check-event-stream-golden-readiness.py` 现在报告 `metadataEventCount` / `metadataSuppressedEventCount` 和对应 match 状态，并默认拒绝已声明但与 `events.jsonl` / `suppressed.jsonl` 行数不一致的 metadata count。 +- **[Required Count Coverage]**: readiness 新增 `--require-metadata-counts`，用于官方 handoff golden fixture 入库时强制要求 `eventCount` 和 `suppressedEventCount` 都存在且与文件行数一致。 +- **[Manifest Suppressed Count]**: readiness 现在也校验 fixture manifest 的 `suppressedEventCount`，避免 fallback evidence 计数只在 metadata 中被检查。 +- **[Smoke Coverage]**: `scripts/test-event-stream-golden-readiness.py` 覆盖 metadata event count 漂移、缺失 metadata counts 和 manifest suppressed count 漂移失败路径。 + +### 🔁 Follow-up (2026-06-27, readiness handoff paths) + +- **[Declared Path Evidence]**: `scripts/check-event-stream-golden-readiness.py` 现在输出 `declaredPaths`，记录 metadata 中 `metadataPath`、`eventsPath`、`sessionPath` 和 `suppressedEventsPath` 的原始值、解析路径和存在性。 +- **[Default Drift Gate]**: readiness 默认拒绝已经声明但不存在的 handoff path，避免官方 fixture 只证明文件内容、不证明 skill handoff 返回路径可消费。 +- **[Required Path Coverage]**: readiness 新增 `--require-handoff-paths`，用于官方 handoff golden fixture 入库时强制要求四个路径字段都存在且指向可读产物。 +- **[Smoke Coverage]**: `scripts/test-event-stream-golden-readiness.py` 覆盖 ready fixture path 通过、缺失 handoff path 字段、`sessionPath` 指向不存在文件和 `eventsPath` 指向不存在 JSONL 的失败路径。 + +### 🔁 Follow-up (2026-06-27, compare handoff paths) + +- **[Compare Path Evidence]**: `scripts/compare-event-stream-recordings.py` 现在输出 `handoffPathEvidence`，分别记录 baseline 和 candidate 的 `metadataPath`、`eventsPath`、`sessionPath`、`suppressedEventsPath` 解析状态。 +- **[Candidate Path Gate]**: recording compare 新增 `--require-handoff-paths`，要求 baseline 和 candidate 都声明四个 handoff path 且解析后文件存在；该 gate 不比较路径字符串本身，避免不同 fixture/candidate 目录导致误报。 +- **[Smoke Coverage]**: `scripts/test-event-stream-recording-compare.py` 覆盖相同样本 path 通过、candidate 缺 `sessionPath` 失败和 candidate `suppressedEventsPath` 指向不存在文件失败路径。 + +### 🔁 Follow-up (2026-06-27, scroll raw action smoke) + +- **[Scroll Raw Coverage]**: `make event-stream-action-smoke` 现在除了外部 HID click 和 Shift-F20 keyboard event，也会发出真实 CG scroll wheel event。 +- **[No Synthetic Scroll Type]**: 由于当前官方字符串样本还没有确认独立 scroll event 名称，smoke 只要求 `type=experimentalRawEvents` / `reason=scrollWheel` / raw `eventType=scrollWheel` 入流，不臆造 `mouse.scroll`。 +- **[Recording-To-Skill Continuity]**: Swift/Python summary 与 scaffold 现在会把 `reason=scrollWheel` 的 raw-only event 纳入 `actionSequence` 并渲染为 Scroll replay step；同一 smoke 继续要求 validator、summary 和 runtime scaffold 通过，确保补 scroll raw evidence 不破坏录制到 skill 草稿的路径脱敏 handoff。 + +### 🔁 Follow-up (2026-06-27, baseline smoke script) + +- **[Reusable Gate]**: 新增 `scripts/run-record-and-replay-baseline-smoke.sh`，把 baseline 候选验证顺序从 Makefile 命令串收敛成可复用脚本。 +- **[Make Entrypoint]**: `make record-and-replay-baseline-smoke` 现在只转调该脚本，后续独立 repo、release gate 或人工验收可以直接复用同一入口。 +- **[Machine Summary]**: 脚本成功时输出 `{"ok":true,"baseline":"record-and-replay",...}` JSON 摘要，便于上层脚本判断组合 baseline 已完成。 +- **[Fixture Set Coverage]**: baseline 脚本现在也运行 `scripts/test-event-stream-official-fixture-set.py`，确保 official successful recording 入库后的集合级 gate 在 opt-in baseline 候选验证中被覆盖。 +- **[Verification]**: 已复跑 `make record-and-replay-baseline-smoke`，覆盖默认 matrix、真实输入 action smoke、官方 1.0.857 surface compare、official fixture set gate smoke、standalone repo smoke 和 npm staged repo smoke。 + +### 🔁 Follow-up (2026-06-27, official fixture set gate) + +- **[Scenario Manifest]**: `scripts/import-event-stream-fixture.py` 新增 `--scenario `，把官方/OCU fixture 的机器可读场景写入 manifest，避免后续靠目录名或人工上下文配对。 +- **[Set-Level Gate]**: 新增 `scripts/check-event-stream-official-fixture-set.py`，默认要求 official fixture set 至少包含 `simple-action-stop` successful recording，并对 official 样本运行 strict readiness。 +- **[Candidate Pairing]**: fixture set gate 支持 `--candidate-root`，按同 scenario 找 OCU candidate，并用 recording compare 固定 event sequence、schema、metadata keys/values、handoff paths、final session evidence、semantic fields 和 MCP response shape。 +- **[Scenario Policies]**: `simple-action-stop` 现在要求 `mouse.click` 与 `recording_controls_stopped` endReason，`keyboard-input-stop` 要求 `keyboard.text_input` 与 `recording_controls_stopped` endReason，`drag-stop` 要求 `mouse.drag` 与 `recording_controls_stopped` endReason，`cancel` 要求 `recording_controls_cancelled` endReason；`timeout` 因官方 endReason 未观测，暂不固定具体值。 +- **[Smoke Coverage]**: 新增 `scripts/test-event-stream-official-fixture-set.py` / `make event-stream-official-fixture-set-smoke`，覆盖 gate 成功、缺 candidate、缺 scenario 和导入 `--scenario` 写 manifest。 +- **[Verification]**: 已复跑 `make event-stream-official-fixture-set-smoke`、`make event-stream-fixture-smoke`、`make event-stream-golden-readiness-smoke` 和 `make event-stream-compare-smoke`。 + +### Follow-up (2026-06-27, timeout official scenario policy smoke) + +- **[Timeout Policy Evidence]**: `scripts/test-event-stream-official-fixture-set.py` 新增 `timeout` 场景 lifecycle-only fixture，验证该 scenario 只要求 `session.started` / `session.ended` 与 MCP transcript，不要求 action event 或 AX payload，也不提前固定尚未观测的官方 timeout endReason。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在强制读取并输出 `checkedTimeoutScenarioPolicy`，避免 opt-in baseline 摘要只证明 stop / keyboard / drag / cancel policy。 +- **[Docs]**: 同步更新 handoff、replication design、reverse-engineering reference、architecture 和 quality score，明确 timeout gate 仍是待官方 golden 校准的生命周期策略。 +- **[Verification]**: 已复跑 `make record-and-replay-baseline-smoke` 通过，最终摘要包含 default matrix、真实输入 action smoke、官方 1.0.857 non-recording surface compare、official fixture set gate、standalone repo smoke 和 npm staged repo smoke；同一摘要中 `checkedTimeoutScenarioPolicy=true`，但 `repoHasRequiredOfficialSuccessfulFixture=false`，继续明确真实 official successful recording 尚未入库。 + +### 🔁 Follow-up (2026-06-27, official start probe refresh) + +- **[Official Probe]**: 复跑 `make event-stream-official-start-probe`，官方 bundled 1.0.857 raw MCP client 仍只正常返回 initialize / tools-list。 +- **[Start Timeout]**: 裸 `event_stream_start`、后续 `event_stream_status` 和 `event_stream_stop` 均 10 秒超时，没有返回 metadata/events/session paths，也没有可导入 official successful recording。 +- **[Fixture Refresh]**: 刷新 `record-and-replay-official-raw-start-timeout-1.0.857.json`，sanitized fixture 现在显式包含 `startResponseShape`、`statusResponseShape` 和 `stopResponseShape` 三个 10 秒 timeout；`scripts/test-event-stream-probe-fixtures.py` 同步要求 transcript 中恰好三次 `tools/call` timeout，避免官方宿主外边界证据退回旧的 start/stop-only 形态。 +- **[Current Baseline Verification]**: 刷新 fixture 后复跑默认 `make ci` 和 opt-in `make record-and-replay-baseline-smoke` 均通过；后者覆盖默认 event-stream matrix、真实输入 action smoke、官方 1.0.857 surface compare、official fixture set gate、standalone repo smoke 和 npm staged repo smoke。 +- **[User-Facing Quickstart]**: README 和 README.zh-CN 新增 Record & Replay 快速开始，覆盖 dedicated R&R MCP 安装、官方三件套 surface、OCU CLI wait/notify、validate/summarize/scaffold-skill 以及 standalone thin skill repo scaffold 自检入口。 +- **[Calibration Boundary]**: official successful recording fixture 仍需要通过正常 Codex 宿主流程，或先补齐官方 runtime 所需宿主状态后再采集。 + +### 🔁 Follow-up (2026-06-27, recording controls callback coverage) + +- **[Controls Callback Coverage]**: `EventStreamRecordingControls` 新增测试探针，保持 panel 私有，但允许单测按按钮标题触发内部 NSButton action。 +- **[Done / Discard Evidence]**: 新增 `testEventStreamRecordingControlsButtonsTriggerRuntimeCallbacks`，验证 Done 只触发 stop callback，Discard 只触发 cancel callback，补齐 OCU 自有控制条点击直接进入 runtime 的按钮级证据。 +- **[Verification]**: 已复跑 `swift test --filter testEventStreamRecordingControlsButtonsTriggerRuntimeCallbacks` 通过。 + +### 🔁 Follow-up (2026-06-27, wait notify failure coverage) + +- **[Notify Failure Coverage]**: 新增 wait notify 回调失败回归，验证 callback 非零退出时 `notification.ok=false`、`reason=nonZeroExit`、保留 exit code，并让 `runOpenComputerUseEventStream` 返回 `hasToolError=true`。 +- **[Notify Timeout Coverage]**: 新增 wait notify 回调超时回归，验证 `OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS` 生效，超时 callback 会被终止并返回 `reason=timeout`、`timedOut=true`、`hasToolError=true`。 +- **[Verification]**: 已复跑 `swift test --filter testRunEventStreamCLIWaitReportsNotifyCommand` 通过。 + +### 🔁 Follow-up (2026-06-27, event-stream CLI error exit) + +- **[CLI Exit Semantics]**: 修正 `open-computer-use event-stream ...` 入口，打印 JSON 后如果 `OpenComputerUseCallOutput.hasToolError=true` 会和普通 `call` 命令一样以非零退出码结束。 +- **[Standalone Contract]**: 生成态 `scripts/wait-notify-contract-smoke.py` 新增合成 completed session，用 `--notify-command` 的非零退出验证 JSON `notification.reason=nonZeroExit`、`exitCode=7`，并断言 CLI 退出码非零。该检查不启动真实桌面录制。 +- **[Verification]**: 已复跑 `swift build --product OpenComputerUse`、`swift test --filter testRunEventStreamCLIWaitReportsNotifyCommand` 和 `make record-and-replay-skill-repo-smoke` 通过。 + +### 🔁 Follow-up (2026-06-27, recording-to-skill failure exit) + +- **[Standalone Recording-To-Skill Failure Contract]**: 生成态 `scripts/recording-to-skill-smoke.py` 新增无动作 recording 负例，验证安装态 `event-stream scaffold-skill --json` 会输出 `ok=false` 且 CLI 退出码非零，避免独立 repo 的 shell/CI 只看 JSON 文本而误判失败 scaffold 为成功。 +- **[Scaffold Smoke Assertion]**: `scripts/test-record-and-replay-skill-repo-scaffold.py` 现在断言生成态 smoke JSON 暴露 `checkedScaffoldSkillFailureExit=true`，确保该契约随 standalone repo scaffold 一起导出。 + +### 🔁 Follow-up (2026-06-27, recording controls frame clamp) + +- **[Controls Placement]**: OCU 自有 Record & Replay 控制条的初始定位改为使用可测试的 visible-frame clamp，常规屏幕居中贴近顶部，多显示器负坐标、窄屏或高度不足时会尽量保持在可见区域内。 +- **[Controls Coverage]**: 新增控制条 frame helper 单测，覆盖常规可见区域、负坐标窄屏和高度不足屏幕，避免独立交互录制入口在非主流屏幕布局下把 Done / Discard 控制放到不可见位置。 + +### 🔁 Follow-up (2026-06-27, event-stream no-arg contract) + +- **[MCP Runtime Contract]**: `event_stream_start/status/stop` 的运行时 dispatcher 现在会拒绝非 object `arguments` 和非空 object `arguments`，与 tools/list 中 `additionalProperties=false` 的空 object schema 保持一致；错误通过既有 tool error 通道返回，不改变官方三件套 surface。 +- **[No Side Effect Coverage]**: 新增 MCP 单测，验证 `event_stream_status` 携带意外字段、`event_stream_start` 携带非 object 参数时都返回 `isError=true`，并且不会创建 `latest-session.json` 或 `active-session.json`。 +- **[Standalone Contract]**: standalone thin skill repo 的生成态 manifest 新增 `mcpServer.requiresObjectArguments=true` 和 `mcpServer.rejectsUnexpectedArguments=true`，`scripts/verify-runtime.py` 也会实际调用带意外参数的 `event_stream_status` 和带数组参数的 `event_stream_start` 并断言 tool error，确保独立 repo 默认自检守住 no-arg runtime contract。 +- **[Tool Name Contract]**: `tools/call.params.name` 现在必须是非空 string；缺失或非 string tool name 会在进入 dispatcher / start approval 前返回 tool error，且不会创建 session 索引文件。standalone thin skill repo 的 manifest 新增 `mcpServer.requiresStringToolName=true`，生成态 `verify-runtime.py` 和 scaffold smoke 同步验证该 malformed request 无副作用。 +- **[Tool Params Contract]**: `tools/call.params` 现在必须是 object；缺失或非 object params 会在解析 tool name 前返回 tool error，且不会触发 start approval 或创建 session 索引文件。standalone thin skill repo 的 manifest 新增 `mcpServer.requiresObjectParams=true`，生成态 `verify-runtime.py` 和 scaffold smoke 同步验证该 malformed request 无副作用。 + +### 🔁 Follow-up (2026-06-27, standalone cancelled recording gate) + +- **[Recording-to-Skill Guard]**: standalone thin skill repo 的生成态 `scripts/recording-to-skill-smoke.py` 新增 cancelled recording 负例，验证 `recording_controls_cancelled` 录制在 strict validation、events-only validation 和 `event-stream scaffold-skill --json` 三条路径都会被拒绝。 +- **[Manifest Contract]**: `record-and-replay-skill-repo.json` 的 `recordingToSkill` 区块新增 `rejectsCancelledRecordings=true`，让独立 repo 默认 contract 机器可读地声明被用户 Discard / cancel 的录制不能用于创建或更新 skill。 +- **[NPM Scaffold Contract]**: npm staging smoke 现在读取安装态 scaffold 生成的 `record-and-replay-skill-repo.json` 时也断言 `recordingToSkill.rejectsCancelledRecordings=true`，避免源码 scaffold 与 npm launcher scaffold 的 recording-to-skill 边界漂移。 +- **[NPM Verification]**: 刷新 release app artifact 后复跑 `make npm-record-and-replay-skill-repo-smoke` 通过，输出新增 `checkedCancelledRecordingContract=true`，确认安装态 scaffold 路径也保留 cancelled recording guard。 + +### 🔁 Follow-up (2026-06-27, baseline smoke evidence summary) + +- **[Standalone Smoke Summary]**: `scripts/test-record-and-replay-skill-repo-scaffold.py` 的最终 JSON 现在输出实际检查过的 runtime contract、no-active response、no-arg malformed request gate、wait/notify contract、recording-to-skill handoff、declared handoff paths、scaffold failure exit、cancelled recording guard、`skill-creator` handoff 和真实 lifecycle smoke 标志。 +- **[Baseline JSON Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在捕获 standalone repo smoke 与 npm staged repo smoke 的 JSON 输出，并显式要求关键 evidence 为 true；最终 baseline 摘要新增 `evidence.standaloneSkillRepo` 和 `evidence.npmStagedSkillRepo`，避免组合 smoke 只看子命令退出码。 +- **[Verification]**: 复跑 `make npm-record-and-replay-skill-repo-smoke` 和 `make record-and-replay-baseline-smoke` 通过；新的 baseline 输出同时证明 standalone / npm 路径保留 recording-to-skill、cancelled recording、wait/notify、runtime contract 和 `skill-creator` handoff 边界。 + +### 🔁 Follow-up (2026-06-27, npm staged generated-check evidence) + +- **[NPM Check Evidence]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 现在解析安装态 scaffold 生成 repo 的 `scripts/check.sh` 输出，并显式要求 `checkedWaitNotifyContract`、`checkedDeclaredHandoffPaths`、`checkedScaffoldSkillFailureExit` 和 `checkedCancelledRecordingRejected` 为 true。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 同步要求 npm staged smoke 输出这些新增 evidence，并把它们写入最终 `evidence.npmStagedSkillRepo` 摘要。 +- **[Verification]**: 复跑 `make npm-record-and-replay-skill-repo-smoke` 与 `make record-and-replay-baseline-smoke` 通过；最终 baseline JSON 显示 npm staged 路径已证明 wait/notify、declared paths、scaffold failure exit、cancelled recording rejection、manifest contract 和 `skill-creator` handoff。 + +### 🔁 Follow-up (2026-06-27, official skill handoff semantics) + +- **[Status Semantics]**: `skills/open-computer-use-record-and-replay/SKILL.md` 现在明确 `event_stream_status` 只能在用户询问状态或录制后返回时使用，不能用作等待录制完成的轮询机制。 +- **[Event Stream Access]**: 同一 thin skill 现在明确 MCP server 不直接暴露 event-stream 内容，agent 必须从 `event_stream_stop` / status 返回的 `eventsPath`、`metadataPath` 和 `sessionPath` 读取本地文件。 +- **[Generated Repo Guard]**: standalone repo 生成态 `scripts/verify-skill-workflow.py` 的 required snippets 同步固定上述两条官方 skill handoff 语义，`scripts/test-record-and-replay-skill-repo-scaffold.py` 也会断言生成物包含这些文案。 +- **[Verification]**: 复跑 `make record-and-replay-skill-repo-smoke` 和 `make npm-record-and-replay-skill-repo-smoke` 通过，确认源码 scaffold 与 npm staged scaffold 都保留这两条官方语义。 + +### 🔁 Follow-up (2026-06-27, official handoff negative guards) + +- **[Workflow Negative Guards]**: `scripts/test-record-and-replay-skill-repo-scaffold.py` 现在会临时删除 status 非轮询说明和 MCP 不直接暴露 event-stream 内容说明，分别断言生成态 `scripts/verify-skill-workflow.py` 失败并报告 `missingRequiredSnippets`。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在要求 standalone smoke 输出 `checkedStatusNotUsedAsWaitLoopGuard=true` 和 `checkedMcpNoDirectEventContentsGuard=true`，并把这两项写入最终 `evidence.standaloneSkillRepo`。 +- **[Verification]**: 复跑 `python3 -m py_compile scripts/test-record-and-replay-skill-repo-scaffold.py` 和 `make record-and-replay-skill-repo-smoke` 通过，确认官方 handoff 语义不只停留在正向文案存在检查。 + +### 🔁 Follow-up (2026-06-27, npm official handoff evidence) + +- **[Generated Verifier Evidence]**: standalone repo scaffold 生成态 `scripts/verify-skill-workflow.py` 成功 JSON 现在显式输出 `checkedStatusNotUsedAsWaitLoopGuard=true` 和 `checkedMcpNoDirectEventContentsGuard=true`。 +- **[NPM Check Evidence]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 现在要求 npm staged launcher 生成 repo 后，其默认 `scripts/check.sh` 输出同样两个 guard 字段，并在 npm smoke 最终 JSON 中公开它们。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在同时要求 `evidence.npmStagedSkillRepo` 证明这两个官方 handoff guard，避免安装态 scaffold 路径只继承文案但不暴露机器可读证据。 + +### 🔁 Follow-up (2026-06-27, official surface baseline evidence) + +- **[Official Surface Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在捕获 `scripts/compare-event-stream-surface.py --use-default-official` 的 JSON 输出，显式要求 `local-open-computer-use` 和 `official-record-and-replay` 两个 label 都通过。 +- **[Sanitized Summary]**: baseline 最终 JSON 新增 `evidence.officialSurfaceCompare`，只保留 fixture 名、protocol version、server name、tool names 和 ok 状态，不把本机官方 plugin 路径写入摘要。 +- **[Verification Scope]**: 这仍然只证明官方 1.0.857 non-recording surface 对齐，不替代 successful recording golden fixture。 + +### 🔁 Follow-up (2026-06-27, matrix and action baseline evidence) + +- **[Matrix Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在捕获默认 matrix stdout，要求 default lifecycle handoff、no-active、timeout、wait-timeout、approval、MCP elicitation 和 app-agent wait/notify 路径都出现，并在最终 JSON 写入脱敏 `evidence.eventStreamMatrix`。 +- **[Action Evidence]**: 同一 baseline 脚本现在捕获真实输入 action smoke stdout，要求录制事件包含 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.click`、scroll raw-only `experimentalRawEvents` 和 `session.ended`，且 runtime scaffold 已生成 skill 草稿；最终 JSON 写入 `evidence.realInputActionSmoke`，不保留临时文件路径。 +- **[Why]**: 这样 baseline 摘要不再只证明 standalone/npm scaffold 证据，也能机器可读地证明“可用版本”的核心录制链路和真实输入链路已经跑过。 + +### 🔁 Follow-up (2026-06-27, official fixture set baseline evidence) + +- **[Fixture Set Evidence]**: `scripts/test-event-stream-official-fixture-set.py` 现在输出具体 JSON evidence，覆盖集合级 gate、`simple-action-stop` / `keyboard-input-stop` / `drag-stop` requirement、candidate pairing、缺 candidate / 缺 scenario 负例、stop / cancel endReason policy 和导入 scenario manifest。 +- **[Baseline Summary]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在捕获该 JSON，要求关键 evidence 为 true，并在最终摘要写入 `evidence.officialFixtureSetGate`。 +- **[Golden Boundary]**: baseline 摘要会扫描仓库真实 official recording fixture scenario 覆盖，输出 `repoHasRequiredOfficialSuccessfulFixture` 与 `missingRepoOfficialScenarios`；当前缺 `simple-action-stop` 时仍明确表示 official successful recording 尚未入库，避免把 gate smoke 通过误读成官方 golden 完成。 + +### 🔁 Follow-up (2026-06-27, hosted official fixture ingest) + +- **[Ingest Script]**: 新增 `scripts/ingest-official-record-and-replay-fixture.py`，用于把正常 Codex hosted Record & Replay 的 `event_stream_stop/status` 返回 JSON 转成脱敏 official fixture。脚本会递归解析 direct JSON、`content[].text` 或 JSON-RPC `result.content[].text` 中的 `metadataPath` / `sessionPath` / `eventsPath`，再调用 `scripts/import-event-stream-fixture.py`。 +- **[Readiness]**: ingest 后会按 scenario 运行 `scripts/check-event-stream-golden-readiness.py`；提供 `--mcp-transcript --require-mcp-transcript-evidence --check-fixture-set` 时，会进一步要求 response-shape evidence 并跑集合级 official fixture set gate。 +- **[Smoke / CI]**: 新增 `scripts/test-official-record-and-replay-fixture-ingest.py` 和 `make event-stream-official-fixture-ingest-smoke`，用合成 hosted response 覆盖导入、脱敏、readiness 和 fixture set gate；该 smoke 已接入默认 `scripts/ci.sh`。 +- **[Boundary]**: 这只是让后续官方 successful recording 一键入库和校准，不代表当前仓库已经拥有 official successful recording fixture。 + +### 🔁 Follow-up (2026-06-27, hosted official fixture stdin ingest) + +- **[CLI Usability]**: `scripts/ingest-official-record-and-replay-fixture.py --status-json -` 现在可以从 stdin 直接读取 hosted `event_stream_stop/status` JSON，适合把 Codex tool 返回内容复制或 pipe 进导入脚本。 +- **[Transcript Reuse]**: `--use-status-json-as-transcript` 在 stdin 模式下会把同一 JSON 临时写成 transcript 输入，完成导入后结果中以 `usedMcpTranscript=""` 表示来源，避免输出已清理的临时路径。 +- **[Smoke Coverage]**: `scripts/test-official-record-and-replay-fixture-ingest.py` 覆盖 stdin JSON、transcript 复用、readiness 和 fixture set gate，确保后续采集 official successful recording 时不必先手工创建中间 JSON 文件。 + +### 🔁 Follow-up (2026-06-27, OCU candidate fixture ingest) + +- **[Candidate Script]**: 新增 `scripts/ingest-ocu-record-and-replay-candidate.py`，用于把已有 OCU recording、action smoke JSONL 或 opt-in `--run-action-smoke` 采集结果导入成脱敏 `source=ocu` candidate fixture。 +- **[Readiness / Pairing]**: candidate 导入前先跑 strict OCU validation，导入后按 scenario 跑 readiness；提供 `--official-root --check-fixture-set` 时会直接复用 official fixture set gate 做 same-scenario 对比。若 official 样本要求 MCP response shape evidence，candidate 也可传 `--mcp-transcript --require-mcp-transcript-evidence`。 +- **[Smoke / CI]**: 新增 `scripts/test-ocu-record-and-replay-candidate-ingest.py` 和 `make event-stream-ocu-candidate-ingest-smoke`，覆盖 direct recording、smoke JSONL、脱敏和 official/candidate pairing；该 smoke 已接入默认 `scripts/ci.sh`。 + +### 🔁 Follow-up (2026-06-27, action smoke MCP transcript) + +- **[MCP Recording Path]**: `make event-stream-action-smoke` 现在通过 proxied `event-stream mcp` start/repeat start/status/stop/repeat stop/final status 录制真实输入，不再只通过 CLI start/stop 生成 action recording。 +- **[Transcript Evidence]**: action smoke 输出 JSON 新增 `mcpTranscriptPath`，指向同一 session 的 MCP transcript；baseline 聚合脚本现在要求该字段存在，并在 `realInputActionSmoke.checkedMcpTranscriptCaptured` 中输出证据。 +- **[Candidate Ingest]**: `scripts/ingest-ocu-record-and-replay-candidate.py --smoke-json` 会自动消费 action smoke JSONL 中的 `mcpTranscriptPath`；`--run-action-smoke` 会用短 `/tmp/ocu-rnr-*` 目录保留录制和 transcript 到导入结束，避免 app-agent Unix socket path 过长。 +- **[Manual Import Note]**: README、handoff、architecture 和 reverse-engineering reference 明确补充：`make event-stream-action-smoke` 默认会清理临时目录，手动保存 stdout 给 `--smoke-json` 时需要设置 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP=1` 或 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR=`；否则优先使用 importer 的 `--run-action-smoke`。 + +### 🔁 Follow-up (2026-06-27, official fixture coverage report) + +- **[Coverage Script]**: 新增 `scripts/check-event-stream-official-fixture-coverage.py`，单独扫描仓库 recording fixtures 中的 official scenario 覆盖；默认要求 `simple-action-stop`，缺失时返回非零，`--allow-missing` 只报告 `coverageOk=false`。 +- **[Smoke / CI]**: 新增 `scripts/test-event-stream-official-fixture-coverage.py` 和 `make event-stream-official-fixture-coverage-smoke`，覆盖 missing、allow-missing、candidate ignored、success、duplicate scenario 和 missing scenario 负例；默认 `scripts/ci.sh` 运行 smoke，并用 `--allow-missing` 输出当前仓库缺 official successful recording 的机器可读报告。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 改为复用 coverage report，而不是内嵌扫描逻辑；最终 `evidence.officialFixtureSetGate` 继续暴露 `coverageOk`、`repoHasRequiredOfficialSuccessfulFixture` 和 `missingRepoOfficialScenarios`，避免把 synthetic fixture set gate 通过误读成 official golden 已完成。 + +### 🔁 Follow-up (2026-06-27, candidate readiness in fixture set gate) + +- **[Candidate Readiness]**: `scripts/check-event-stream-official-fixture-set.py` 在 `--candidate-root` 模式下，找到同 scenario OCU candidate 后会先跑 `source=ocu` golden readiness，再做 official-vs-candidate compare。 +- **[Negative Coverage]**: `scripts/test-event-stream-official-fixture-set.py` 新增 candidate `metadata.json` / `session.json` alias 漂移负例，并让 baseline 聚合脚本要求 `checkedCandidateReadinessFailure=true`。 +- **[Why]**: 这避免手动传入结构不完整的 candidate fixture 时只靠 compare 通过；后续 official successful recording 入库后，candidate 必须先证明自身 metadata/session/count/path/MCP evidence 完整。 + +### 🔁 Follow-up (2026-06-27, screenshot path containment) + +- **[Validator Contract]**: runtime Swift validator 与源码 Python validator 现在要求事件 payload 里的 `screenshotPath` 必须解析到当前 session 目录内，并且文件存在。 +- **[Negative Coverage]**: 新增 Swift validator 单测和 `scripts/test-event-stream-skill-scaffold.py` 负例，使用“session 外部但存在”的截图文件，确认失败原因是外部引用而不是 missing file。 +- **[Why]**: 录制到 skill 和后续 official fixture 校准都应该消费自包含 recording 包；外部截图引用会让独立 repo、脱敏 fixture 或用户分享 recording 时出现不可复现上下文。 + +### 🔁 Follow-up (2026-06-27, standalone screenshot containment evidence) + +- **[Standalone Contract]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 standalone repo 现在会在 `recordingToSkill.strictValidation.requiresScreenshotPathsInsideSession=true` 中机器可读声明截图路径必须留在 session 目录内。 +- **[Generated Self-Check]**: 生成态 `scripts/recording-to-skill-smoke.py` 会构造一个 `screenshotPath` 指向 session 外部但文件存在的 recording，要求安装态 `event-stream validate --strict-ocu` 拒绝，并在成功自检中输出 `checkedScreenshotPathContainment=true`。 +- **[Smoke Evidence]**: 源码 standalone smoke、npm staged standalone smoke 和 `record-and-replay-baseline-smoke` 都开始要求该 evidence，避免主仓 validator 修好后，未来拆出去的 thin skill repo 自检遗漏同一边界。 + +### 🔁 Follow-up (2026-06-27, npm staged screenshot containment preflight) + +- **[NPM Artifact Gate]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 现在会在调用安装态 scaffold 前，用 staged launcher 直接校验 session 外部 `screenshotPath` 负例。 +- **[Failure Mode]**: 如果本地 `dist/Open Computer Use.app` 里的 native binary 还没有该 validator gate，smoke 会明确提示重建 `dist` artifacts，而不是等生成 repo 的 `scripts/check.sh` 才以 `external screenshotPath validation should fail` 晚失败。 +- **[Why]**: npm standalone smoke 依赖当前 `dist/` native artifacts；把 artifact freshness 检查前置，可以让 release/staging baseline 的失败原因更接近真实问题。 + +### 🔁 Follow-up (2026-06-27, official fixture stdin base dir) + +- **[Hosted Fixture Ingest]**: `scripts/ingest-official-record-and-replay-fixture.py` 新增 `--status-json-base-dir`，用于解析 `--status-json -` 或转存 status JSON 里出现的相对 `metadataPath` / `sessionPath` / `eventsPath`。 +- **[Smoke Coverage]**: `scripts/test-official-record-and-replay-fixture-ingest.py` 现在覆盖 stdin hosted JSON 中相对 handoff path 的导入，并断言最终 `recordingInput` 解析到指定 base dir 下的 recording。 +- **[Why]**: 官方 hosted 返回通常应是绝对路径，但手工复制、脱敏或中转时可能变成相对路径；显式 base dir 可以避免导入脚本按当前工作目录误解析，提升后续 official golden recording 入库稳定性。 + +### 🔁 Follow-up (2026-06-27, official fixture transcript stdin) + +- **[Hosted Fixture Ingest]**: `scripts/ingest-official-record-and-replay-fixture.py` 新增 `--mcp-transcript -`，允许在 `--status-json` 来自文件时从 stdin 读取单独 MCP transcript / response-shape JSON。 +- **[Input Guard]**: `--status-json -` 与 `--mcp-transcript -` 同时使用会明确失败；同一份 JSON 既是 status 又是 transcript 的场景继续用 `--use-status-json-as-transcript`。 +- **[Smoke Coverage]**: `scripts/test-official-record-and-replay-fixture-ingest.py` 覆盖 transcript stdin 正路径和双 stdin 错误，确认 readiness / fixture set gate 可以消费 stdin transcript evidence。 +- **[Why]**: 正常 Codex hosted 采样后经常会分别拿到 stop/status JSON 与 raw MCP transcript；允许 transcript 直接从 stdin 输入，可以减少临时文件并降低 official golden fixture 入库摩擦。 + +### 🔁 Follow-up (2026-06-27, keyboard and drag official scenario policies) + +- **[Scenario Policies]**: `scripts/check-event-stream-official-fixture-set.py` 新增 `keyboard-input-stop` 和 `drag-stop` 可选 scenario policy，分别要求 `keyboard.text_input` / `mouse.drag`、`recording_controls_stopped` endReason、full AX payload 和 completed lifecycle MCP response-shape evidence。 +- **[Smoke Coverage]**: `scripts/test-event-stream-official-fixture-set.py` 新增 keyboard / drag 成功场景和 keyboard 缺失必需事件负例，最终 JSON evidence 输出 `checkedKeyboardInputScenarioPolicy=true` 和 `checkedDragScenarioPolicy=true`。 +- **[Why]**: 默认 required scenario 仍是 `simple-action-stop`，但后续官方 golden 入库不能只校准点击；提前固化 keyboard / drag 场景策略，可以在采到官方样本后直接进入 same-scenario readiness / compare。 + +### 🔁 Follow-up (2026-06-27, recommended official fixture coverage) + +- **[Coverage Report]**: `scripts/check-event-stream-official-fixture-coverage.py` 现在把 required 和 recommended official scenario coverage 分开输出。required 默认仍是 `simple-action-stop`，会影响非 `--allow-missing` 退出码；recommended 默认报告 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`，只作为后续 official golden 采样路线图。 +- **[Smoke Coverage]**: `scripts/test-event-stream-official-fixture-coverage.py` 覆盖默认 recommended 缺失报告、只覆盖 required 时 recommended 仍未齐、以及自定义 `--recommended-scenario simple-action-stop` 时 recommended coverage 为 true。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 的最终 `evidence.officialFixtureSetGate` 同步输出 `recommendedOfficialScenarios`、`repoHasRecommendedOfficialScenarioCoverage` 和 `missingRepoRecommendedOfficialScenarios`，避免 opt-in baseline 只暴露 required 最小缺口。 +- **[Verification]**: 已复跑 `make event-stream-official-fixture-coverage-smoke`、`make check-docs`、`./scripts/ci.sh` 和 `make record-and-replay-baseline-smoke` 通过；baseline 最终摘要显示 recommended 清单为 `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout`，当前全部缺失，且 required `simple-action-stop` 仍未入库。 +- **[Why]**: 当前最小 official golden 门槛仍是 `simple-action-stop`，但要复刻官方 Record & Replay 不能只校准点击；把 keyboard、drag、cancel、timeout 放进 recommended report，可以让后续采样和比对顺序可见，同时不阻塞当前 baseline。 + +### 🔁 Follow-up (2026-06-27, OCU candidate keyboard and drag readiness) + +- **[Candidate Ingest]**: `scripts/ingest-ocu-record-and-replay-candidate.py` 的场景级 validation/readiness 现在覆盖 `keyboard-input-stop` 和 `drag-stop`，分别要求 `keyboard.text_input` / `mouse.drag`、AX context、`recording_controls_stopped`、full AX payload，以及启用 MCP transcript evidence 时的 repeat start / repeat stop / final status response-shape evidence。 +- **[Smoke Coverage]**: `scripts/test-ocu-record-and-replay-candidate-ingest.py` 新增合成 keyboard / drag OCU recording，确认 candidate ingest 在没有 official fixture 的情况下也会先跑同 scenario readiness，并输出 `checkedKeyboardInputScenarioReadiness=true` / `checkedDragScenarioReadiness=true`。 +- **[Why]**: official fixture set gate 已经定义 keyboard / drag scenario policy；candidate ingest 也必须提前守住同样的最低证据，避免后续采到官方 keyboard / drag 样本时才发现 OCU candidate 导入阶段只做了通用结构检查。 + +### 🔁 Follow-up (2026-06-27, baseline ingest pipeline evidence) + +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在把 hosted official fixture ingest smoke 和 OCU candidate ingest smoke 纳入同一条 opt-in baseline，并在最终摘要新增 `evidence.fixtureIngestPipelines`。 +- **[Gate Coverage]**: 该 evidence 要求 `checkedOfficialFixtureIngest`、`checkedOcuCandidateIngest`、`checkedOcuSmokeJsonImport`、`checkedOcuOfficialCandidatePairing`、`checkedOcuCandidateRedaction`、`checkedOcuKeyboardInputScenarioReadiness` 和 `checkedOcuDragScenarioReadiness` 都为 true。 +- **[Why]**: baseline 不能只证明当前 runtime 能录、standalone repo 能 scaffold；它也要证明后续采到官方 successful recording 后，官方样本入库、OCU candidate 入库、同 scenario pairing 和 keyboard/drag readiness 都有同一条可复跑链路。 + +### 🔁 Follow-up (2026-06-27, baseline screenshot context evidence) + +- **[Smoke Output]**: `scripts/run-event-stream-smoke-tests.sh` 的截图模式现在会输出 `screenshotContextChecked`、`screenshotNeededForContextCount`、`screenshotAvailableCount` 和 `screenshotPathCount`，让调用方能区分“已触发截图上下文标记”和“当前环境实际产出了截图文件”。 +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在显式运行截图上下文 smoke，并在最终摘要新增 `evidence.screenshotContextSmoke`；该 evidence 要求 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=always` 下出现 `screenshotNeededForContext`，如果当前环境实际返回截图数据，则要求 `screenshotPath` 存在。 +- **[Why]**: 截图上下文是官方 Record & Replay 已观察到的关键字段；opt-in baseline 应覆盖它的可用性，同时不能把当前无截图环境误判为失败，也不能把 OCU 的触发策略误称为官方 golden。 + +### 🧠 Design Intent (Why) +这次讨论形成了后续实现会依赖的关键约束：先复刻官方 `record-and-replay` 1.0.857 的 observable behavior，再补充 OCU 自有控制条和 wait/通知扩展。第一版实现先打通官方兼容 MCP surface、可消费的文件产物、基础事件流、AX payload 和按需截图上下文，给后续官方 golden fixtures、算法级 AX compact diff 校准和独立 skill/plugin repo 提供稳定落点。 + +### 📁 Files Modified +- `docs/exec-plans/active/20260626-record-and-replay-event-stream.md` +- `docs/design-docs/record-and-replay-replication.md` +- `docs/ARCHITECTURE.md` +- `docs/QUALITY_SCORE.md` +- `docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md` +- `docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json` +- `docs/references/codex-computer-use-reverse-engineering/README.md` +- `docs/histories/2026-06/20260626-1737-document-record-and-replay-plan.md` +- `README.md` +- `Makefile` +- `scripts/run-event-stream-smoke-tests.sh` +- `scripts/run-event-stream-smoke-matrix.sh` +- `scripts/run-record-and-replay-baseline-smoke.sh` +- `scripts/ingest-official-record-and-replay-fixture.py` +- `scripts/test-official-record-and-replay-fixture-ingest.py` +- `scripts/ingest-ocu-record-and-replay-candidate.py` +- `scripts/test-ocu-record-and-replay-candidate-ingest.py` +- `scripts/ci.sh` +- `scripts/package-skill.sh` +- `scripts/compare-event-stream-surface.py` +- `scripts/probe-event-stream-recording.py` +- `scripts/test-event-stream-probe-fixtures.py` +- `scripts/scaffold-event-stream-skill.py` +- `scripts/test-event-stream-skill-scaffold.py` +- `scripts/install-codex-record-and-replay-mcp.sh` +- `scripts/test-codex-record-and-replay-installer.mjs` +- `scripts/scaffold-record-and-replay-skill-repo.py` +- `scripts/test-record-and-replay-skill-repo-scaffold.py` +- `scripts/install-config-helper.mjs` +- `scripts/npm/build-packages.mjs` +- `docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json` +- `.github/workflows/ci.yml` +- `.github/workflows/docs-check.yml` +- `.github/workflows/repo-hygiene.yml` +- `.github/workflows/supply-chain-security.yml` +- `.github/PULL_REQUEST_TEMPLATE.md` +- `.github/ISSUE_TEMPLATE/bug_report.yml` +- `.github/ISSUE_TEMPLATE/feature_request.yml` +- `.github/ISSUE_TEMPLATE/config.yml` +- `.github/dependency-review-config.yml` +- `.editorconfig` +- `.markdownlint.json` +- `scripts/import-event-stream-fixture.py` +- `scripts/check-event-stream-official-fixture-coverage.py` +- `scripts/test-event-stream-official-fixture-coverage.py` +- `scripts/check-event-stream-official-fixture-set.py` +- `scripts/test-event-stream-official-fixture-set.py` +- `scripts/compare-event-stream-recordings.py` +- `scripts/test-event-stream-fixture-import.py` +- `scripts/test-event-stream-recording-compare.py` +- `scripts/validate-event-stream-recording.py` +- `scripts/summarize-event-stream-recording.py` +- `skills/open-computer-use/SKILL.md` +- `skills/open-computer-use/references/record-and-replay.md` +- `skills/open-computer-use-record-and-replay/SKILL.md` +- `apps/OpenComputerUse/Sources/OpenComputerUse/MCPAppRuntime.swift` +- `apps/OpenComputerUse/Sources/OpenComputerUse/MacOSAppAgentProxy.swift` +- `apps/OpenComputerUse/Sources/OpenComputerUse/OpenComputerUseMain.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamMCPServer.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingControls.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingSummary.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingValidation.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamSkillScaffold.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamService.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamToolDefinitions.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift` +- `packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift` +- `packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift` + +## 2026-06-27 Record & Replay baseline no-active evidence + +- `scripts/run-record-and-replay-baseline-smoke.sh` 现在显式运行 `scripts/compare-event-stream-no-active.py`，要求本地 OCU MCP no-active `event_stream_status` / `event_stream_stop` 与 Codex-hosted 官方 1.0.857 fixture 的最小 text JSON 对齐。 +- baseline 成功摘要新增 `evidence.officialNoActiveResponse`，记录 fixture 名、checked tools、status/stop shape 和 no-session-files evidence，避免后续只看 surface compare 时漏掉 hosted no-active response 语义。 + +## 2026-06-27 Standalone official evidence manifest + +- `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 standalone repo manifest 新增 `officialEvidence`，机器可读声明当前 contract 基于官方 `record-and-replay/1.0.857` 的 surface fixture、Codex-hosted no-active fixture 和 hostless raw timeout fixture。 +- standalone / npm staged / baseline smoke 现在都会检查 `checkedOfficialEvidenceManifest=true`，同时 manifest 明确 `hasSuccessfulRecordingGolden=false`，避免独立 repo 把 non-recording 证据误读成 official successful recording golden 已完成。 +- `officialEvidence` 现在还写入 required `simple-action-stop` 与 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` 成功录制场景清单，并通过 `officialEvidence.scenarioRecipes` 固化每个场景的 capture goal、预期 action event、预期 endReason、evidence 要求和 OCU candidate 来源；源码 standalone 与 npm staged scaffold smoke 都按 exact manifest 断言，避免后续拆独立 repo 时丢失官方 golden 采样路线或采样动作细节。 +- standalone / npm staged / baseline smoke 现在额外输出 `checkedOfficialEvidenceScenarioManifest=true`，把场景清单检查从泛化的 official evidence manifest 检查中拆出来，便于 baseline 摘要直接证明采样路线仍被守住。 + +## 2026-06-27 Baseline readiness status + +- `scripts/run-record-and-replay-baseline-smoke.sh` 的最终 JSON 新增顶层 `status`，直接输出 `usableBaseline`、`officialNonRecordingBaselineVerified`、`standaloneRepoScaffoldBaselineVerified`、`officialSuccessfulRecordingGoldenComplete`、`recommendedOfficialRecordingCoverageComplete`、`requiresOfficialGoldenCapture` 和缺失的 required / recommended official successful recording scenarios；`officialSuccessfulRecordingGoldenComplete` 现在要求 required scenario 覆盖和 required readiness 同时通过。 +- 该字段只汇总已有 smoke / coverage 结果，不改变 gate 通过条件；目的是把“当前 baseline 可用”和“官方 successful recording golden 仍未完成”放到同一层机器可读结论里，避免后续人工从 `evidence.officialFixtureSetGate` 里间接推断。 + +## 2026-06-27 Strict official golden gate + +- `scripts/run-record-and-replay-baseline-smoke.sh` 新增 `--require-official-golden`，在完整跑完 baseline 聚合检查并输出最终 JSON 后，如果 required official successful recording fixture 仍缺失就返回非零。 +- `Makefile` 新增 `make record-and-replay-official-golden-gate`，作为后续 official golden 入库后的 release / 拆 standalone repo 验收入口。 +- 严格模式会在最终摘要写入 `ok=false`、`status.strictOfficialGoldenRequired=true`、`status.officialGoldenGatePassed=false` 和缺失的 required official scenario；当前缺 `simple-action-stop` 时该命令预期失败，默认 `make record-and-replay-baseline-smoke` 仍保持“baseline 可用但 official golden 未完成”的通过语义。 + +## 2026-06-27 Baseline summary helper + +- `scripts/run-record-and-replay-baseline-smoke.sh` 不再内嵌最终摘要的大段 Python heredoc，改为调用 `scripts/build-record-and-replay-baseline-summary.py` 生成机器可读 JSON 并处理 `--require-official-golden` 退出码。 +- 新增 `scripts/test-record-and-replay-baseline-summary.py` 和 `make record-and-replay-baseline-summary-smoke`，用合成 JSON/JSONL 证据覆盖默认模式允许缺 official golden、严格模式缺 required official fixture 失败、严格模式覆盖 required official fixture 通过三种分支。 +- 默认 `scripts/ci.sh` 接入该快测，让 `status.strictOfficialGoldenRequired`、`status.officialGoldenGatePassed` 和 required official scenario 缺口字段的回归不再只依赖 opt-in 完整 baseline smoke。 + +## 2026-06-27 Baseline usable evidence derivation + +- `scripts/build-record-and-replay-baseline-summary.py` 不再把 `status.usableBaseline` 写死为 `true`；现在会从 event-stream matrix required modes、截图上下文、真实输入 action required event types、官方 non-recording surface、官方 no-active response、official fixture set gate、fixture ingest pipeline、standalone repo self-check 和 npm staged repo self-check 的 evidence 推导。 +- 默认 baseline 仍允许缺少 official successful recording golden；但如果上述必需 evidence 任一缺失，最终摘要会写入 `ok=false`、`status.usableBaseline=false` 和 `status.missingUsableBaselineEvidence=[...]`，并返回非零。 +- `scripts/test-record-and-replay-baseline-summary.py` 新增缺失 npm staged `checkedSkillCreatorHandoff` 的负例，固定默认模式下 usable baseline evidence 不完整也会失败。 + +## 2026-06-27 Official fixture ingest coverage gate + +- `scripts/ingest-official-record-and-replay-fixture.py` 新增 `--check-coverage`，导入 hosted official recording 后会调用 `scripts/check-event-stream-official-fixture-coverage.py --allow-missing --check-readiness`，并在返回 JSON 中写入 `coverage` report。 +- 新增 `--require-coverage`，导入后要求 output root 满足 required official successful recording scenario 覆盖，且 required fixture readiness 通过；当前 required 仍是 `simple-action-stop`，缺失或 readiness 未过时命令返回非零。 +- `scripts/test-official-record-and-replay-fixture-ingest.py` 新增 post-ingest coverage/readiness 成功路径和 cancel-only fixture 下 require-coverage 失败路径；baseline summary 也把 `checkedPostIngestCoverageReport` / `checkedPostIngestCoverageReadiness` / `checkedPostIngestRequireCoverageFailure` 纳入 `fixtureIngestPipelines` evidence。 + +## 2026-06-27 Official fixture not-ready diagnostics + +- `scripts/check-event-stream-official-fixture-coverage.py` 新增顶层 `notReadyOfficialScenarios`，只在 required scenario 已有 official fixture 但集合级 readiness 未通过时填充；缺样本仍只通过 `missingOfficialScenarios` 表示。 +- `scripts/build-record-and-replay-baseline-summary.py` 将该字段汇总到 `status.notReadyRequiredOfficialSuccessfulRecordingScenarios` 和 `evidence.officialFixtureSetGate.notReadyRepoOfficialScenarios`，严格 official golden gate 失败时 stderr 会分开输出 `missing=...` 与 `notReady=...`。 +- `make event-stream-official-fixture-coverage-smoke` 与 `make record-and-replay-baseline-summary-smoke` 已覆盖 missing、covered-but-not-ready 和 ready 三类分支。 + +## 2026-06-27 Official fixture inspect-only ingest + +- `scripts/ingest-official-record-and-replay-fixture.py` 新增 `--inspect-only`，只解析 hosted status JSON / recording input / handoff paths / MCP transcript evidence，不创建 fixture，用于官方 successful recording 采样后先检查返回体是否可导入。 +- `scripts/test-official-record-and-replay-fixture-ingest.py` 覆盖 inspect-only 成功、不创建 output dir、stdin status JSON 复用 transcript 成功，以及要求 MCP transcript evidence 但未提供时失败。 +- `scripts/build-record-and-replay-baseline-summary.py` 与合成 summary smoke 新增 `checkedOfficialFixtureInspectOnly` evidence；README、架构、handoff、逆向参考和执行计划同步说明先 inspect 再正式导入。 + +## 2026-06-27 Scenario-aware OCU action candidate smoke + +- `scripts/run-event-stream-smoke-tests.sh` 的真实输入 action smoke 新增 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO`，默认仍跑 `mixed-action-stop` 覆盖 click 和 scroll raw-only，显式设置 `simple-action-stop`、`drag-stop` 时分别采样 click 和 drag。 +- `scripts/ingest-ocu-record-and-replay-candidate.py --run-action-smoke --scenario ` 现在会把 simple / drag 映射到同名 action smoke 场景，并在 capture evidence 里记录 `actionScenario`；其它非 keyboard scenario 回退到 mixed，只作为通用真实监听链路证据。 +- `keyboard-input-stop` candidate 仍通过已有 recording / smoke JSON 导入和 readiness gate 覆盖；因当前 macOS 对 synthetic keyboard event 的 session tap 过滤不稳定，暂不把 keyboard 纳入 `--run-action-smoke` 承诺。README、架构、handoff、逆向参考、执行计划和质量记录同步说明该能力属于 OCU candidate baseline，后续仍需要 official successful recording golden 做同场景校准。 +- 复跑 `make record-and-replay-baseline-smoke` 通过，最终摘要证明当前 usable baseline 仍完整：matrix、截图上下文、默认真实输入 action smoke、官方 1.0.857 non-recording surface、官方 no-active response、official fixture set gate、official/OCU ingest pipeline、standalone repo 和 npm staged repo evidence 均通过；同时继续输出 `officialSuccessfulRecordingGoldenComplete=false` 和缺失的 official successful recording 场景，避免把 baseline 可用误读为官方 golden 已完成。 diff --git a/docs/histories/2026-06/20260627-0112-record-and-replay-doc-handoff.md b/docs/histories/2026-06/20260627-0112-record-and-replay-doc-handoff.md new file mode 100644 index 00000000..59dbeb61 --- /dev/null +++ b/docs/histories/2026-06/20260627-0112-record-and-replay-doc-handoff.md @@ -0,0 +1,915 @@ +## [2026-06-27 01:12] | Task: Record & Replay 文档交接 + +### 🤖 Execution Context +* **Agent ID**: `Codex` +* **Base Model**: `GPT-5` +* **Runtime**: `Codex CLI` + +### 📥 User Query +> 把方案和相关上下文落到 docs 里，后续会基于这个推进解决。 + +### 🛠 Changes Overview +**Scope:** docs + +**Key Actions:** +- **[Handoff]**: 新增 `docs/design-docs/record-and-replay-handoff.md`，作为 Record & Replay 后续推进的短入口。 +- **[Context]**: 汇总官方兼容层 / OCU 扩展层边界、已沉淀官方证据、当前 OCU baseline、待解决问题、下一轮推进顺序和常用验证入口。 +- **[Navigation]**: 更新 `docs/design-docs/index.md`，让后续 Agent 可以从设计索引直接发现 handoff 文档。 +- **[Concurrent Recording]**: 补齐官方 one-active handoff：重复 start 返回 active session 时，agent 应询问用户要使用当前录制还是等待它结束，不能静默把它当成本次新 demonstration。 + +### 🧠 Design Intent (Why) +Record & Replay 的实现上下文已经横跨设计、逆向、执行计划、skill 和验证脚本。新增短 handoff 文档用于把“下一轮应该从哪里读、按什么顺序推进、哪些边界不能破坏”固定到仓库里，避免后续继续依赖聊天上下文。 + +### 📁 Files Modified +- `docs/design-docs/record-and-replay-handoff.md` +- `docs/design-docs/index.md` +- `docs/histories/2026-06/20260627-0112-record-and-replay-doc-handoff.md` + +### 🔁 Follow-up (2026-06-27, clarify replication boundaries) + +- **[Official Replication Boundary]**: 补充“直接复刻官方”的仓库口径：复刻可观察协议、session 文件、事件流、AX payload、截图上下文和 skill handoff，不复制官方私有实现，也不把 OCU 私有扩展塞进三件套 MCP surface。 +- **[Codex.app Boundary]**: 明确当前 asar 观察只支持把 Codex.app 视为 host / feature gate / 通用 elicitation UI；OCU 必须自己承担开始确认、录制中 bar、Done / Discard、wait 唤醒和 notify callback。 +- **[Calibration Notes]**: 在 handoff 中补齐 screenshot、compact AX diff、skill handoff 和 golden fixture 的易混边界，后续推进时必须区分官方已确认、OCU baseline 和待 official golden 校准。 + +### 🔁 Follow-up (2026-06-27, make the plan actionable) + +- **[Execution Protocol]**: 在 `record-and-replay-handoff.md` 增加后续推进协议，要求每个改动先归类为官方兼容层、OCU 扩展层或待官方校准 baseline，再按证据、实现、验证、文档同步闭环推进。 +- **[Decision Scope]**: 在 `record-and-replay-replication.md` 固定当前方案口径：macOS first、接受 `event-stream` 子命令、直接复刻官方可观察行为、OCU 自己承载控制条 / wait / notify，独立 repo 只包装 runtime。 +- **[Plan Entry]**: 在 active execution plan 中补充下一轮先读 handoff 的入口说明，避免后续推进继续依赖聊天上下文。 + +### 🔁 Follow-up (2026-06-27, standalone repo contract) + +- **[Manifest Contract]**: standalone thin skill repo scaffold 的 `record-and-replay-skill-repo.json` 新增默认 `checks`，把 package、runtime contract、synthetic recording-to-skill smoke 和组合 self-check 明确为发布/安装前门禁；`optionalChecks` 只保留会真实创建 recording session 的 lifecycle smoke。 +- **[Docs Sync]**: 在 handoff、replication、architecture、CI/CD、quality score 和 active execution plan 中同步“默认自检不启动真实录制，可选 lifecycle 才启动录制”的边界。 +- **[Test Sync]**: 更新 standalone repo scaffold smoke 断言，确保生成态 manifest 的 `checks` / `optionalChecks` 分层不会回退。 + +### 🔁 Follow-up (2026-06-27, freeze replication decision) + +- **[Decision Entry]**: 在 handoff 中补充本轮结论，明确后续以“直接复刻官方可观察行为 + OCU 独立扩展”为推进基线。 +- **[Interface Contract]**: 在 replication 文档中增加官方兼容合同、OCU 扩展合同、校准合同和独立 repo 合同，方便后续评审判断新能力应落在哪一层。 +- **[Evidence Boundary]**: 在逆向参考中补充当前证据能证明和不能证明的内容，避免把 Codex.app feature gate / elicitation 观察误解成 Codex.app 自己实现录制控制条或事件文件。 + +### 🔁 Follow-up (2026-06-27, doc landing map) + +- **[Landing Map]**: 在 handoff 中补充方案落库地图，明确短入口、完整设计、执行计划、官方逆向证据、thin skill、通用 skill 参考和 standalone repo scaffold 的位置。 +- **[Next Paths]**: 补充后续最短路径，分别覆盖继续官方复刻和拆真实 standalone repo 两条推进方式。 +- **[Backlink]**: 在 replication 文档关联文档区补充 handoff 反链，方便从完整设计回到短入口。 + +### 🔁 Follow-up (2026-06-27, doc completeness gate) + +- **[Completion Criteria]**: 在 handoff 中补充落库完成判定，明确后续必须能从仓库回答官方复刻原因、官方兼容层 / OCU 扩展层边界、待 golden 校准 baseline、官方样本采集路径和 standalone repo 自检分层。 +- **[Context Hygiene]**: 固定规则：如果新改动的关键判断只能从聊天上下文推断，应先补文档再继续实现。 + +### 🔁 Follow-up (2026-06-27, official golden capture checklist) + +- **[Capture Checklist]**: 新增 `docs/design-docs/record-and-replay-official-golden-capture.md`，把官方 successful recording golden fixture 的采集前检查、hosted JSON inspect-only、正式导入、OCU candidate 对比和完成判定收敛成短操作清单。 +- **[Navigation]**: 更新 `docs/design-docs/index.md` 和 `docs/design-docs/record-and-replay-handoff.md`，让后续 R&R 推进可以从 handoff 直接跳到 official golden 采集流程。 + +### 🔁 Follow-up (2026-06-27, official golden capture preflight) + +- **[Preflight Script]**: 新增 `scripts/prepare-record-and-replay-official-golden-capture.py`，作为采集官方 successful recording 前的只读 preflight，检查 official plugin cache、当前 scenario coverage/readiness，并输出 inspect-only、正式导入、strict gate 和 OCU candidate 命令。 +- **[Smoke]**: 新增 `scripts/test-record-and-replay-official-golden-capture-preflight.py` 和 `make record-and-replay-official-golden-capture-preflight-smoke`，覆盖缺 required scenario、`--require-ready` 失败、缺 official plugin、允许缺 plugin 以及 keyboard 场景不使用 synthetic action smoke 的输出合同；`make record-and-replay-official-golden-capture-preflight` 保留为真实只读 preflight 入口。 +- **[Docs Sync]**: 更新 official golden capture 文档、handoff 和架构说明，明确采样前优先跑 preflight，且该入口不会启动录制或写 fixture。 + +### 🔁 Follow-up (2026-06-27, fast official fixture gate) + +- **[Fast Gate]**: 新增 `make record-and-replay-official-golden-fixture-gate`，直接复用 `scripts/check-event-stream-official-fixture-coverage.py --require-readiness`，用于导入官方 successful recording 后快速验证 required scenario 覆盖和 readiness。 +- **[Docs Sync]**: 更新 README、中文 README、official capture checklist、handoff、architecture 和 quality score，明确 fast fixture-only gate 与完整 `make record-and-replay-official-golden-gate` release gate 的区别。 + +### 🔁 Follow-up (2026-06-27, OCU candidate pairing preflight) + +- **[Pairing Preflight]**: 新增 `scripts/prepare-record-and-replay-ocu-candidate-pairing.py`，在 official fixture 入库后只读检查 same-scenario OCU candidate 状态，并生成 candidate ingest 与 fixture-set compare 命令。 +- **[Smoke]**: 新增 `scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py` 和 `make record-and-replay-ocu-candidate-pairing-preflight-smoke`，覆盖缺 official、official ready 但缺 candidate、candidate paired 通过和 keyboard 场景不走 synthetic action smoke 的输出合同。 +- **[Docs Sync]**: 更新 README、中文 README、official capture checklist、handoff、architecture 和 quality score，明确 pairing preflight 不启动 action smoke、不写 candidate fixture，只负责组织官方入库后的下一步。 + +### 🔁 Follow-up (2026-06-27, scenario recipe preflight output) + +- **[Scenario Catalog]**: 新增 `scripts/record_and_replay_scenarios.py`，把 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel` 和 `timeout` 的采集动作、预期 action event、预期 endReason、OCU candidate 来源方式和注意事项收敛成共享机器可读 recipe。 +- **[Preflight Contract]**: `prepare-record-and-replay-official-golden-capture.py` 和 `prepare-record-and-replay-ocu-candidate-pairing.py` 的 JSON 输出新增 `scenarioRecipe`，让后续官方 golden 采样和 OCU candidate 配对不再依赖聊天上下文。 +- **[Smoke]**: 更新两个 preflight smoke，固定 required click 场景和 keyboard 场景的 recipe 输出，尤其是 keyboard 不走 synthetic `--run-action-smoke` 的约束。 + +### 🔁 Follow-up (2026-06-27, scenario recipe fixture manifest) + +- **[Fixture Manifest]**: `scripts/import-event-stream-fixture.py` 现在会把 `scenarioRecipe` 写入 `fixture-manifest.json`，让脱敏后的 official / OCU fixture 仍保留采集合同。 +- **[Gate]**: `scripts/check-event-stream-official-fixture-set.py` 会校验 manifest recipe 与当前 scenario catalog 一致，拒绝只有 scenario 名但 recipe 漂移的 fixture。 +- **[Smoke]**: 更新 fixture import smoke 和 official fixture set smoke，覆盖 recipe 写入和 recipe drift 负例。 + +### 🔁 Follow-up (2026-06-27, scenario catalog readiness policy) + +- **[Catalog Policy]**: `scripts/record_and_replay_scenarios.py` 新增 shared readiness args 生成逻辑，让 expected action event、expected endReason、full AX / no-action 放行和 MCP response-shape evidence 都从 scenario catalog 派生。 +- **[Gate Sync]**: official fixture set gate、official coverage report、hosted official ingest 和 OCU candidate ingest 都改为复用同一份 scenario catalog；direct ingest 不再维护弱于集合 gate 的场景规则。 +- **[Smoke]**: official ingest smoke 新增 `keyboard-input-stop` / `drag-stop` direct readiness 覆盖，并复跑 fixture set、coverage、official ingest、OCU candidate ingest 相关 smoke。 + +### 🔁 Follow-up (2026-06-27, coverage recipe drift gate) + +- **[Coverage Gate]**: `scripts/check-event-stream-official-fixture-coverage.py` 现在也会校验 official fixture manifest 的 `scenarioRecipe` 是否匹配共享 catalog；不跑 readiness 的快速 coverage report 也不会把 recipe 漂移的 fixture 算作有效 golden 覆盖。 +- **[Smoke]**: `scripts/test-event-stream-official-fixture-coverage.py` 新增 recipe drift 负例，确认 `scenarioRecipeMatches=false` 会进入错误输出。 +- **[Docs Sync]**: 同步 architecture、official capture checklist 和 quality score，明确 coverage 层也会拒绝只有 scenario 名但采集合同漂移的样本。 + +### 🔁 Follow-up (2026-06-27, baseline coverage error evidence) + +- **[Summary Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 现在会把 official fixture coverage report 的 `errors` 保留到 `status.officialFixtureCoverageErrors` 和 `evidence.officialFixtureSetGate.coverageErrors`。 +- **[Strict Gate Output]**: 严格 official golden gate 失败时会把 recipe drift、重复 scenario 或 manifest 错误作为 `coverageErrors=...` 输出到 stderr，不再退化成模糊的 readiness 失败。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary.py` 新增 coverage error 负例，确认最终 JSON 与 stderr 都能暴露具体 coverage 错误。 + +### 🔁 Follow-up (2026-06-27, preflight coverage errors) + +- **[Official Preflight]**: `scripts/prepare-record-and-replay-official-golden-capture.py` 现在把 coverage report `errors` 提升为顶层 `coverageErrors`，并在 `nextActions` 中要求先修复 recipe drift、重复 scenario 或 manifest 错误。 +- **[Pairing Preflight]**: `scripts/prepare-record-and-replay-ocu-candidate-pairing.py` 现在会先跑 official fixture coverage report，再跑 fixture set gate；输出 `officialCoverageErrors`，避免坏 manifest 的 official fixture 进入 OCU candidate pairing 判断。 +- **[Smoke]**: official capture preflight 和 OCU pairing preflight smoke 都新增 recipe drift 负例，确认 coverage error 会在 JSON 和 next action 中显式出现。 + +### 🔁 Follow-up (2026-06-27, standalone scenario recipes) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `record-and-replay-skill-repo.json` 现在在 `officialEvidence.scenarioRecipes` 中写入 recommended scenario 的完整 recipe，而不只保留场景名清单。 +- **[Contract Scope]**: 这些 recipe 固化 capture goal、预期 action event、预期 endReason、evidence 要求和 OCU candidate 来源，方便独立 repo 后续按同一合同采集官方 golden 和配对 OCU candidate。 +- **[NPM Packaging]**: npm package 现在随 `scaffold-record-and-replay-skill-repo.py` 一起复制 `record_and_replay_scenarios.py`，保证安装态 scaffold 命令也能生成同一份 manifest recipe。 +- **[Smoke]**: 源码 scaffold smoke 与 npm staged scaffold smoke 都按 exact manifest 断言 `scenarioRecipes`，npm staged smoke 还会显式检查 scenario catalog helper 已被打包，避免后续拆 repo 时丢失采样动作细节。 + +### 🔁 Follow-up (2026-06-27, official capture packet) + +- **[Preflight Packet]**: `scripts/prepare-record-and-replay-official-golden-capture.py --capture-packet-dir ` 现在会生成官方采集包，包含 `README.md`、`preflight.json`、`scenario-recipe.json`、hosted status/transcript placeholder 和 inspect/import/check wrapper。 +- **[Packet Set]**: 加 `--capture-packet-recommended-scenarios` 时，会为 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel` 和 `timeout` 各生成一个子目录，并写入根级 `capture-packets.json`、`inspect-all.sh`、`import-all.sh`、`check-all.sh` 和 `ingest-ocu-candidates.sh`。 +- **[Boundary]**: 该入口仍不会启动官方录制，也不会写 fixture；它只服务录完官方样本后的无写入 inspect-only 与脱敏导入交接。 +- **[Smoke]**: official golden capture preflight smoke 覆盖单场景 / 批量 packet 文件结构、placeholder JSON、scenario recipe 和 wrapper 脚本输入路径。 + +### 🔁 Follow-up (2026-06-27, baseline summary gate wording) + +- **[Baseline Smoke]**: 复跑 `make record-and-replay-baseline-smoke` 通过，证明当前 usable baseline 仍完整；摘要继续报告 required `simple-action-stop` 和 recommended official successful recording fixtures 缺失。 +- **[Status Semantics]**: `scripts/build-record-and-replay-baseline-summary.py` 现在区分 `officialGoldenGatePassed` 和 `officialGoldenRequirementSatisfied`。默认模式允许缺 official golden 时只让 requirement satisfied，不再把 missing golden 误标为 gate passed。 +- **[Smoke]**: baseline summary smoke 更新默认缺 golden、严格缺 golden、baseline evidence 缺失和严格 covered 四类断言。 + +### 🔁 Follow-up (2026-06-27, docs navigation tightening) + +- **[Design Index]**: 在 `docs/design-docs/index.md` 增加 Record & Replay 默认阅读顺序，固定 handoff、完整设计、official golden capture checklist、逆向参考和 active execution plan 的入口关系。 +- **[Reference Index]**: 在 `docs/references/README.md` 明确 Record & Replay 逆向参考入口，说明该文档承载官方插件包装、Codex.app asar 分工、runtime 字符串、event-stream surface、non-recording fixtures 和 official golden 缺口。 + +### 🔁 Follow-up (2026-06-27, baseline runner early evidence gate) + +- **[Runner Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在在 source / npm staged standalone 子 smoke 后立即检查完整必需 evidence，再进入 summary builder。 +- **[Evidence Coverage]**: 早期 gate 新增 generated README scenario list、`sourceRepoBaselineAudit` manifest、malformed MCP request 副作用、notify suppressed path、callback failure 和 callback timeout evidence，避免缺字段到最终 summary / artifact audit 才暴露。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-runner-summary-json.py` 新增篡改 standalone evidence 的负例，确认缺关键字段时 runner 返回非零且不会写 `--summary-json` artifact。 + +### 🔁 Follow-up (2026-06-27, standalone evidence shared contract) + +- **[Shared Contract]**: `scripts/record_and_replay_baseline_contract.py` 新增 standalone / npm staged skill repo smoke required keys 和 summary evidence keys，作为 baseline runner early gate 与 artifact audit 的共享来源。 +- **[Runner Sync]**: `scripts/run-record-and-replay-baseline-smoke.sh` 的 source / npm staged standalone early gate 改为读取 shared contract，不再手写长 key 列表。 +- **[Audit Sync]**: `scripts/check-record-and-replay-baseline-summary.py` 的 standalone / npm staged direct evidence audit 改为读取 shared contract。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary.py` 断言最终 summary 的 `standaloneSkillRepo` / `npmStagedSkillRepo` evidence key 集合与 shared contract 完全一致。 + +### 🔁 Follow-up (2026-06-27, baseline contract self-test) + +- **[Contract Smoke]**: 新增 `scripts/test-record-and-replay-baseline-contract.py` 和 `make record-and-replay-baseline-contract-smoke`，直接校验 baseline shared contract 非空、无重复，并固定 standalone lifecycle smoke keys 到 summary evidence keys 的重命名关系。 +- **[CI Sync]**: `scripts/ci.sh` 默认运行该 contract smoke，避免 runner、summary builder 和 artifact audit 都消费同一份 contract 时仍遗漏 contract 内部映射错误。 + +### 🔁 Follow-up (2026-06-28, baseline contract artifact evidence) + +- **[Baseline Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在会运行 baseline contract smoke，并把 JSON 写入最终 summary 的 `evidence.baselineContract`。 +- **[Summary Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 将 shared checks、standalone/npm key 集和 standalone lifecycle rename mapping 纳入 `usableBaseline` 必需 evidence。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 会审计 `evidence.baselineContract`；summary smoke 与 audit smoke 都覆盖缺失 / 篡改 contract evidence 的负例。 + +### 🔁 Follow-up (2026-06-28, standalone baseline contract manifest) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `officialEvidence.sourceRepoBaselineChecks` 新增 `baselineContract=baseline-contract-smoke`，README 同步说明源仓 baseline 会先跑 baseline contract smoke。 +- **[Smoke]**: source scaffold smoke 与 npm staged scaffold smoke 的 exact manifest 断言同步覆盖该字段，避免独立 repo handoff 漏掉 shared contract self-check 证据入口。 + +### 🔁 Follow-up (2026-06-28, refreshed baseline audit artifact) + +- **[Full Baseline Audit]**: 复跑 `make record-and-replay-baseline-audit` 通过，刷新 `dist/record-and-replay-baseline-summary.json`。 +- **[Artifact Evidence]**: 新 summary 的 `checks` 包含 `baseline-contract-smoke`，`evidence.baselineContract.ok=true`，并通过 `scripts/check-record-and-replay-baseline-summary.py` 审计。 +- **[Current Gap]**: summary 继续保持 `usableBaseline=true` 和 `standaloneRepoBaselineReady=true`，同时保持 `officialSuccessfulRecordingGoldenComplete=false` / `officialSuccessfulRecordingEquivalenceReady=false`，缺口仍是 required `simple-action-stop` 以及 recommended `cancel`、`drag-stop`、`keyboard-input-stop`、`simple-action-stop`、`timeout` official successful recording fixtures。 + +### 🔁 Follow-up (2026-06-27, malformed MCP request side effects) + +- **[Runtime Contract]**: standalone thin skill repo manifest 新增 `mcpServer.rejectedRequestsDoNotCreateSessionFiles=true`，把 malformed official-compatible `tools/call` 请求不创建 session 文件的边界显式落进机器可读 contract。 +- **[Evidence]**: 生成态 `verify-runtime.py`、源码 standalone smoke、npm staged smoke 和 baseline summary 现在都会输出 / 消费 `checkedRejectsUnexpectedArguments`、`checkedRejectsNonObjectArguments` 和 `checkedRejectedRequestsDoNotCreateSessionFiles`。 +- **[Smoke]**: baseline summary smoke 新增负例，确认缺少 rejected request side-effect evidence 时 `usableBaseline=false`。 + +### 🔁 Follow-up (2026-06-27, npm staged request-shape parity) + +- **[Parity]**: npm staged standalone smoke 现在也要求生成 repo 的 `check.sh` 输出 `checkedRequiresObjectParams`、`checkedRequiresStringToolName` 和 `checkedRequiresObjectArguments`，与源码 standalone smoke 的 runtime contract 证据保持一致。 +- **[Summary Gate]**: baseline summary 将 npm staged request-shape evidence 纳入 `usableBaseline` 必需条件，缺少安装态 params/name/arguments shape 证明时会返回非零。 +- **[Smoke]**: baseline summary smoke 新增 npm staged request-shape 负例，确认 `npmStagedSkillRepo.checkedRequiresObjectParams` 等证据不会被遗漏。 + +### 🔁 Follow-up (2026-06-27, official raw timeout baseline evidence) + +- **[Probe Evidence]**: `scripts/test-event-stream-probe-fixtures.py` 现在输出官方 raw start/status/stop 宿主外 timeout 的细分 evidence，包含三次 `tools/call` timeout、无 recording handoff paths、官方 surface 和脱敏检查。 +- **[Summary Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 会显式运行 raw timeout fixture smoke；`scripts/build-record-and-replay-baseline-summary.py` 写入 `evidence.officialRawStartTimeout`，并把这些字段纳入 `usableBaseline` 必需 evidence。 +- **[Smoke]**: baseline summary smoke 新增 raw timeout 负例，确认缺少 `checkedOfficialRawStartDoesNotReturnRecordingPaths` 时默认 baseline 会失败，而不是把 hostless raw timeout 误读成 successful recording 证据。 + +### 🔁 Follow-up (2026-06-27, preflight baseline evidence) + +- **[Preflight Evidence]**: official golden capture preflight smoke 与 OCU candidate pairing preflight smoke 现在都会输出机器可读 JSON 摘要，便于 baseline 聚合脚本消费。 +- **[Summary Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 会显式运行两条 preflight smoke；`scripts/build-record-and-replay-baseline-summary.py` 写入 `evidence.preflightPipelines`，并把 capture packet Make targets、capture packet、recommended packet set、packet set 根级 placeholder guard、缺 official 样本、缺插件、coverage error、keyboard recording-required、缺 candidate、paired candidate 和 compare 成功路径纳入 `usableBaseline` 必需 evidence。 +- **[Smoke]**: baseline summary smoke 新增 official capture preflight 与 OCU pairing preflight 负例，确认缺少这些下一步操作入口证据时默认 baseline 会失败。 + +### 🔁 Follow-up (2026-06-27, standalone preflight manifest contract) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `officialEvidence` 新增 `sourceRepoBaselineChecks`，声明 hostless raw timeout、official golden capture preflight 和 OCU candidate pairing preflight 由 OCU 源仓 baseline 验证。 +- **[Boundary Contract]**: 同一 manifest 新增 `standaloneRepoBoundary`，明确独立 repo 默认自检不启动官方录制、不复制 preflight 脚本、不复制 OCU runtime 源码；生成 README 同步说明这些 preflight 只属于源仓 release / baseline gate。 +- **[Evidence]**: 源码 scaffold smoke、npm staged scaffold smoke 和 baseline summary 都新增 `checkedOfficialEvidencePreflightManifest=true`，并在 baseline summary smoke 中加入缺失负例，避免后续拆 repo 时只保留 successful recording 场景清单却丢失采样/配对前置门禁。 + +### 🔁 Follow-up (2026-06-27, baseline readiness split) + +- **[Status Split]**: `scripts/build-record-and-replay-baseline-summary.py` 新增 `standaloneRepoBaselineReady` 和 `officialSuccessfulRecordingEquivalenceReady`。前者由 usable baseline + standalone/npm scaffold evidence 推导，后者由 usable baseline + required official successful recording golden 推导。 +- **[Misclaim Guard]**: 当前缺 official successful recording golden 时，baseline 可以明确显示 `standaloneRepoBaselineReady=true` 但 `officialSuccessfulRecordingEquivalenceReady=false`，避免把“可拆 standalone repo 的 baseline”误读成“官方事件 schema / AX / screenshot 等价已经完成”。 +- **[Smoke]**: baseline summary smoke 新增默认缺 golden、baseline evidence 缺失和 strict covered 三类断言，固定这两个派生状态的语义。 + +### 🔁 Follow-up (2026-06-27, baseline next actions) + +- **[Summary Guidance]**: `scripts/build-record-and-replay-baseline-summary.py` 新增顶层 `nextActions`，根据当前 evidence 输出下一步机器可读动作。 +- **[Action Branches]**: 缺 usable baseline evidence 时输出修复 baseline 的命令；缺 required official golden 时输出 official capture preflight、inspect-only ingest 和 strict golden gate 命令；recommended 场景未齐时输出 capture packet set 命令；standalone baseline ready 时输出 scaffold repo、生成 repo `check.sh` 和 optional lifecycle smoke 命令。 +- **[Smoke]**: baseline summary smoke 覆盖默认缺 golden、baseline evidence 缺失和 strict covered 三类 `nextActions` 分支，确保后续发版 / 拆 repo 不需要人工从底层 evidence 拼下一步。 + +### 🔁 Follow-up (2026-06-27, baseline summary artifact output) + +- **[Baseline Artifact]**: `scripts/run-record-and-replay-baseline-smoke.sh --summary-json ` 现在可以把最终机器可读 baseline 摘要写入文件，同时继续向 stdout 打印同一 JSON。 +- **[Exit Semantics]**: `--summary-json` 不改变默认 baseline 或 `--require-official-golden` 严格 gate 的退出码；严格 gate 缺官方 successful recording golden 时仍会在写出摘要后返回非零。 +- **[Docs Sync]**: handoff、replication、official capture guide、execution plan 和 quality score 都记录该参数，方便后续 release / standalone audit 留存 evidence。 + +### 🔁 Follow-up (2026-06-27, official capture packet make targets) + +- **[Make Targets]**: 新增 `make record-and-replay-official-golden-capture-packet` 和 `make record-and-replay-official-golden-capture-packet-set`，分别生成单场景 official capture packet 和 required+recommended packet set。 +- **[Safety Boundary]**: 两个 target 只调用 read-only preflight 与 packet writer，不启动官方录制、不写 fixture；输出目录默认在系统临时目录，可用 `RNR_PACKET_DIR=` 覆盖，单场景可用 `RNR_SCENARIO=` 覆盖。 +- **[Docs Sync]**: official capture guide、handoff、architecture 和 quality score 已同步记录这两个入口，后续采集 official successful recording golden 时不用手写长参数。 + +### 🔁 Follow-up (2026-06-27, packet make target smoke coverage) + +- **[Make Variables]**: capture packet Make targets 新增 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`、`RNR_FIXTURE_ROOT` 和 `RNR_OFFICIAL_PLUGIN_ROOT`，允许在没有官方 bundled plugin cache 的干净环境里只生成 packet 命令。 +- **[Smoke]**: `scripts/test-record-and-replay-official-golden-capture-preflight.py` 现在直接调用两个 Make target，验证单场景 packet、recommended packet set、missing-plugin allowed 模式和生成文件结构。 +- **[Baseline Evidence]**: baseline summary 现在消费 `checkedMakeCapturePacketTargets`，如果 Make packet 入口不再生成可用采集包，`usableBaseline` 会失败并报告 `preflightPipelines.checkedOfficialCapturePacketMakeTargets`。 +- **[Docs Sync]**: official capture guide、handoff、architecture、execution plan 和 quality score 已同步说明这些变量。 + +### 🔁 Follow-up (2026-06-27, baseline next actions packet commands) + +- **[Summary Guidance]**: baseline summary 的 `nextActions` 现在把 official capture 下一步直接指向 `make record-and-replay-official-golden-capture-packet` 和 `make record-and-replay-official-golden-capture-packet-set`。 +- **[Packet Wrappers]**: required golden 缺口会列出 packet 内 `inspect-only.sh` / `import-fixture.sh` 和 strict gate；recommended 缺口会列出 packet set 内 `inspect-all.sh` / `import-all.sh` / `ingest-ocu-candidates.sh`。 +- **[Smoke]**: baseline summary smoke 新增命令级断言，确保默认缺 official golden 时机器可读摘要不会退回旧的手写 preflight 参数。 + +### 🔁 Follow-up (2026-06-27, baseline next action packet execution) + +- **[Smoke]**: baseline summary smoke 现在会取 `nextActions` 里的 packet Make 命令，替换临时 `RNR_PACKET_DIR` 后实际执行，验证 required 单场景 packet 能生成 `preflight.json`、`scenario-recipe.json`、`inspect-only.sh` 和 `import-fixture.sh`。 +- **[Smoke]**: 同一 smoke 也会执行 recommended packet set 命令，验证 `capture-packets.json`、`inspect-all.sh`、`import-all.sh` 和 `ingest-ocu-candidates.sh` 存在，并固定 recommended 场景顺序。 +- **[Offline Mode]**: 测试执行时追加 `RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1`、`RNR_FIXTURE_ROOT` 和 `RNR_OFFICIAL_PLUGIN_ROOT` 覆盖，保证干净环境也能验证 summary 命令链路本身。 + +### 🔁 Follow-up (2026-06-27, capture packet placeholder guard) + +- **[Wrapper Guard]**: capture packet 的 `inspect-only.sh` / `import-fixture.sh` 现在会在执行导入命令前检查 input JSON 是否仍带 `_placeholder=true`。 +- **[Failure Mode]**: 如果用户还没有把 hosted official JSON 写入 `inputs/event_stream_stop-response.json` 或 `inputs/mcp-transcript.json`，wrapper 会以明确错误退出，避免把占位 JSON 传进 ingest 链路。 +- **[Smoke]**: official capture preflight smoke 覆盖 status response 与 MCP transcript 两类 placeholder guard，也执行 packet set 根级 `inspect-all.sh` / `import-all.sh`，确认它们会传播子 packet 的 placeholder 失败；`check-coverage.sh` 这类不消费 hosted JSON 的 wrapper 不会被该 guard 阻塞。baseline summary 也把 `checkedOfficialCapturePacketPlaceholderGuard`、`checkedOfficialCapturePacketTranscriptPlaceholderGuard` 和 `checkedOfficialCapturePacketSetRootPlaceholderGuard` 纳入 `preflightPipelines` evidence。 + +### 🔁 Follow-up (2026-06-27, handoff context consolidation) + +- **[Context Handoff]**: `docs/design-docs/record-and-replay-handoff.md` 新增“快速上下文结论”，把官方 plugin / Codex.app / Computer Use runtime 的分工、OCU 自有 bar / wait / notify 边界、官方三件套 no-arg surface、截图上下文口径和 compact AX diff 定义集中写清。 +- **[Calibration Boundary]**: 同一段明确当前只有 non-recording surface、no-active response 和 hostless raw timeout evidence，仍没有 official successful recording fixture；事件 schema、AX compact diff、截图触发、timeout endReason 和 raw event 结构都不能宣称官方等价。 +- **[Next Step]**: 文档明确后续缺 official golden 时应优先走 capture packet、替换 hosted JSON、运行 packet wrapper inspect/import，再进入 fixture gate 和 OCU candidate pairing，避免继续依赖聊天上下文拼命令。 + +### 🔁 Follow-up (2026-06-28, baseline action scenario evidence) + +- **[Baseline Audit]**: 完整 `make record-and-replay-baseline-audit` 现在在默认 `mixed-action-stop` action smoke 之外，额外运行 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop|drag-stop`，把 click / drag 两类 OCU 同场景 candidate 采样纳入同一份 baseline evidence。 +- **[Summary Evidence]**: `scripts/build-record-and-replay-baseline-summary.py`、`scripts/check-record-and-replay-baseline-summary.py` 和相关 summary / audit smoke 已把 `realInputActionSmoke.checkedSimpleActionStopCandidate` 与 `checkedDragStopCandidate` 作为 `usableBaseline` 必需 evidence；缺少任一场景候选都会让 release / standalone audit 摘要失败。 +- **[Runner Smoke]**: `scripts/test-record-and-replay-baseline-runner-summary-json.py` 的 fake action smoke stub 按 `mixed-action-stop` / `simple-action-stop` / `drag-stop` 分别返回单场景 JSONL，确认 runner 真实传递 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO`，避免聚合 fixture 掩盖参数漂移。 +- **[Docs Sync]**: handoff、replication、architecture、active execution plan 和 quality score 同步说明该证据只证明 OCU candidate 链路可用，不替代 required `simple-action-stop` official successful recording golden；当前 official equivalence 仍应保持 false。 +- **[Validation]**: 已跑通 `make record-and-replay-baseline-audit`，落盘摘要显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`checkedSimpleActionStopCandidate=true`、`checkedDragStopCandidate=true`，同时继续报告 required `simple-action-stop` official successful recording 缺口。 + +### 🔁 Follow-up (2026-06-28, runtime surface evidence split) + +- **[Runtime Evidence]**: 生成态 `scripts/verify-runtime.py` 在 `checkedRuntimeContract` 之外新增 `checkedInitializeSurfaceContract`、`checkedToolMetadataContract` 和 `checkedToolInputSchemaNoArguments`，分别证明 initialize surface、三件套完整 tool metadata 和 no-arg input schema。 +- **[Baseline Gate]**: shared baseline contract、source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费这三项 evidence，缺失时 release / standalone audit 会把对应 `standaloneSkillRepo.*` 或 `npmStagedSkillRepo.*` 标为缺口。 +- **[Validation]**: 已跑 `python3 scripts/test-record-and-replay-baseline-contract.py`、`python3 scripts/test-record-and-replay-baseline-summary.py`、`python3 scripts/test-record-and-replay-baseline-runner-summary-json.py`、`python3 scripts/test-record-and-replay-skill-repo-scaffold.py` 和 `make record-and-replay-baseline-audit`；刷新后的 baseline artifact 显示 source / npm staged 三项新 evidence 都为 true。未启动官方录制。 + +### 🔁 Follow-up (2026-06-27, import wrapper placeholder evidence) + +- **[Wrapper Guard]**: official capture preflight smoke 现在同时执行单场景 `import-fixture.sh` 的 status response placeholder 和 MCP transcript placeholder 负例，确认正式导入 wrapper 自身会在 ingest 前拒绝未替换输入。 +- **[Summary Gate]**: baseline summary 新增 `preflightPipelines.checkedOfficialCapturePacketImportPlaceholderGuard` evidence；缺少该证据时 `usableBaseline=false`，避免只验证 inspect-only wrapper 却漏掉正式导入路径。 +- **[Docs Sync]**: architecture、handoff、replication、execution plan 和 quality score 同步说明单场景 inspect/import wrapper 与 packet set 根级 wrapper 都属于 official capture packet baseline 证据。 + +### 🔁 Follow-up (2026-06-27, capture packet set root preflight) + +- **[Wrapper Guard]**: packet set 根级 `inspect-all.sh` / `import-all.sh` 现在会先检查所有场景子目录里的 hosted status response 与 MCP transcript 输入是否仍带 `_placeholder=true`，任一场景未替换就直接退出，不会先执行前置场景的子 wrapper。 +- **[Smoke]**: official capture preflight smoke 会把第一个场景输入替换成非占位 JSON、保留后续场景占位，再运行根级 `import-all.sh`，确认它在执行 `simple-action-stop` 子 wrapper 前就因后续 `keyboard-input-stop` 占位输入失败。 +- **[Summary Gate]**: baseline summary 新增 `checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard` evidence；该字段缺失或为 false 时，`usableBaseline=false` 并在 `missingUsableBaselineEvidence` 中报告 `preflightPipelines.checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard`。 +- **[Docs Sync]**: architecture、handoff、replication、official capture guide、execution plan 和 quality score 同步说明根级批量 wrapper 的 preflight-all 语义，避免后续把它误解为只传播子 wrapper 的 placeholder guard。 + +### 🔁 Follow-up (2026-06-27, baseline runner summary artifact smoke) + +- **[Runner Smoke]**: 新增 `scripts/test-record-and-replay-baseline-runner-summary-json.py`，用临时 fake repo 和 stub evidence 运行真实 `run-record-and-replay-baseline-smoke.sh --summary-json` shell 路径，避免完整桌面 baseline 才能发现 artifact 输出回归。 +- **[Artifact Contract]**: 测试确认 `--summary-json` 写出的 JSON 与 stdout 中的最终机器摘要一致，也确认默认模式缺 official golden 时仍返回 0 且 `officialGoldenRequirementSatisfied=true`。 +- **[Strict Exit]**: 同一测试打开 `--require-official-golden`，确认缺 required `simple-action-stop` official successful recording fixture 时会写出失败摘要、保留非零退出码，并在 stderr 中暴露 missing scenario。 +- **[CI Sync]**: 默认 `scripts/ci.sh` 接入该 runner smoke；`docs/CICD.md`、handoff 和 quality score 同步记录这条 release / standalone audit evidence。 + +### 🔁 Follow-up (2026-06-27, baseline summary artifact audit) + +- **[Audit CLI]**: 新增 `scripts/check-record-and-replay-baseline-summary.py`，直接消费已落盘的 baseline summary JSON，不重跑完整桌面 smoke。 +- **[Default Policy]**: 默认审计要求 `usableBaseline`、`standaloneRepoBaselineReady`、official non-recording baseline 和 raw timeout boundary 成立；允许缺 official successful recording golden，但要求 `officialSuccessfulRecordingEquivalenceReady=false`，避免把可拆 standalone baseline 误读成官方事件 schema 等价。 +- **[Strict Policy]**: `--require-official-golden` 审计要求 summary 本身处于 strict mode，并要求 golden gate、required scenario gaps、coverage errors 和 `officialSuccessfulRecordingEquivalenceReady` 全部满足。 +- **[Runner Integration]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在生成 summary 后会自动审计同一份摘要；audit 输出走 stderr，stdout 和 `--summary-json` 文件仍保持最终 summary JSON。 +- **[Smoke]**: 新增 `scripts/test-record-and-replay-baseline-summary-audit.py` 和 `make record-and-replay-baseline-summary-audit-smoke`，覆盖默认缺 golden 可过、严格缺 golden 必 fail、strict covered 通过、inconsistent equivalence 失败和 incomplete baseline 失败，并接入默认 CI；runner summary-json smoke 同步确认 runner 会调用 audit。 + +### 🔁 Follow-up (2026-06-27, baseline summary top-level audit) + +- **[Audit Invariant]**: `scripts/check-record-and-replay-baseline-summary.py` 现在不仅审计 `status.*`，也要求顶层 `baseline=record-and-replay`，并要求顶层 `ok` 与 `status.usableBaseline && status.officialGoldenRequirementSatisfied` 一致。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增顶层 `ok` 被篡改和 baseline 名称被篡改两个负例，避免 release / standalone audit 阶段误消费非本 baseline 或状态不一致的 summary artifact。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步记录顶层 summary 不变量；这不改变当前缺 official successful recording golden 时 `officialSuccessfulRecordingEquivalenceReady=false` 的边界。 + +### 🔁 Follow-up (2026-06-27, scenario default catalog source) + +- **[Default Source]**: official capture preflight、OCU candidate pairing preflight、official hosted ingest、OCU candidate ingest、official fixture coverage report 和 official fixture set gate 的默认 required scenario / help 文案统一从 `scripts/record_and_replay_scenarios.py` 派生。 +- **[Drift Guard]**: 相关 preflight / ingest 测试现在覆盖无 `--scenario` 时使用 `DEFAULT_REQUIRED_SCENARIOS[0]`，避免 required 场景清单后续变化时 CLI 默认值和采集/导入链路继续写死旧场景。 +- **[Docs Sync]**: official golden capture guide 与 quality score 同步记录这条 catalog-first 约束；当前 official successful recording golden 仍未入库，官方事件 schema 等价状态不变。 + +### 🔁 Follow-up (2026-06-27, wait notify callback failure evidence) + +- **[Standalone Contract]**: 生成态 `record-and-replay-skill-repo.json` 的 `extensionLayer.waitNotify` 新增 `callbackFailureMakesCliFail=true`，把失败 callback 会让 CLI 非零退出的语义写成机器可读 contract。 +- **[Smoke Evidence]**: 生成态 `scripts/wait-notify-contract-smoke.py` 现在输出 `checkedNotifyCallbackFailureExit` 和 `checkedNotifyCallbackFailureReason`；源码 standalone smoke 与 npm staged smoke 都要求这些字段为 true。 +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 把 standalone / npm staged 的 callback failure evidence 纳入 `usableBaseline` 必需项；baseline summary smoke 新增缺失负例，避免独立 repo 只验证 callback skipped，却漏掉 callback 真实失败时的通知和退出码合同。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步记录该 wait/notify 扩展边界；官方兼容 MCP surface 仍保持三件套无参数。 + +### 🔁 Follow-up (2026-06-27, wait notify callback timeout evidence) + +- **[Standalone Contract]**: 生成态 `record-and-replay-skill-repo.json` 的 `extensionLayer.waitNotify` 新增 `callbackTimeoutMakesCliFail=true`，把 notify callback 超时也会让 CLI 非零退出的语义写成机器可读 contract。 +- **[Smoke Evidence]**: 生成态 `scripts/wait-notify-contract-smoke.py` 现在用合成 completed session 运行 sleep callback，并要求 `notification.reason=timeout`、`notification.timedOut=true` 且 CLI 非零；源码 standalone smoke 与 npm staged smoke 都消费 `checkedNotifyCallbackTimeoutFailureExit` 和 `checkedNotifyCallbackTimeoutReason`。 +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 把 standalone / npm staged 的 callback timeout evidence 纳入 `usableBaseline` 必需项；baseline summary smoke 新增缺失负例，避免独立 repo 只验证非零退出而漏掉 callback 超时失败路径。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步记录非零退出与超时两类 wait/notify callback 失败都属于 OCU 扩展层，不进入官方三件套 MCP surface。 + +### 🔁 Follow-up (2026-06-27, standalone lifecycle semantic evidence) + +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 现在把 standalone lifecycle smoke 的 `checkedOneActive`、`checkedIdempotentStop` 和 `checkedFinalStatus` 映射成 `checkedLifecycleOneActive`、`checkedLifecycleIdempotentStop` 和 `checkedLifecycleFinalStatus`，并纳入 `usableBaseline` 必需项。 +- **[Runner Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 解析 standalone smoke JSON 时也要求这三个字段为 true，避免完整 opt-in baseline 只确认 lifecycle smoke 退出成功。 +- **[Smoke Evidence]**: baseline summary smoke 新增缺失负例，确认 one-active、stop 幂等或 completed final status 任一语义缺失时都会进入 `missingUsableBaselineEvidence`。 +- **[Docs Sync]**: handoff、replication、execution plan 和 quality score 同步说明 npm staged 默认自检仍不启动真实录制，源码 standalone baseline 会额外消费可选 lifecycle smoke 的具体语义 evidence。 + +### 🔁 Follow-up (2026-06-27, standalone recording-to-skill concrete evidence) + +- **[Baseline Gate]**: baseline summary 现在把 source standalone `recording-to-skill` smoke 的 `checkedStrictValidation`、`checkedEventsOnlyValidation`、`checkedScaffoldSkill` 和 `checkedSkillCreatorHandoff` 纳入 `usableBaseline` 必需项。 +- **[Runner Gate]**: `scripts/run-record-and-replay-baseline-smoke.sh` 解析 standalone smoke JSON 时提前要求这些字段为 true，避免最终聚合前输入证据已经不完整。 +- **[Smoke Evidence]**: baseline summary smoke 新增缺失负例，确认完整 OCU session strict gate、events-only gate、scaffold-skill 正路径或 skill-creator handoff 任一缺失时都会进入 `missingUsableBaselineEvidence`。 +- **[Docs Sync]**: handoff、replication、execution plan 和 quality score 同步说明 source standalone baseline 不只消费 recording-to-skill 总开关，还消费具体 validation/scaffold/handoff 子路径证据。 + +### 🔁 Follow-up (2026-06-27, npm staged recording-to-skill concrete evidence) + +- **[NPM Smoke]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 现在从安装态 launcher 生成 repo 的 `check.sh` 输出中断言 `checkedStrictValidation`、`checkedEventsOnlyValidation` 和 `checkedScaffoldSkill`，并把这些字段写入 npm staged smoke 最终 JSON。 +- **[Baseline Gate]**: baseline summary 与 runner 都把 npm staged strict validation、events-only validation 和 scaffold-skill 正路径 evidence 纳入 `usableBaseline` 必需项。 +- **[Smoke Evidence]**: baseline summary smoke 新增 npm staged 缺失负例，确认安装态 concrete evidence 缺失会报告到 `npmStagedSkillRepo.*`。 +- **[Docs Sync]**: handoff、replication、execution plan 和 quality score 同步说明 recording-to-skill concrete evidence 同时适用于 source standalone 与 npm staged 路径。 + +### 🔁 Follow-up (2026-06-27, real input action concrete evidence) + +- **[Action Smoke]**: `scripts/run-event-stream-smoke-tests.sh` 的真实输入 action 模式现在把已内部断言的 MCP response shapes、skill readiness、`skill-creator` finalization handoff 和路径脱敏结果写成机器可读 JSON 字段。 +- **[Baseline Gate]**: baseline summary 与 runner 都把 `checkedMcpResponseShapesCaptured`、`checkedSkillReadinessCanCreateDraft`、`checkedSkillCreatorFinalizationHandoff` 和 `checkedGeneratedSkillPathRedaction` 纳入 `realInputActionSmoke` 必需项。 +- **[Smoke Evidence]**: baseline summary smoke 新增 action concrete evidence 缺失负例，确认缺失时会报告到 `realInputActionSmoke.*`。 +- **[Docs Sync]**: handoff、replication、execution plan 和 quality score 同步说明真实输入录制到 skill 的 baseline 证据不再只依赖 `skillPath` / `mcpTranscriptPath` 存在。 + +### 🔁 Follow-up (2026-06-27, baseline audit make targets) + +- **[Make Targets]**: 新增 `make record-and-replay-baseline-audit` 和 `make record-and-replay-official-golden-gate-audit`，分别包装默认 baseline 与 strict official golden gate 的 `--summary-json` 路径。 +- **[Artifact Path]**: 两个 target 默认把 summary artifact 写到 `dist/record-and-replay-baseline-summary.json`，可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖，方便 release / standalone audit 留存同一份机器 evidence。 +- **[Exit Semantics]**: 默认 audit 仍保持缺 official successful recording golden 时可通过的 baseline 语义；strict audit 仍保持缺 required official golden 时非零退出但写出失败摘要的 gate 语义。 +- **[Docs Sync]**: official capture guide、handoff、replication、execution plan 和 quality score 同步记录 Make audit 入口，避免后续使用方手写 runner 参数。 + +### 🔁 Follow-up (2026-06-27, baseline audit target smoke) + +- **[Smoke]**: 新增 `scripts/test-record-and-replay-baseline-audit-make-targets.py` 和 `make record-and-replay-baseline-audit-targets-smoke`，用 `make -n` 验证两个 audit Make target 的命令展开。 +- **[Coverage]**: smoke 覆盖默认 summary 路径、自定义 `RNR_BASELINE_SUMMARY_JSON` 路径和 strict target 的 `--require-official-golden` 参数，避免 Make 包装入口漂移。 +- **[CI Sync]**: 默认 `scripts/ci.sh` 接入该 dry-run smoke；它不启动完整 baseline、不触发官方录制，只守住 release / standalone audit 的短命令入口。 +- **[Docs Sync]**: CICD、handoff、replication、execution plan 和 quality score 同步说明该 smoke 的边界。 + +### 🔁 Follow-up (2026-06-27, baseline audit target summary evidence) + +- **[Runner]**: `scripts/run-record-and-replay-baseline-smoke.sh` 现在也会运行 `scripts/test-record-and-replay-baseline-audit-make-targets.py`，并把输出 JSON 传给 baseline summary builder。 +- **[Summary Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 新增 `--baseline-audit-targets-json` 输入，并在 `evidence.preflightPipelines` 写入 `checkedBaselineAuditMakeTargets`、`checkedBaselineAuditDefaultSummaryPath`、`checkedBaselineAuditCustomSummaryPath` 和 `checkedBaselineAuditStrictOfficialGoldenTarget`。 +- **[Usable Baseline]**: 上述四项现在属于 `usableBaseline` 必需 evidence；如果 release / standalone audit 的 Make wrapper 或 `--summary-json` / `--require-official-golden` 参数漂移，完整 baseline summary 会失败并报告对应 `missingUsableBaselineEvidence`。 +- **[Tests]**: 更新 summary smoke 和 runner summary-json stub smoke，覆盖新增参数、正向 evidence 和 audit target evidence 缺失负例。 + +### 🔁 Follow-up (2026-06-27, baseline audit artifact evidence) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在会直接审计落盘 summary 中的 baseline audit Make target evidence，而不是只相信 `status.usableBaseline`。 +- **[Checks]**: 默认 artifact audit 新增 `checkedBaselineAuditMakeTargetsEvidence`、`checkedBaselineAuditDefaultSummaryPathEvidence`、`checkedBaselineAuditCustomSummaryPathEvidence` 和 `checkedBaselineAuditStrictOfficialGoldenTargetEvidence`。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增负例：即使 summary 的 status 仍声称 usable，只要 `evidence.preflightPipelines` 里的 audit target evidence 被改成 false，artifact audit 也会失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone summary 复审会直接检查该 evidence。 + +### 🔁 Follow-up (2026-06-27, official boundary artifact evidence) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在也会直接审计落盘 summary 中的 official boundary evidence，而不是只相信 `status.officialNonRecordingBaselineVerified` 或 `status.officialRawStartTimeoutBoundaryVerified`。 +- **[Checks]**: 默认 artifact audit 新增 official surface local/bundled evidence、official no-active status/stop shape 与 no-session-files evidence，以及 raw start/status/stop timeout、无 recording paths、raw surface 和 fixture redaction evidence。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增负例：即使 summary 的 status 仍声称 usable，只要 `evidence.officialSurfaceCompare`、`evidence.officialNoActiveResponse` 或 `evidence.officialRawStartTimeout` 被改成 false，artifact audit 也会失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone summary 复审会直接检查官方 non-recording / raw-timeout 边界 evidence。 + +### 🔁 Follow-up (2026-06-27, capture packet input verification) + +- **[Packet Wrapper]**: official capture packet 现在生成 `verify-inputs.sh`，在 inspect/import 前单独检查 hosted status response 和 MCP transcript 输入是否存在、JSON 可解析且不再是 `_placeholder=true`。 +- **[Packet Set]**: recommended packet set 现在生成根级 `verify-all.sh`，逐个运行子 packet 的输入校验；根级 `inspect-all.sh` / `import-all.sh` 仍保留先全量 placeholder preflight、再执行子 wrapper 的半批次导入防护；根级 `ingest-ocu-candidates.sh` 负责批量运行可自动生成 OCU candidate 的子场景。 +- **[Baseline Evidence]**: `scripts/build-record-and-replay-baseline-summary.py` 和 runner 现在消费 `checkedCapturePacketVerifyInputs` / `checkedCapturePacketSetVerifyAll`，缺失时 `usableBaseline=false` 并进入 `missingUsableBaselineEvidence`。 +- **[Smoke]**: `scripts/test-record-and-replay-official-golden-capture-preflight.py` 覆盖单场景 `verify-inputs.sh` 的 status/transcript 占位失败与替换后成功，也覆盖 packet set `verify-all.sh`；`scripts/test-record-and-replay-baseline-summary.py` 新增这两项 evidence 缺失负例。 +- **[Docs Sync]**: official capture guide、handoff、replication、architecture、CICD、execution plan 和 quality score 同步记录 verify wrapper 属于采集包输入合同，不启动官方录制，也不替代 inspect-only / readiness gate。 + +### 🔁 Follow-up (2026-06-27, capture packet nextActions verify step) + +- **[Summary Guidance]**: `scripts/build-record-and-replay-baseline-summary.py` 的 official capture `nextActions` 现在会显式列出 `verify-inputs.sh` 和 `verify-all.sh`，避免调用方只按机器命令跑 inspect/import 而跳过输入校验。 +- **[Preflight Guidance]**: `scripts/prepare-record-and-replay-official-golden-capture.py --capture-packet-dir` 的 `nextActions` 文案现在也指向 `capturePacket.verifyInputsShell`，和生成的 packet manifest 保持一致。 +- **[Smoke]**: baseline summary smoke 断言 required / recommended capture commands 包含 verify 与 OCU candidate ingest 步骤，并实际执行 packet Make 命令确认 `verify-inputs.sh`、`verify-all.sh` 与 `ingest-ocu-candidates.sh` 都被生成；official capture preflight smoke 断言 packet JSON 暴露 `verifyInputsShell`。 +- **[Docs Sync]**: handoff、replication、execution plan 和 quality score 同步说明 baseline summary 的机器下一步是 verify → inspect → import。 + +### 🔁 Follow-up (2026-06-27, full baseline revalidation) + +- **[Baseline Smoke]**: 复跑 `make record-and-replay-baseline-smoke` 通过，完整覆盖默认 event-stream matrix、截图上下文 smoke、真实输入 action smoke、官方 1.0.857 non-recording surface 对比、official no-active response、raw start/status/stop timeout fixture、official fixture set gate、fixture ingest、OCU candidate ingest、capture/pairing preflight、source standalone repo smoke 和 npm staged standalone repo smoke。 +- **[Status Evidence]**: 最终摘要保持 `status.usableBaseline=true`、`status.standaloneRepoBaselineReady=true`、`status.officialNonRecordingBaselineVerified=true` 和 `status.officialRawStartTimeoutBoundaryVerified=true`。 +- **[Golden Gap]**: 同一摘要仍保持 `status.officialSuccessfulRecordingGoldenComplete=false`、`status.officialSuccessfulRecordingEquivalenceReady=false`，并报告缺少 required `simple-action-stop` 以及 recommended `simple-action-stop` / `keyboard-input-stop` / `drag-stop` / `cancel` / `timeout` official successful recording fixtures。 +- **[Boundary]**: 本次没有启动官方 successful recording；官方 raw start/status/stop 边界仍只证明宿主外 timeout 与 non-recording surface，不替代 Codex-hosted successful recording golden。 + +### 🔁 Follow-up (2026-06-27, strict gate audit env isolation) + +- **[Bug]**: 运行 `RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-official-golden-summary.json make record-and-replay-official-golden-gate-audit` 时，完整 baseline 前置检查通过，但内嵌的 audit Make target dry-run smoke 误继承外层 `RNR_BASELINE_SUMMARY_JSON`，导致“默认 summary 路径”断言失败。 +- **[Fix]**: `scripts/test-record-and-replay-baseline-audit-make-targets.py` 现在在调用 `make -n` 前显式移除继承的 `RNR_BASELINE_SUMMARY_JSON`；自定义 summary 路径仍通过 make 命令参数覆盖。 +- **[CI Coverage]**: `scripts/ci.sh` 现在带外层 `RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-summary.json` 运行同一 dry-run smoke，确保环境隔离回归进入默认 CI，而不只依赖手工命令。 +- **[Verification]**: `RNR_BASELINE_SUMMARY_JSON=/tmp/outer-should-not-affect-default.json make record-and-replay-baseline-audit-targets-smoke` 通过，证明默认路径检查不再受外层环境污染。 +- **[CI]**: `./scripts/ci.sh` 已复跑通过，覆盖文档/仓库卫生、脚本语法、skill 打包、Record & Replay 默认 smoke、Swift 单测、默认 event-stream matrix 和 Windows / Linux Go test；新的外层环境变量 smoke 已进入这条默认路径。 +- **[Strict Gate Evidence]**: 重跑 strict audit 后，所有 baseline 子证据与 standalone / npm staged smoke 都通过，最终只因缺 required `simple-action-stop` official successful recording fixture 而非零退出，并写出 strict summary artifact；artifact 中 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`officialGoldenRequirementSatisfied=false`、`officialSuccessfulRecordingEquivalenceReady=false`。 + +### 🔁 Follow-up (2026-06-27, capture packet repo-root handoff) + +- **[Packet README]**: official capture packet 生成的 README 现在说明 wrapper 会写入创建 packet 时的仓库路径，并支持用 `REPO_ROOT=/path/to/repo` 覆盖，方便把 packet 目录交给另一个 checkout 使用。 +- **[Fix]**: 删除旧文案里“通过 `git rev-parse` 推断仓库”的说法；真实 wrapper 使用的是 embedded `default_repo_root`，旧说法会误导后续 official hosted JSON 采集交接。 +- **[Smoke]**: `scripts/test-record-and-replay-official-golden-capture-preflight.py` 已断言 packet README 包含 embedded repo path / `REPO_ROOT` 覆盖说明，且不包含 `git rev-parse`。 + +### 🔁 Follow-up (2026-06-27, no-transcript capture packet set) + +- **[Bug]**: 单场景 capture packet 支持 `--no-include-transcript`，但 recommended packet set 的根级 `inspect-all.sh` / `import-all.sh` 仍无条件检查每个子目录的 `inputs/mcp-transcript.json`，会让无 transcript 采集包在批量模式下误失败。 +- **[Fix]**: packet set root wrapper 现在按 `include_transcript` 配置生成 placeholder preflight；默认路径仍检查 status + transcript，`--no-include-transcript` 路径只检查 status response。 +- **[Evidence]**: official capture preflight smoke 输出 `checkedCapturePacketSetNoTranscript=true`，baseline summary 将其映射为 `preflightPipelines.checkedOfficialCapturePacketSetNoTranscript` 并纳入 `usableBaseline` 必需 evidence；summary smoke 新增该 evidence 缺失负例。 + +### 🔁 Follow-up (2026-06-27, capture packet transcript manifest) + +- **[Manifest Contract]**: 单场景 `capturePacket`、根级 `capturePacketSet` 和 `capture-packets.json` 现在都会写入 `includeTranscript` / `requiresMcpTranscriptInput`，并在单场景 packet 中写入 status response 与 MCP transcript input path，方便后续采集包消费方直接从 JSON 判断是否需要 transcript 输入。 +- **[No Transcript Mode]**: 使用 `--no-include-transcript` 生成 recommended packet set 时，根级 manifest 与每个子 packet 的 transcript requirement 字段都会是 `false`，`mcpTranscriptInputPath=null`，与 wrapper 不检查 `inputs/mcp-transcript.json` 的行为一致。 +- **[Baseline Evidence]**: baseline summary 现在消费 `preflightPipelines.checkedOfficialCapturePacketTranscriptManifest`，并在 summary smoke 中覆盖缺失负例；manifest transcript requirement 漂移会让 `usableBaseline=false`。 +- **[Docs Sync]**: official capture guide、handoff、replication、execution plan 和 quality score 同步说明 transcript requirement 是机器可读交接合同，不需要解析 README 或猜测文件存在性。 +- **[Smoke]**: official capture preflight smoke 已覆盖默认 `true`、无 transcript `false`、README 文案和无 transcript child packet manifest 字段。 + +### 🔁 Follow-up (2026-06-27, single no-transcript capture packet) + +- **[Packet Smoke]**: official capture preflight smoke 现在也生成单场景 `--no-include-transcript` packet，确认只创建 `inputs/event_stream_stop-response.json`，不创建 `inputs/mcp-transcript.json`，wrapper 不带 `--mcp-transcript`，`verify-inputs.sh` 替换 status response 后即可通过。 +- **[Baseline Evidence]**: baseline summary 新增 `preflightPipelines.checkedOfficialCapturePacketNoTranscript`，并在 summary smoke 中覆盖缺失负例；单场景 no-transcript packet 漂移会让 `usableBaseline=false`。 +- **[Why]**: required `simple-action-stop` 最可能先用单场景 capture packet 采集；如果官方 hosted transcript 暂时不可得，单场景无 transcript 交接也必须是可复跑、可验证的 baseline 合同。 + +### 🔁 Follow-up (2026-06-27, preflight evidence artifact audit) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在会直接审计 `evidence.preflightPipelines` 里的 official capture packet、no-transcript packet、packet set、transcript manifest、placeholder guard、verify-all、official capture failure path 和 OCU pairing preflight evidence。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增 preflight evidence 篡改负例；即使 summary 的 status 仍声称 usable，只要 official capture / OCU pairing preflight evidence 被改成 false，artifact audit 也会失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone audit 会直接检查 preflight evidence，不只依赖 `status.usableBaseline` 的间接结果。 + +### 🔁 Follow-up (2026-06-27, usable baseline direct artifact audit) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在会直接审计所有 `usableBaseline` 子证据，包括 event-stream matrix、截图上下文、真实输入 action smoke、official fixture set gate、fixture ingest pipeline、source standalone repo 和 npm staged repo evidence。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增 usable baseline direct evidence 篡改负例；即使 summary 的 status 仍声称 usable，只要 action / fixture ingest / standalone / npm 等 evidence 被改成 false，artifact audit 也会失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone audit 复审的是完整 usable baseline evidence，而不是只审 official boundary、preflight 或 Make target evidence。 + +### 🔁 Follow-up (2026-06-27, nextActions command artifact audit) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在会审计落盘 summary 的 `nextActions.commands`，不再只检查 action kind 是否存在。 +- **[Checks]**: 缺 required official golden 时，artifact audit 要求保留 `make record-and-replay-official-golden-capture-packet`、`verify-inputs.sh`、`inspect-only.sh`、`import-fixture.sh` 和 strict gate；缺 recommended golden 时要求保留 packet set Make、`verify-all.sh`、`inspect-all.sh`、`import-all.sh` 和 `ingest-ocu-candidates.sh`；standalone baseline ready 时要求保留 scaffold、生成 repo `scripts/check.sh` 和 optional lifecycle smoke。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增 `nextActions` command 篡改负例，确认删除 packet Make、verify-all 或 generated repo check 命令都会让 artifact audit 失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone audit 会验证下一步命令 handoff，不只验证 status / evidence。 + +### 🔁 Follow-up (2026-06-27, derived status artifact audit) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在会审计 baseline summary 的派生状态不变量。 +- **[Checks]**: `requiresOfficialGoldenCapture` 必须和 official golden complete 互为反向；`officialSuccessfulRecordingEquivalenceReady` 必须等于 usable baseline + official golden complete；`standaloneRepoBaselineReady=true` 不能出现在 `usableBaseline=false` 的 artifact 中；official golden complete 不能同时保留 required scenario gaps、not-ready gaps 或 coverage errors。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增派生状态篡改负例和 golden complete 仍带 gaps 的负例，确认这些状态组合不一致会让 artifact audit 失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone audit 会验证派生状态语义，不只验证 evidence 和 nextActions。 + +### 🔁 Follow-up (2026-06-27, baseline implementation context) + +- **[Baseline Scope]**: handoff 新增 baseline 实施基准，明确第一波目标是“能用且不误报官方等价”：默认 baseline 可以证明 OCU / standalone 可用，但缺 official successful recording golden 时仍必须保持 `officialSuccessfulRecordingEquivalenceReady=false`。 +- **[Contract Sync]**: replication 文档新增 baseline 版本定义，把官方三件套、session 文件、基础事件、AX / screenshot baseline、OCU 自有控制条、wait/notify、validation、summary、skill scaffold 和 standalone repo scaffold 归为第一波必须具备能力。 +- **[Plan Sync]**: active execution plan 补充当前 baseline 推进基准，要求拆独立 repo 前保持默认自检不启动真实录制，真实 lifecycle smoke 继续 opt-in，并要求任何官方等价声明先有 official fixture 与 OCU candidate 对比证据。 + +### 🔁 Follow-up (2026-06-27, declared checks artifact audit) + +- **[Audit CLI]**: `scripts/check-record-and-replay-baseline-summary.py` 现在要求顶层 `checks` 声明包含所有必需 baseline smoke / compare / preflight / standalone 检查。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增删除 `checks` 条目的负例，确认 otherwise valid 的 summary artifact 也会因为检查声明漂移而失败。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、execution plan 和 quality score 同步说明 release / standalone audit 会验证顶层检查清单，不只验证 status、evidence、派生状态和 `nextActions`。 + +### 🔁 Follow-up (2026-06-27, declared checks audit diagnostics) + +- **[Audit CLI]**: 顶层 `checks` 审计失败时，`scripts/check-record-and-replay-baseline-summary.py` 会在 `declaredChecks.missingRequired` 输出缺失的具体 check 名称。 +- **[Smoke]**: declared-checks 篡改负例现在断言缺失清单包含被删除的 `official-no-active-response-compare` 与 `npm-staged-skill-repo-smoke`，让 release / standalone audit 失败信息可直接定位缺口。 +- **[Docs Sync]**: handoff、CICD 和 quality score 同步说明 audit JSON 会携带 missing-required diagnostics。 + +### 🔁 Follow-up (2026-06-27, builder/audit checks consistency) + +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 现在会导入 `scripts/check-record-and-replay-baseline-summary.py`，并断言 builder 产出的顶层 `checks` 与 audit 的 `REQUIRED_BASELINE_CHECKS` 完全一致。 +- **[Why]**: 这条检查用于防止 baseline 检查清单只在生成器或审计器一侧更新，导致 release / standalone audit 消费到看似完整但实际口径漂移的 summary artifact。 +- **[Docs Sync]**: handoff、replication、CICD、execution plan 和 quality score 已同步记录 builder/audit required checks 一致性约束。 + +### 🔁 Follow-up (2026-06-27, canonical declared checks audit) + +- **[Audit CLI]**: 顶层 `checks` 现在不仅要包含全部 required baseline checks，还不能带未知或重复 check 名称；失败时会分别通过 `declaredChecks.missingRequired`、`declaredChecks.unknown` 和 `declaredChecks.duplicates` 输出诊断。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 新增 unknown / duplicate check 篡改负例，确认追加 `obsolete-record-and-replay-smoke` 或重复 `event-stream-smoke-matrix` 都会让 artifact audit 失败。 +- **[Docs Sync]**: handoff、replication、CICD、execution plan 和 quality score 同步说明顶层 `checks` 是 canonical set，不是可扩展备注列表。 + +### 🔁 Follow-up (2026-06-27, shared declared checks contract) + +- **[Contract]**: 新增 `scripts/record_and_replay_baseline_contract.py`，把 baseline summary 顶层 `checks` 的 canonical list 抽成 builder 与 audit 共用的唯一来源。 +- **[Builder / Audit]**: `scripts/build-record-and-replay-baseline-summary.py` 按 shared tuple 顺序输出 `checks`，`scripts/check-record-and-replay-baseline-summary.py` 按同一 tuple/set 复审 missing / unknown / duplicate 声明。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 断言 builder 输出顺序等于 shared contract，并继续覆盖 missing / unknown / duplicate 篡改负例，避免后续只更新生成器或审计器一侧。 +- **[Runner Smoke]**: `scripts/test-record-and-replay-baseline-runner-summary-json.py` 的 fake repo 会同步复制 shared contract 文件，确保 runner 级 `--summary-json` stub 测试覆盖真实 builder/audit 依赖，而不是只复制入口脚本。 +- **[Docs Sync]**: handoff、replication、CICD、execution plan 和 quality score 同步说明 shared contract 是顶层 `checks` 的唯一源码。 + +### 🔁 Follow-up (2026-06-27, generated README scenario list evidence) + +- **[Scaffold]**: standalone thin skill repo README 的 required / recommended official successful recording scenario 清单现在从 `scripts/record_and_replay_scenarios.py` 生成，不再手写 `simple-action-stop` 等列表。 +- **[Smoke Evidence]**: source scaffold smoke 和 npm staged scaffold smoke 都新增 `checkedGeneratedReadmeScenarioList=true`，并断言 README 文案与 scenario catalog 当前清单一致。 +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 把 source / npm staged 的 `checkedGeneratedReadmeScenarioList` 纳入 `usableBaseline` 必需 evidence；summary smoke 增加缺失负例，artifact audit 也会直接复审该 evidence。 +- **[Docs Sync]**: replication 和 quality score 同步说明 README handoff 场景清单也受 scenario catalog 与 baseline summary 约束。 + +### 🔁 Follow-up (2026-06-27, standalone runtime timeout diagnostics) + +- **[Runtime Diagnostics]**: 生成态 `scripts/verify-runtime.py` 和可选 `scripts/recording-lifecycle-smoke.py` 的 MCP response timeout 现在会输出 runtime path、`event-stream mcp` command、`open-computer-use --version` 结果和 `OPEN_COMPUTER_USE_CLI` 修复提示，避免旧全局 runtime 或错误 launcher 只报裸 timeout。 +- **[Smoke Evidence]**: source scaffold smoke 新增 fake stale runtime 负例，用 `OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS=0.2` 快速触发 timeout 并断言错误里包含 fake `0.1.51` 版本和修复提示；npm staged scaffold smoke 静态验证安装态生成物包含同样诊断字段。 +- **[Docs Sync]**: architecture、handoff、replication 和 quality score 同步说明该诊断属于 standalone repo runtime contract 的一部分，后续拆独立 repo 时应优先让 `OPEN_COMPUTER_USE_CLI` 指向当前 runtime 再运行自检。 + +### 🔁 Follow-up (2026-06-27, runtime timeout diagnostics baseline gate) + +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py` 现在把 source standalone 与 npm staged 的 `checkedRuntimeTimeoutDiagnostics` 纳入 `usableBaseline` 必需 evidence；完整 baseline runner 也会在聚合 summary 前先断言这两个 smoke 输出该字段。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 会直接复审 `evidence.standaloneSkillRepo.checkedRuntimeTimeoutDiagnostics` 和 `evidence.npmStagedSkillRepo.checkedRuntimeTimeoutDiagnostics`；summary smoke 与 audit smoke 都新增篡改负例。 +- **[Docs Sync]**: architecture、handoff、replication 和 quality score 同步说明 release / standalone audit 不只验证 runtime 成功路径，也验证旧 runtime / 错误 launcher 的可诊断失败路径。 + +### 🔁 Follow-up (2026-06-27, baseline audit with runtime diagnostics evidence) + +- **[Full Audit]**: 复跑 `make record-and-replay-baseline-audit` 通过，生成 `dist/record-and-replay-baseline-summary.json`；随后直接运行 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-baseline-summary.json` 复审通过。 +- **[Status Evidence]**: 该 artifact 保持 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、source / npm staged `checkedRuntimeTimeoutDiagnostics=true`，同时仍明确 `officialSuccessfulRecordingGoldenComplete=false`、`officialSuccessfulRecordingEquivalenceReady=false`，缺 required `simple-action-stop` official successful recording fixture。 +- **[Boundary]**: 本次验证没有启动官方录制；official successful recording schema、AX compact diff 和 screenshot 触发策略仍必须等 Codex-hosted official golden fixture 校准。 + +### 🔁 Follow-up (2026-06-27, strict gate after runtime diagnostics evidence) + +- **[Strict Audit]**: 复跑 `make record-and-replay-official-golden-gate-audit`，命令按预期非零退出并写出 strict `dist/record-and-replay-baseline-summary.json`。 +- **[Failure Boundary]**: 落盘摘要显示 `strictOfficialGoldenRequired=true`、`usableBaseline=true`、`standaloneRepoBaselineReady=true`、source / npm staged `checkedRuntimeTimeoutDiagnostics=true`，同时 `officialGoldenRequirementSatisfied=false`、`officialGoldenGatePassed=false`、`officialSuccessfulRecordingEquivalenceReady=false`。 +- **[Remaining Gap]**: strict gate 当前只缺 required `simple-action-stop` official successful recording fixture；`notReadyRequiredOfficialSuccessfulRecordingScenarios=[]`、`officialFixtureCoverageErrors=[]`，说明新增 runtime timeout diagnostics gate 没有引入额外失败点。 + +### 🔁 Follow-up (2026-06-27, npm Python launcher diagnostics gate) + +- **[Smoke Evidence]**: npm staged standalone smoke 现在输出 `checkedNpmPythonLauncherDiagnostics=true`，证明安装态 scaffold launcher 在 `PYTHON` 指向不存在的文件或 Python 2 时会返回 command-specific Python 3 诊断和 `PYTHON=/path/to/python3` 修复提示。 +- **[Baseline Gate]**: `scripts/build-record-and-replay-baseline-summary.py`、`scripts/run-record-and-replay-baseline-smoke.sh` 和 `scripts/check-record-and-replay-baseline-summary.py` 都消费该 evidence；缺失时 `usableBaseline=false` 并报告 `npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics`。 +- **[Smoke]**: 复跑 `make record-and-replay-baseline-summary-smoke`、`make record-and-replay-baseline-summary-audit-smoke`、`make npm-record-and-replay-skill-repo-smoke` 和 `make record-and-replay-baseline-audit` 通过，确认 summary/audit 负例、安装态输出和完整 baseline artifact 都覆盖这条合同。 + +### 🔁 Follow-up (2026-06-27, generated README prerequisites evidence) + +- **[Generated README]**: standalone thin skill repo 生成态 README 新增 `Prerequisites` 段，明确自检脚本要求 Python 3 可通过 `python3` 访问，并提示 npm launcher 用户可设置 `PYTHON=/path/to/python3`。 +- **[Evidence Gate]**: source scaffold smoke 和 npm staged scaffold smoke 都输出 `checkedGeneratedReadmePrerequisites=true`；baseline runner、summary 和 artifact audit 把该字段纳入 `standaloneSkillRepo` / `npmStagedSkillRepo` 必需 evidence。 +- **[Docs Sync]**: architecture、handoff、replication、quality score 和 active execution plan 已同步这条独立 repo handoff 约束；它不改变官方三件套 MCP surface，只补齐 standalone repo 用户可见前置条件。 + +### 🔁 Follow-up (2026-06-27, official capture packet handoff scripts) + +- **[Capture Packet]**: 单场景 official capture packet 的 README 和 `capturePacket` JSON 现在显式暴露 `check-fixture-set.sh`、`strict-golden-gate.sh` 和可用时的 `ingest-ocu-candidate.sh`，并返回 `checkFixtureSetShell`、`strictGoldenGateShell`、`ingestOcuCandidateShell` 字段。 +- **[Evidence Gate]**: official golden capture preflight smoke 输出 `checkedCapturePacketHandoffScripts=true`；baseline runner、summary 和 artifact audit 通过 `checkedOfficialCapturePacketHandoffScripts` 消费该 evidence。 +- **[Why]**: 这条约束确保拿到 Codex-hosted official JSON 后，packet 不只指导 inspect/import，也能继续跑 fixture-set gate、strict official golden gate 和 same-scenario OCU candidate 导入。 + +### 🔁 Follow-up (2026-06-27, OCU candidate ingest handoff commands) + +- **[Candidate Handoff]**: `scripts/ingest-ocu-record-and-replay-candidate.py` 成功导入后会输出 `commands.pairingPreflightShell` 和 `commands.fixtureSetGateShell`，指向 same-scenario pairing preflight 与集合级 official-vs-OCU candidate gate。 +- **[Evidence Gate]**: `make event-stream-ocu-candidate-ingest-smoke` 输出 `checkedCandidateIngestHandoffCommands=true`；baseline runner、summary 和 artifact audit 通过 `checkedOcuCandidateIngestHandoffCommands` 消费该 evidence，summary smoke 也新增 `checkedCandidateIngestHandoffCommandsEvidence=true` 和缺失负例。 +- **[Why]**: 这让 OCU candidate 采样/导入后的下一步不再依赖人工拼命令；后续 official fixture 入库或 candidate 重采后，可以直接按输出命令进入 readiness/compare 校准。 + +### 🔁 Follow-up (2026-06-27, capture packet candidate output root) + +- **[Bug]**: official capture packet 的 `ingest-ocu-candidate.sh` 会跟随自定义 `--official-root`，但此前没有显式传 `--output-dir`，导致自定义 fixture root 时 OCU candidate 仍可能落回仓库默认 candidate 目录。 +- **[Fix]**: `scripts/prepare-record-and-replay-official-golden-capture.py` 生成的 candidate command 现在传入 `--output-dir /ocu-candidates`，保证 official fixture 与 same-scenario OCU candidate 在同一 fixture root 下配对。 +- **[Evidence Gate]**: official capture preflight smoke 输出 `checkedCapturePacketOcuCandidateOutputDir=true`；baseline runner、summary 和 artifact audit 通过 `preflightPipelines.checkedOfficialCapturePacketOcuCandidateOutputDir` 消费该 evidence，summary smoke 与 artifact audit smoke 都覆盖篡改负例。 +- **[Docs Sync]**: architecture、official capture guide、replication、execution plan 和 quality score 已同步这条 handoff 约束。 + +### 🔁 Follow-up (2026-06-27, capture packet set candidate handoff) + +- **[Packet Set Handoff]**: recommended official capture packet set 根目录现在生成 `ingest-ocu-candidates.sh`，批量运行存在 `ingest-ocu-candidate.sh` 的子场景，并跳过 keyboard / cancel / timeout 这类不生成 synthetic candidate wrapper 的场景。 +- **[Evidence Gate]**: official capture preflight smoke 输出 `checkedCapturePacketSetOcuCandidateHandoff=true`；baseline runner、summary 和 artifact audit 通过 `preflightPipelines.checkedOfficialCapturePacketSetOcuCandidateHandoff` 消费该 evidence，summary smoke 与 artifact audit smoke 都覆盖篡改负例。 +- **[Why]**: recommended set 采完官方样本后，不再需要人工逐个进入 simple/drag 子目录查找可用 OCU candidate wrapper，后续官方/OCU 配对校准入口更稳定。 +- **[Docs Sync]**: architecture、official capture guide、replication、execution plan 和 quality score 已同步这条批量 handoff 约束。 + +### 🔁 Follow-up (2026-06-27, recommended next action OCU candidate handoff) + +- **[Summary Guidance]**: baseline summary 的 `capture-recommended-official-golden-set` `nextActions` 现在显式包含 `cd && ./ingest-ocu-candidates.sh`，与 recommended packet set 实际生成的根级 helper 保持一致。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 新增 `checkedRecommendedGoldenNextActionIngestOcuCandidatesStep`，release / standalone audit 会拒绝缺少该 OCU candidate handoff 命令的 summary artifact。 +- **[Smoke]**: summary smoke 断言推荐分支命令列表和临时生成的 packet set 都包含 `ingest-ocu-candidates.sh`；summary audit smoke 会篡改删除该命令并要求 audit 失败。 + +### 🔁 Follow-up (2026-06-27, handoff short entry sync) + +- **[Docs Sync]**: `docs/design-docs/record-and-replay-handoff.md` 现在也把 recommended packet set 的根级 `ingest-ocu-candidates.sh` 写入采集步骤、常用命令和 artifact audit 说明。 +- **[Why]**: 后续推进通常先读 handoff 短入口；如果这里仍只写 `verify-all.sh` / `inspect-all.sh` / `import-all.sh`，就会和 baseline summary / artifact audit 的真实合同漂移。 + +### 🔁 Follow-up (2026-06-27, CI audit doc sync) + +- **[Docs Sync]**: `docs/CICD.md` 的 Record & Replay baseline artifact check 现在也声明 summary audit 会检查 recommended packet set 的 `ingest-ocu-candidates.sh`。 +- **[Evidence Contract]**: 同一段落同步说明 official capture preflight smoke 会守住 packet set 根级 `ingest-ocu-candidates.sh`，完整 baseline runner 会把 recommended packet set OCU candidate handoff 写入 `evidence.preflightPipelines`。 + +### 🔁 Follow-up (2026-06-27, check-docs R&R contract) + +- **[Docs Gate]**: `scripts/check-docs.sh` 现在把 Record & Replay handoff / replication / official capture 文档、逆向参考文档、官方 non-recording fixture 文件和 thin skill 入口纳入必要文件检查。 +- **[Handoff Contract]**: 同一脚本还会检查 handoff、official capture、replication 和 CICD 文档继续声明 recommended packet set 的 `ingest-ocu-candidates.sh` handoff 或对应 artifact audit check，避免文档合同再次漂移。 + +### 🔁 Follow-up (2026-06-27, standalone source audit handoff) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `officialEvidence` 新增 `sourceRepoBaselineAudit`，声明默认 baseline audit、strict official golden gate audit 和默认 summary artifact。 +- **[Evidence]**: 源码 scaffold smoke、npm staged scaffold smoke、baseline summary 和 artifact audit 新增 `checkedOfficialEvidenceAuditManifest`，缺失时 `usableBaseline=false`。 +- **[Docs Sync]**: 同步 handoff、replication、architecture、CI/CD 和 quality score，明确 standalone repo handoff 不能丢失源仓 release / audit 证据入口。 +- **[Docs Gate]**: `scripts/check-docs.sh` 现在检查 handoff / replication / architecture / CI 文档继续保留 `sourceRepoBaselineAudit`、baseline audit 命令和 `checkedOfficialEvidenceAuditManifest` 文案。 + +### 🔁 Follow-up (2026-06-27, standalone next action audit step) + +- **[Summary Guidance]**: baseline summary 的 `scaffold-standalone-record-and-replay-repo` `nextActions` 现在先给出 `make record-and-replay-baseline-audit`，再给出 scaffold、生成 repo `scripts/check.sh` 和 optional lifecycle smoke。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 新增 `checkedStandaloneNextActionBaselineAuditCommand`，release / standalone audit 会拒绝缺少 source baseline audit handoff 的 summary artifact。 +- **[Smoke]**: summary smoke 断言 standalone nextActions 包含 baseline audit 命令；summary audit smoke 会篡改删除该命令并要求 audit 失败。 +- **[Docs Gate]**: `scripts/check-docs.sh` 现在要求 CI 文档保留 `checkedStandaloneNextActionBaselineAuditCommand` 文案。 + +### 🔁 Follow-up (2026-06-27, notify suppressed events path handoff) + +- **[Runtime]**: `event-stream wait --notify-command` 成功回调现在会通过 `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` 暴露 `suppressed.jsonl`，让独立监听方在录制结束后能直接读取主事件流之外的降级 / 诊断证据。 +- **[Standalone Contract]**: standalone thin skill repo scaffold 的 manifest、README/thin skill、生成态 `scripts/wait-notify-contract-smoke.py`、source scaffold smoke 和 npm staged scaffold smoke 都同步该环境变量；合成 completed session 的成功 callback 会验证该变量指向存在的 suppressed stream。 +- **[Baseline Gate]**: baseline summary 与 artifact audit 新增 `checkedNotifySuppressedEventsPathEnv` 必需 evidence，summary smoke 和 audit smoke 均覆盖缺失负例。 +- **[Boundary]**: 这仍属于 OCU 扩展层 `wait --notify-command`，不改变官方兼容 `event_stream_start/status/stop` 三件套 MCP surface，也不替代 required `simple-action-stop` official successful recording golden。 + +### 🔁 Follow-up (2026-06-28, strict audit artifact path split) + +- **[Risk]**: `make record-and-replay-official-golden-gate-audit` 原先默认复用 `dist/record-and-replay-baseline-summary.json`；当前缺 required official successful recording fixture 时，strict gate 的预期失败摘要会覆盖刚通过的 baseline audit artifact。 +- **[Fix]**: Makefile 新增 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON ?= dist/record-and-replay-official-golden-gate-summary.json`，strict audit 默认写该独立路径；默认 baseline audit 继续写 `dist/record-and-replay-baseline-summary.json`，仍可用 `RNR_BASELINE_SUMMARY_JSON=` 覆盖。 +- **[Smoke]**: `scripts/test-record-and-replay-baseline-audit-make-targets.py` 现在清理继承的 `RNR_BASELINE_SUMMARY_JSON` 与 `RNR_OFFICIAL_GOLDEN_SUMMARY_JSON`，并用 `make -n` 检查 baseline 默认/自定义路径、strict 默认/自定义路径、strict `--require-official-golden` 和 strict 默认路径不复用 baseline artifact。 +- **[Summary Evidence]**: `scripts/run-record-and-replay-baseline-smoke.sh`、summary builder 和 artifact audit 都消费 `checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPath`、`checkedBaselineAuditStrictOfficialGoldenCustomSummaryPath` 与 `checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath`，缺失时 `usableBaseline=false`。 +- **[Standalone Handoff]**: standalone skill repo manifest 的 `officialEvidence.sourceRepoBaselineAudit` 现在分别声明 `baselineSummaryArtifact` 和 `strictOfficialGoldenSummaryArtifact`；生成态 README、source scaffold smoke 和 npm staged smoke 同步覆盖这两个 artifact handoff。 + +### 🔁 Follow-up (2026-06-28, audit summary variable isolation) + +- **[Smoke]**: audit Make target dry-run smoke 新增双向变量隔离断言：`record-and-replay-baseline-audit RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=` 仍使用 baseline 默认 artifact，`record-and-replay-official-golden-gate-audit RNR_BASELINE_SUMMARY_JSON=` 仍使用 strict 默认 artifact。 +- **[Summary Evidence]**: baseline runner、summary builder 和 artifact audit 新增 `checkedBaselineAuditIgnoresStrictSummaryVar` 与 `checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar`，缺失时 `usableBaseline=false`。 +- **[Why]**: 这补住了只检查默认路径分离但没检查“传错覆盖变量”的缝隙，避免后续 Makefile 重构时又让 strict 失败摘要覆盖 baseline 可用证据。 + +### 🔁 Follow-up (2026-06-28, standalone audit isolation manifest) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `officialEvidence.sourceRepoBaselineAudit` 新增 `auditTargetDryRunSmoke`、`baselineSummaryEnvVar`、`strictOfficialGoldenSummaryEnvVar`、`verifiesSummaryArtifactSeparation` 和 `verifiesSummaryEnvVarIsolation`。 +- **[Smoke]**: source scaffold smoke 与 npm staged scaffold smoke 的 exact manifest 断言同步覆盖这些字段，避免独立 repo handoff 只知道有两个 audit 命令，却不知道源仓还会机械验证 artifact 分离和覆盖变量隔离。 +- **[Docs]**: 架构、handoff、replication、CI 和质量文档同步这份机器可读合同。 + +### 🔁 Follow-up (2026-06-28, baseline audit after isolation manifest) + +- **[Full Audit]**: 在 standalone audit isolation manifest 合同补齐后复跑 `make record-and-replay-baseline-audit` 通过，刷新 `dist/record-and-replay-baseline-summary.json`。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-baseline-summary.json` 复审通过；摘要中 `usableBaseline=true`、`standaloneRepoBaselineReady=true`，并且 `checkedBaselineAuditIgnoresStrictSummaryVar=true` / `checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar=true`。 +- **[Remaining Gap]**: official successful recording golden 仍未完成，required `simple-action-stop` 仍缺失；因此 `officialSuccessfulRecordingGoldenComplete=false`、`officialSuccessfulRecordingEquivalenceReady=false` 是当前正确状态。 + +### 🔁 Follow-up (2026-06-28, strict audit expected-failure evidence) + +- **[Strict Audit]**: 复跑 `make record-and-replay-official-golden-gate-audit`，命令按预期非零退出并刷新 `dist/record-and-replay-official-golden-gate-summary.json`；该 strict artifact 与 `dist/record-and-replay-baseline-summary.json` 不同，未覆盖可用 baseline 证据。 +- **[Expected Failure]**: strict artifact 显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`strictOfficialGoldenRequired=true`、`officialGoldenRequirementSatisfied=false`、`officialGoldenGatePassed=false`，失败原因是 required `simple-action-stop` official successful recording 缺失，`officialFixtureCoverageErrors=[]`。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 新增 `--allow-strict-official-golden-missing`，用于审计 strict gate 缺 required official fixture 时写出的预期失败摘要；该模式要求 baseline / standalone / official non-recording / raw timeout evidence 完整、strict missing gap 可解释且无 coverage errors。 +- **[Validation]**: 已用 `--allow-strict-official-golden-missing` 审计当前 strict artifact 通过，并补了 `scripts/test-record-and-replay-baseline-summary-audit.py` 的正负例，避免后续 release / standalone audit 只能人工区分预期 strict golden 缺口和 baseline 回归。 + +### 🔁 Follow-up (2026-06-28, standalone strict expected-failure audit handoff) + +- **[Standalone Manifest]**: `scripts/scaffold-record-and-replay-skill-repo.py` 生成的 `officialEvidence.sourceRepoBaselineAudit` 新增 `strictOfficialGoldenExpectedFailureAudit`，直接暴露 `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing`。 +- **[Generated README]**: standalone README 的 Official Evidence 段落补充 strict gate 在 official successful recording fixtures 入库前预期失败，并给出 expected-failure audit 命令。 + +### 🔁 Follow-up (2026-06-28, standalone manifest self-check) + +- **[Generated Repo]**: standalone thin skill repo 生成态新增 `scripts/verify-manifest.py`，默认 `scripts/check.sh` 会运行它来校验 `record-and-replay-skill-repo.json` 的机器合同。 +- **[Manifest Contract]**: 生成态 manifest 新增 `checks.manifestContract=scripts/verify-manifest.py`，自检范围覆盖 official evidence、required / recommended scenario、`scenarioRecipes`、`sourceRepoBaselineAudit`、`strictOfficialGoldenExpectedFailureAudit`、OCU `extensionLayer`、官方三件套 no-arg MCP surface 和 recording-to-skill handoff。 +- **[Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline shared contract 和 docs gate 都同步 `checkedManifestContract=true`，避免后续拆 standalone repo 时只验证 runtime / workflow smoke，却漏掉 manifest 自身漂移。 +- **[Docs Sync]**: architecture、handoff、replication、CICD 和 quality score 同步说明该检查属于 standalone 默认自检层，不启动真实录制，也不向官方兼容 MCP surface 增加任何 OCU 扩展。 +- **[Smoke]**: source scaffold smoke 与 npm staged scaffold smoke 的 exact manifest / README 断言同步覆盖该字段，避免未来独立 repo 只知道 strict gate artifact 路径，却不知道如何机器审计“预期缺 official golden”。 +- **[Docs Gate]**: 架构、handoff、replication、CI、质量文档和 `scripts/check-docs.sh` 同步该 manifest 字段。 + +### 🔁 Follow-up (2026-06-28, manifest contract evidence negatives) + +- **[Summary Gate]**: `scripts/test-record-and-replay-baseline-summary.py` 新增 source / npm staged `checkedManifestContract=false` 负例，确认最终 summary 会进入 `usableBaseline=false` 并在 `missingUsableBaselineEvidence` 中分别报告 `standaloneSkillRepo.checkedManifestContract` 与 `npmStagedSkillRepo.checkedManifestContract`。 +- **[Artifact Audit]**: `scripts/test-record-and-replay-baseline-summary-audit.py` 的 direct evidence 篡改负例同步覆盖 `checkedStandaloneSkillRepoCheckedManifestContractEvidence` 和 `checkedNpmStagedSkillRepoCheckedManifestContractEvidence`，确保落盘 artifact 审计也会拒绝缺 manifest self-check evidence 的摘要。 +- **[Docs Sync]**: handoff 的 standalone repo contract 列表补入 `scripts/verify-manifest.py`，明确它是默认 `checks` 的一部分，不启动真实录制。 + +### 🔁 Follow-up (2026-06-28, npm staged manifest verifier negative) + +- **[NPM Staged Smoke]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 现在会直接运行安装态 scaffold 生成 repo 内的 `scripts/verify-manifest.py`，要求输出 `checkedManifestContract=true` 和 `checkedStrictExpectedFailureAudit=true`。 +- **[Negative Guard]**: 同一 smoke 会临时篡改生成态 `record-and-replay-skill-repo.json` 的 `strictOfficialGoldenExpectedFailureAudit`，确认 `verify-manifest.py` 非零退出并报告 `strict expected-failure audit command drifted`，再恢复原 manifest 后继续跑默认 `scripts/check.sh`。 +- **[Why]**: 这让 npm 安装态生成物不只通过 `check.sh` 间接覆盖 manifest self-check，也能证明 standalone repo 自己的 manifest verifier 对关键 source audit handoff 漂移有正负例。 + +### 🔁 Follow-up (2026-06-28, full baseline audit refresh) + +- **[Full Audit]**: 复跑 `make record-and-replay-baseline-audit` 通过，并刷新默认 baseline summary artifact。 +- **[Evidence]**: 本轮 audit 覆盖 baseline contract、event-stream matrix、screenshot context、真实输入 action smoke、官方 1.0.857 surface compare、Codex-hosted no-active response compare、hostless raw timeout fixture、official fixture set gate、official/OCU fixture ingest、capture/pairing preflight、source standalone repo smoke 和 npm staged standalone repo smoke。 +- **[Status]**: 摘要继续显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`、`officialNonRecordingBaselineVerified=true` 和 `officialRawStartTimeoutBoundaryVerified=true`；同时保持 `officialSuccessfulRecordingGoldenComplete=false`、`officialSuccessfulRecordingEquivalenceReady=false`，required `simple-action-stop` official successful recording 仍缺失。 + +### 🔁 Follow-up (2026-06-28, OCU scenario action candidates) + +- **[Simple Candidate]**: 复跑 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop ./scripts/run-event-stream-smoke-tests.sh` 通过；事件流包含 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.click` 和 `session.ended`，并且 `skillReadiness.canCreateDraft`、MCP response shape capture、`skill-creator` handoff 和 generated skill path redaction 均通过。 +- **[Drag Candidate]**: 复跑 `OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=drag-stop ./scripts/run-event-stream-smoke-tests.sh` 通过；事件流包含 `session.started`、`window.changed`、`AX.focusedWindowChanged`、`mouse.drag` 和 `session.ended`，并且同样能生成可交接的 skill 草稿。 +- **[Boundary]**: 这两条只证明 OCU 侧 required / recommended action candidate 可复跑，不替代 official successful recording golden；required `simple-action-stop` official fixture 仍需要通过正常 Codex hosted Record & Replay 流程采集和脱敏入库。 + +### 🔁 Follow-up (2026-06-28, standalone README handoff verifier) + +- **[Generated Repo]**: standalone thin skill repo 生成态新增 `scripts/verify-readme-handoff.py`，默认 `scripts/check.sh` 会运行它来校验 README 的交接合同。 +- **[README Contract]**: verifier 检查官方 evidence fixture、baseline / strict audit 命令、strict 缺 official golden 的 expected-failure audit、required / recommended successful recording 场景、recording-to-skill 命令和 wait/notify 边界仍写在 README 中；该检查不启动真实录制，也不改变官方兼容 MCP surface。 +- **[Evidence]**: `record-and-replay-skill-repo.json` 新增 `checks.readmeHandoffContract=scripts/verify-readme-handoff.py`；source scaffold smoke、npm staged scaffold smoke、shared baseline contract 和 baseline summary builder 都新增 / 消费 `checkedReadmeHandoffContract`、`checkedReadmeOfficialEvidenceHandoff`、`checkedReadmeOfficialGoldenGap` 与 `checkedReadmeWaitNotifyBoundary`。 +- **[Negative Guard]**: source / npm staged scaffold smoke 都会临时篡改生成态 README 的 strict gate handoff，确认 README verifier 非零退出；summary smoke 也新增 README handoff evidence 缺失负例，确保 release / standalone audit 不能遗漏该层文档合同。 + +### 🔁 Follow-up (2026-06-28, official fixture set AX/suppressed strong compare) + +- **[Policy]**: official fixture set gate 继续保持“直接复刻官方可观察行为，再补 OCU 扩展”的方向；同 scenario OCU candidate 不再只比较事件序列、metadata、semantic fields 和 MCP response shape，也默认要求 AX compact diff evidence / marker 与 suppressed event sequence / schema 对齐。 +- **[Implementation]**: `scripts/check-event-stream-official-fixture-set.py` 固定向底层 compare 传入 `--require-ax-diff-evidence`、`--require-same-ax-diff-markers`、`--require-same-suppressed-event-sequence` 和 `--require-same-suppressed-schema`，并在输出中声明 `comparePolicy`。 +- **[Smoke]**: `scripts/test-event-stream-official-fixture-set.py` 新增 official fixture 含 diff / cumulative diff、OCU candidate 缺 diff 的失败负例，要求错误报告包含缺 AX diff payload 与缺 cumulative AX diff payload。 +- **[Summary Gate]**: baseline runner、summary builder 和 artifact audit 新增 `checkedAxDiffComparisonPolicy`、`checkedSuppressedStreamComparisonPolicy` 与 `checkedAxDiffComparisonFailure`，缺失时 `usableBaseline=false`。 +- **[Docs Sync]**: replication / handoff / architecture / execution plan / quality score 同步说明后续 official golden 入库后，fixture set gate 会直接暴露 OCU compact diff 或 suppressed fallback 漂移；当前没有 official successful recording fixture，仍不能宣称官方 successful recording 等价。 + +### 🔁 Follow-up (2026-06-28, standalone fixture-set compare policy manifest) + +- **[Manifest]**: standalone thin skill repo manifest 新增 `officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate.sameScenarioComparePolicy`，机器声明源仓 fixture set gate 会要求 AX diff evidence、AX diff marker、suppressed event sequence 和 suppressed schema。 +- **[Verifier]**: 生成态 `scripts/verify-manifest.py` 会输出 `checkedOfficialFixtureSetComparePolicy=true`，并新增 suppressed schema policy 漂移负例；source scaffold smoke 和 npm staged scaffold smoke 都覆盖该负例。 +- **[Summary Gate]**: shared baseline contract、summary builder 和 artifact audit 新增 / 消费 `checkedOfficialFixtureSetComparePolicyManifest`，缺失时 source 或 npm staged standalone evidence 会让 `usableBaseline=false`。 +- **[Docs Gate]**: handoff、replication、architecture、CI、quality score 和 `scripts/check-docs.sh` 同步该字段，避免后续拆独立 repo 时丢失官方 golden 入库后的 AX / suppressed 强对比边界。 + +### 🔁 Follow-up (2026-06-28, standalone source baseline summary evidence) + +- **[Generated Repo]**: standalone thin skill repo 生成态新增 `scripts/verify-source-baseline-summary.py`，默认 `scripts/check.sh` 会运行它来审计随 scaffold 复制的 `evidence/source-baseline-summary.json`。 +- **[Evidence Projection]**: scaffold 会从源仓 `dist/record-and-replay-baseline-summary.json` 复制一份脱敏投影，只保留 baseline 状态、official fixture set gate policy evidence、source / npm staged standalone repo 关键 evidence，不复制源码或完整运行日志。 +- **[Manifest Contract]**: `officialEvidence.sourceRepoBaselineAudit` 新增 `copiedBaselineSummaryEvidence=evidence/source-baseline-summary.json`，`checks` 新增 `sourceBaselineSummaryEvidence=scripts/verify-source-baseline-summary.py`。 +- **[Summary Gate]**: source scaffold smoke、npm staged scaffold smoke、shared baseline contract、summary builder 和 artifact audit 新增 / 消费 `checkedSourceBaselineSummaryEvidence`；缺失时 source 或 npm staged standalone evidence 会让 `usableBaseline=false`。 +- **[NPM Packaging]**: npm package 在存在 `dist/record-and-replay-baseline-summary.json` 时会携带该 artifact，安装态 scaffold 生成的 repo 也能产生同一份 source baseline summary projection。 +- **[Docs Gate]**: handoff、replication、architecture、CI、quality score 和 `scripts/check-docs.sh` 同步该字段，避免 standalone repo 只保留 audit 命令文案，却不消费已落盘 baseline audit evidence。 + +### 🔁 Follow-up (2026-06-28, npm baseline summary artifact gate) + +- **[Release Gate]**: `scripts/npm/build-packages.mjs` 不再静默跳过缺失的 `dist/record-and-replay-baseline-summary.json`；打包 `open-computer-use` 时必须复制该 artifact，供安装态 scaffold 投影出 `evidence/source-baseline-summary.json`。 +- **[Failure Mode]**: 缺失 artifact 时 npm build 直接失败并输出 `Missing Record & Replay baseline summary artifact`，同时提示先运行 `make record-and-replay-baseline-audit`。 +- **[Negative Guard]**: `scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs` 会临时移走 baseline summary artifact，验证打包失败和诊断文案，再恢复 artifact 继续跑 npm staged scaffold smoke。 +- **[Docs Sync]**: architecture、handoff、replication、CICD、quality score、execution plan 和 `scripts/check-docs.sh` 同步这条发布前置条件，避免后续 release 生成缺 source baseline summary evidence 的 npm 包。 + +### 🔁 Follow-up (2026-06-28, missing golden nextAction strict audit) + +- **[Summary Guidance]**: `scripts/build-record-and-replay-baseline-summary.py` 的 `capture-official-successful-recording-golden` nextAction 现在以 `make record-and-replay-official-golden-gate-audit` 收尾，确保 official fixture 导入后下一步直接刷新 strict summary artifact。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py` 新增 `checkedMissingGoldenNextActionStrictAuditStep`，缺 required official golden 时会复审 nextAction 是否包含 strict audit Make target。 +- **[Why]**: 这把 missing-golden handoff 和此前已建立的 release / standalone audit artifact 口径对齐，避免采到 `simple-action-stop` official fixture 后只跑普通 strict gate、没有留下 `dist/record-and-replay-official-golden-gate-summary.json` 证据。 + +### 🔁 Follow-up (2026-06-28, capture packet semantic guard) + +- **[Capture Packet]**: official capture packet 新增 `capture-contract.json`，记录 scenario、fixture name、预期 action event、预期 `endReason`、status handoff path evidence 和 transcript evidence requirement。 +- **[Wrapper Guard]**: `verify-inputs.sh` 现在除合法 JSON / 非 `_placeholder=true` 外，还要求 hosted status JSON 至少包含 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath` 这类 handoff path evidence；transcript enabled 时还要求 `inputs/mcp-transcript.json` 包含 response-shape 或 transcript evidence。`inspect-only.sh` / `import-fixture.sh` 在执行导入前也会先跑同一语义校验。 +- **[Baseline Evidence]**: official capture preflight smoke 输出 `checkedCapturePacketInputSemanticGuard=true`；baseline runner、summary builder 和 artifact audit 通过 `checkedOfficialCapturePacketInputSemanticGuard` 消费该 evidence，缺失时 `usableBaseline=false`。 + +### 🔁 Follow-up (2026-06-28, capture packet set contract manifest) + +- **[Packet Set Contract]**: recommended capture packet set 的根级 `capture-packets.json` 新增 `captureContracts` / `captureContractPaths`，把每个子 packet 的 `capture-contract.json` 合同内联到集合级 manifest。 +- **[Baseline Evidence]**: official capture preflight smoke 输出 `checkedCapturePacketSetContractManifest=true`；baseline runner、summary builder 和 artifact audit 通过 `checkedOfficialCapturePacketSetContractManifest` 消费该 evidence，缺失时 `usableBaseline=false`。 +- **[Why]**: 后续独立 repo 或 skill 只读根级 manifest 就能获得每个 scenario 的 expected action event、expected `endReason`、handoff path evidence 和 transcript requirement，不需要靠 README 或遍历子目录推断采样合同。 + +### 🔁 Follow-up (2026-06-28, standalone source summary capture packet contract) + +- **[Standalone Projection]**: standalone scaffold 复制的 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketSetContractManifest`，生成态 `scripts/verify-source-baseline-summary.py` 会校验它并输出 `checkedSourceBaselineSummaryCapturePacketSetContractManifest=true`。 +- **[Manifest Contract]**: `record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 现在显式声明 `checkedOfficialCapturePacketInputSemanticGuard` 和 `checkedOfficialCapturePacketSetContractManifest`。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费 `checkedSourceBaselineSummaryCapturePacketSetContractManifest`，缺失时 `usableBaseline=false`。 + +### 🔁 Follow-up (2026-06-28, standalone source summary official state) + +- **[Verifier Contract]**: 生成态 `scripts/verify-source-baseline-summary.py` 现在接受两种 official golden 状态：当前 required `simple-action-stop` gap 明确且 equivalence 未声明，或未来 official golden gate / successful recording equivalence 已完成且 required gaps、not-ready gaps 和 coverage errors 为空。 +- **[Negative Guard]**: source scaffold smoke 和 npm staged scaffold smoke 都新增矛盾状态负例：仍缺 required scenario 却把 `officialSuccessfulRecordingEquivalenceReady` 置为 `true` 时，verifier 必须非零退出。 +- **[Why]**: 这样当前 standalone repo 仍不会误报官方等价，同时未来 official golden 入库后，独立 repo 的 source summary self-check 不会被旧的 missing-golden 合同卡住。 + +### 🔁 Follow-up (2026-06-28, baseline artifact consumes source summary official state) + +- **[Baseline Evidence]**: source scaffold smoke 和 npm staged scaffold smoke 现在都把 `checkedSourceBaselineSummaryOfficialGoldenState=true` 输出到最终 JSON。 +- **[Summary Gate]**: shared baseline contract、summary builder 和 artifact audit 都消费 `checkedSourceBaselineSummaryOfficialGoldenState`；缺失时 `usableBaseline=false`，并通过具体 `standaloneSkillRepo.*` 或 `npmStagedSkillRepo.*` evidence 名称报告。 +- **[Why]**: 这把生成态 `verify-source-baseline-summary.py` 的 official golden 状态自洽检查提升到 release / standalone audit artifact，而不是只停留在单独 scaffold smoke 里。 + +### 🔁 Follow-up (2026-06-28, capture packet strict audit handoff) + +- **[Capture Packet]**: official capture preflight 的 `commands.strictOfficialGoldenGate` 现在指向 `make record-and-replay-official-golden-gate-audit`，生成的 `strict-golden-gate.sh` wrapper 也会运行 audit target，而不是只跑不落盘的 strict gate。 +- **[Docs Gate]**: official golden capture 文档和 handoff 文档同步说明 `strict-golden-gate.sh` 会刷新 strict summary artifact；`scripts/check-docs.sh` 新增针脚要求官方采集入口保留 `record-and-replay-official-golden-gate-audit`。 +- **[Why]**: 采到 `simple-action-stop` official successful recording 后，后续流程应直接留下 `dist/record-and-replay-official-golden-gate-summary.json` 证据，避免 release / standalone audit 还要人工补跑一次。 + +### 🔁 Follow-up (2026-06-28, capture packet strict audit baseline evidence) + +- **[Evidence]**: official capture preflight smoke 新增 `checkedCapturePacketStrictAuditHandoff=true`，证明生成的 `strict-golden-gate.sh` wrapper 指向 `make record-and-replay-official-golden-gate-audit`。 +- **[Summary Gate]**: baseline summary 投影为 `preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff`，artifact audit 通过 `checkedOfficialCapturePacketStrictAuditHandoffEvidence` 复审；缺失时 `usableBaseline=false`。 +- **[Why]**: 这把上一条 handoff 行为从单脚本测试提升为 release / standalone audit artifact 的必需 evidence，防止采样包后续回退到只跑普通 strict gate。 + +### 🔁 Follow-up (2026-06-28, standalone source summary strict audit handoff) + +- **[Standalone Projection]**: standalone scaffold 复制的 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff`，生成态 `scripts/verify-source-baseline-summary.py` 会校验它并输出 `checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff=true`。 +- **[Manifest Contract]**: `record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 现在显式声明 `checkedOfficialCapturePacketStrictAuditHandoff`。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费 `checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff`，缺失时 `usableBaseline=false`。 +- **[Why]**: 独立 repo 后续只读 source baseline summary 也能证明 official fixture 入库后会刷新 strict summary artifact，而不是只继承源仓聊天或 README 里的操作提示。 + +### 🔁 Follow-up (2026-06-28, README artifact-first R&R commands) + +- **[User Entry]**: README / README.zh-CN 的 Record & Replay 命令清单现在显式推荐 `make record-and-replay-baseline-audit` 作为 release / standalone 留证入口，而不只展示 `record-and-replay-baseline-smoke`。 +- **[Capture Handoff]**: README / README.zh-CN 增加 `make record-and-replay-official-golden-capture-packet` 与 `record-and-replay-official-golden-capture-packet-set` 示例，要求替换 hosted JSON 占位后按 `verify` / `inspect` / `import` / strict artifact wrapper 顺序执行。 +- **[Strict Artifact]**: README / README.zh-CN 增加 `make record-and-replay-official-golden-gate-audit` 和 `scripts/check-record-and-replay-baseline-summary.py ... --allow-strict-official-golden-missing`，让缺 required official golden 时的 strict 失败也有可解释 artifact。 +- **[Docs Gate]**: `scripts/check-docs.sh` 新增 README 针脚，避免公开入口退回只跑不落盘的 smoke / strict gate。 + +### 🔁 Follow-up (2026-06-28, capture packet strict expected-failure audit) + +- **[Capture Packet]**: official capture packet 和 recommended packet set 现在都会生成 `strict-expected-failure-audit.sh`，用于审计已落盘 strict summary 的 `--allow-strict-official-golden-missing` 口径。 +- **[Baseline Evidence]**: official capture preflight smoke 输出 `checkedCapturePacketStrictExpectedFailureAuditHandoff=true`，baseline summary 投影为 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，artifact audit 通过 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence` 复审；缺失时 `usableBaseline=false`。 +- **[Docs Gate]**: README、official capture、handoff、replication、CI、quality score、execution plan 和 `scripts/check-docs.sh` 同步这条 wrapper / evidence，避免 required official golden 尚缺时只能人工判断 strict 失败摘要是否预期。 + +### 🔁 Follow-up (2026-06-28, standalone source summary strict expected-failure handoff) + +- **[Standalone Projection]**: standalone scaffold 复制的 `evidence/source-baseline-summary.json` 现在也保留 `preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`，生成态 `scripts/verify-source-baseline-summary.py` 会校验它并输出 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`。 +- **[Manifest Contract]**: `record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 同步声明 `checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff`。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff`，缺失时 `usableBaseline=false`。 +- **[Why]**: 独立 repo 后续即使只读取脱敏 source baseline summary，也能证明 required official golden 尚缺时的 strict expected-failure audit handoff 没有在 scaffold / npm 发布链路里丢失。 + +### 🔁 Follow-up (2026-06-28, refreshed baseline artifacts after strict expected-failure handoff) + +- **[Baseline Artifact]**: 复跑 `make record-and-replay-baseline-audit`，刷新 `dist/record-and-replay-baseline-summary.json`。默认 summary 现在显示 `usableBaseline=true`、`standaloneRepoBaselineReady=true`，并且 source / npm staged standalone evidence 都包含 `checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff=true`。 +- **[Strict Artifact]**: 复跑 `make record-and-replay-official-golden-gate-audit`，命令按预期非零退出并刷新 `dist/record-and-replay-official-golden-gate-summary.json`。失败仍只来自 required `simple-action-stop` official successful recording fixture 尚未入库。 +- **[Artifact Audit]**: `scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-baseline-summary.json` 通过；`scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing` 也通过，证明 strict expected-failure artifact 处于可解释缺口状态。 + +### 🔁 Follow-up (2026-06-28, official capture post workflow contract) + +- **[Capture Contract]**: official capture packet 的 `capture-contract.json` 和返回 JSON 新增 `postCaptureWorkflow`，把录完官方样本后的替换 hosted JSON、verify、inspect、import、coverage / fixture-set gate、可选 OCU candidate ingest、strict audit 和 strict expected-failure audit 固化成机器可读步骤。 +- **[Packet Set Contract]**: recommended packet set 的根级 `capture-packets.json` 新增按 scenario 分组的 `postCaptureWorkflow`，让独立 repo、skill 或批量采集脚本可以只读根 manifest 获取录制后的有序 handoff。 +- **[Evidence]**: official capture preflight smoke、baseline runner、summary builder 和 artifact audit 现在消费 `checkedOfficialCapturePacketPostCaptureWorkflow` 与 `checkedOfficialCapturePacketSetPostCaptureWorkflow`，缺失时 `usableBaseline=false`，避免后续仍要从 README 或聊天上下文拼采样后流程。 + +### 🔁 Follow-up (2026-06-28, standalone source summary post workflow evidence) + +- **[Standalone Projection]**: standalone scaffold 复制的 `evidence/source-baseline-summary.json` 现在保留 `preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow` 与 `preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow`，生成态 `scripts/verify-source-baseline-summary.py` 会校验它们并输出 `checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow=true` 和 `checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow=true`。 +- **[Manifest Contract]**: `record-and-replay-skill-repo.json` 的 `officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence` 同步声明 `checkedOfficialCapturePacketPostCaptureWorkflow` 与 `checkedOfficialCapturePacketSetPostCaptureWorkflow`。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费这两个 source summary workflow evidence，缺失时 `usableBaseline=false`，避免后续拆独立 repo 时丢失采样包录制后的有序 handoff。 + +### 🔁 Follow-up (2026-06-28, standalone package artifact verifier) + +- **[Generated Repo]**: standalone scaffold 新增 `scripts/verify-package-artifact.py`，默认 `scripts/check.sh` 会在 `scripts/package-skill.sh` 后运行它。 +- **[Artifact Gate]**: verifier 会打开 `.skill` archive，确认 `.skill` alias 与 zip 字节一致、archive 路径只在预期 skill 目录下、packaged `SKILL.md` frontmatter 和 Record & Replay handoff 片段仍存在。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder 和 artifact audit 都消费 `checkedPackageArtifact=true`，缺失时 `usableBaseline=false`，避免独立 repo 只证明 `.skill` 文件存在却没有证明制品内容。 + +### 🔁 Follow-up (2026-06-28, official capture workflow verifier) + +- **[Capture Packet]**: official capture packet 和 recommended packet set 新增 `verify-workflow.sh`，不读取 hosted JSON，只校验 `postCaptureWorkflow`、input 文件、wrapper command、可执行 bit 和 transcript 开关一致。 +- **[Baseline Evidence]**: official capture preflight smoke 输出 `checkedCapturePacketWorkflowVerifier=true` / `checkedCapturePacketSetWorkflowVerifier=true`；baseline summary 与 artifact audit 消费 `checkedOfficialCapturePacketWorkflowVerifier` / `checkedOfficialCapturePacketSetWorkflowVerifier`。 +- **[Standalone Handoff]**: generated standalone repo 的 `evidence/source-baseline-summary.json` 继承 workflow verifier evidence，并由 `checkedSourceBaselineSummaryCapturePacketWorkflowVerifier` / `checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier` 守住，避免拆独立 repo 时丢失官方采样包自身的执行计划校验。 + +### 🔁 Follow-up (2026-06-28, hosted official capture finalizer) + +- **[Official Fidelity]**: official successful recording 的 MCP transcript gate 对齐官方 skill 流程，要求 `event_stream_start`、active `event_stream_status`、`event_stream_stop` 和 final `event_stream_status` response shape，不再把额外 `repeatStartResponseShape` 当作 Codex-hosted official capture 的必需证据。 +- **[Finalizer]**: 新增 `scripts/finalize-record-and-replay-official-capture-packet.py`，可把 Codex-hosted start/status/stop/final-status JSON 写入 capture packet inputs，并生成 readiness 可消费的 `mcp-transcript.json`。 +- **[Smoke]**: 新增 `make record-and-replay-official-capture-finalizer-smoke` / `scripts/test-record-and-replay-official-capture-finalizer.py`，用合成 hosted JSON 验证 finalizer 会跑通 `verify-workflow.sh` / `verify-inputs.sh`，并覆盖缺 final status 与缺 handoff path 的失败路径。 + +### 🔁 Follow-up (2026-06-29, hosted official completed session observation) + +- **[Safety]**: 经用户授权启动 Codex-hosted Record & Replay 后，用户通过官方录制控制结束；随后确认 `event_stream_status` / `event_stream_stop` 都显示 completed，`endReason=recording_controls_stopped`，没有 active recorder 留在后台。 +- **[Observation]**: 该 completed session 约 5 分钟、434 行事件，事件使用顶层 `kind` 字段，观察到 lifecycle、window、selection、mouse click / context menu / drag、keyboard text / submit / shortcut 等事件，以及 AX `fullTree` / `diffFromPrevious` 结构。 +- **[Boundary]**: 该样本捕获的是多应用真实工作流和私有可见内容，不是预期的低风险 `simple-action-stop` 单击场景；因此只把脱敏 schema 事实写入逆向参考，不入库、不导入 fixture、不作为 official successful recording golden。 + +### 🔁 Follow-up (2026-07-01, event kind compatibility baseline) + +- **[Implementation]**: OCU event stream 写入主事件和 suppressed 事件前会规范化 payload：已有 `type` 时补同值 `kind`，只有 `kind` 时补回兼容 `type`。这让新产物贴近 Codex-hosted 官方 completed session 的顶层 `kind` 观察，同时不破坏既有 OCU 工具链。 +- **[Consumer Compatibility]**: Swift validator / summarizer 和 Python `validate` / `summarize` / fixture import / golden readiness / recording compare 脚本都改为用 `kind` 优先、`type` fallback 识别事件名，后续 official `kind`-only 样本可以直接进入校验、摘要、导入和同场景对比链路。 +- **[Validation]**: 已跑 Python 语法编译、`scripts/test-event-stream-golden-readiness.py`、fixture import、recording compare、skill scaffold smoke、`kind`-only validate / summarize 临时探针，以及 Swift `testEventStreamServiceStopsAtMaximumDuration` / `testEventStreamSuppressedEventsUpdateMetadata`。 + +### 🔁 Follow-up (2026-07-01, hosted-style MCP handoff response) + +- **[MCP Response]**: `event_stream_start` / `event_stream_status` / `event_stream_stop` 的官方兼容 MCP text JSON 现在在 active / completed 状态下补 `sessionDirectoryPath`，并把对外 `metadataPath` 指向 `session.json`，同时保留 `sessionID`、`sessionId`、`sessionPath`、`eventsPath`、`isRecording` 和 `maxDurationSeconds`，贴近 2026-06-29 Codex-hosted completed session 观察。 +- **[Boundary]**: 这次只调整官方兼容 MCP surface；OCU CLI `event-stream start/status/stop --json`、`wait --notify-command` 和 strict validator 仍继续使用完整 OCU metadata dictionary，避免破坏 wait/notify 扩展和本地 recording-to-skill 结构校验。 +- **[Validation]**: 已跑 `swift test --filter OpenComputerUseKitTests/testEventStreamMCPServerListsRecordAndReplayTools` 和默认 `scripts/run-event-stream-smoke-tests.sh`，确认 start/status/stop handoff path、runtime validator 和 summarizer 仍能通过。 + +### 🔁 Follow-up (2026-07-01, official-style session handoff file) + +- **[File Contract]**: OCU 录制目录里的 `metadata.json` 继续写完整 OCU status，用于 wait、strict validation、recording-to-skill 和本地工具链；`session.json` 改为官方风格最小 handoff，只写 `id`、`startedAt`、结束后的 `endedAt` / `endReason` 和 `eventsPath`，贴近 2026-06-29 Codex-hosted completed session 观察。 +- **[Validator Compatibility]**: Swift runtime validator、源码 Python validator 和 golden readiness gate 不再要求 `metadata.json` / `session.json` 字节相等，而是要求 session id、`eventsPath`、`startedAt` 和结束字段语义兼容；直接传入 `session.json` 且同目录有 `metadata.json` 时，strict 校验会优先读取完整 metadata，同时仍检查 handoff 文件兼容性。 +- **[Validation]**: 已跑 Python 编译、`scripts/test-event-stream-golden-readiness.py`、Swift `testEventStreamServiceWritesSessionFilesAndStops` / `testEventStreamRecordingValidationAcceptsOfficialStyleSessionHandoffAlias`，以及默认 `scripts/run-event-stream-smoke-tests.sh`。 + +### 🔁 Follow-up (2026-07-01, official-compatible AX fullTree payload) + +- **[AX Payload]**: `AX.focusedWindowChanged` 的 full `accessibilityInspectorPayload` 现在双写官方观察到的 `fullTree` 和 OCU 既有 `treeLines`；diff payload 继续保留 `diffFromPrevious=true`、previous diff 和 cumulative diff baseline，避免把 compact diff 伪装成 full tree。 +- **[Fixture Tooling]**: golden readiness 输出新增 `hasFullTreePayload`，recording compare 的 semantic paths / AX evidence 纳入 `accessibilityInspectorPayload.fullTree[]`，fixture importer 把 `fullTree` 作为敏感 AX 文本脱敏。 +- **[Validation]**: 已跑 Python 编译、`scripts/test-event-stream-golden-readiness.py`、`scripts/test-event-stream-recording-compare.py`、`scripts/test-event-stream-fixture-import.py` 和 Swift `testEventStreamServiceWritesAXFullDiffAndCumulativePayloads`。 + +### 🔁 Follow-up (2026-07-01, official minimal session fixture import) + +- **[Import Compatibility]**: `scripts/import-event-stream-fixture.py` 现在可直接导入 official hosted 最小 recording 目录，也就是只有 `events.jsonl` 和官方风格 `session.json` 的产物；导入时会把 `session.json.id` 当作 session id 脱敏，并生成 fixture 侧完整 `metadata.json`。 +- **[Fixture Normalization]**: 导入后的 fixture 固定写 `metadata.json`、官方风格 `session.json`、`events.jsonl` 和空 `suppressed.jsonl`，并补齐 event / suppressed counts 与四个相对 handoff path，方便 readiness / compare gate 不依赖手工补文件。 +- **[Validation]**: `scripts/test-event-stream-fixture-import.py` 新增 official-minimal 样本，覆盖 `kind`-only 事件、AX `fullTree` 脱敏、空 suppressed 文件、metadata count 和 handoff path readiness。 + +### 🔁 Follow-up (2026-07-01, official capture packet minimal handoff guard) + +- **[Capture Contract]**: official capture packet 的 status JSON 语义校验不再要求 hosted response 自带 `suppressedEventsPath`。现在 required handoff evidence 是 `eventsPath` 加 `metadataPath` 或 `sessionPath`；`suppressedEventsPath` / `sessionDirectoryPath` 只作为 optional evidence 记录。 +- **[Reasoning]**: 这与 Codex-hosted completed session 观察一致：官方原始目录可能只有 `events.jsonl` 和最小 `session.json`，fixture import 会在脱敏入库时生成空 `suppressed.jsonl` 和完整 fixture metadata。 +- **[Validation]**: official capture preflight smoke 和 hosted capture finalizer smoke 覆盖无 `suppressedEventsPath` 的 hosted JSON 仍可通过 `verify-inputs.sh`，同时 no-handoff JSON 仍会失败。 + +### 🔁 Follow-up (2026-07-01, official sessionDirectoryPath ingest) + +- **[Ingest Compatibility]**: `scripts/ingest-official-record-and-replay-fixture.py` 现在会从 hosted status JSON 递归提取 `sessionDirectoryPath`，并在没有 `metadataPath` / `sessionPath` / `eventsPath` 时把该目录作为 recording input。 +- **[Official Minimal Package]**: 这条路径会继续复用 fixture import 的 official minimal session 支持，读取目录内 `events.jsonl + session.json`，并为脱敏 fixture 生成完整 `metadata.json`、官方风格 `session.json` 和空 `suppressed.jsonl`。 +- **[Validation]**: `scripts/test-official-record-and-replay-fixture-ingest.py` 新增 hosted JSON 只含 `sessionDirectoryPath` 的样本，并输出 `checkedOfficialSessionDirectoryPathHandoff=true`；`scripts/run-record-and-replay-baseline-smoke.sh`、summary builder 和 summary artifact audit 也消费 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff`，缺失时 `usableBaseline=false`。该补丁不放宽 capture packet strict semantic guard，packet 仍要求 `eventsPath` 加 `metadataPath` 或 `sessionPath`。 + +### 🔁 Follow-up (2026-07-01, standalone sessionDirectoryPath source evidence) + +- **[Standalone Manifest]**: `record-and-replay-skill-repo.json` 现在在 `officialEvidence.sourceRepoBaselineChecks.officialFixtureIngest` 中声明 `checkedOfficialSessionDirectoryPathHandoff` 是源仓 baseline evidence，避免独立 repo 只知道 capture packet / fixture set / pairing preflight。 +- **[Source Summary Projection]**: `evidence/source-baseline-summary.json` 会保留 `fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff`，生成态 `scripts/verify-source-baseline-summary.py` 校验该字段并输出 `checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff=true`。 +- **[Baseline Evidence]**: source scaffold smoke、npm staged scaffold smoke、baseline summary builder、shared baseline contract 和 artifact audit 都消费这条 source summary evidence；缺失时 `usableBaseline=false`，防止拆 standalone repo 时丢掉 official completed response 只给 `sessionDirectoryPath` 的导入证据。 + +### 🔁 Follow-up (2026-07-01, required official fixture ready) + +- **[Official Fixture]**: 通过 Codex-hosted Record & Replay 采集并脱敏入库 required `simple-action-stop` official successful recording fixture，保存在 `docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/`。 +- **[Gate Status]**: 复核 golden readiness、coverage `--require-readiness` 和 `make record-and-replay-official-golden-fixture-gate`，确认 required official fixture 已 ready；recommended `keyboard-input-stop`、`drag-stop`、`cancel`、`timeout` 仍是后续采样缺口。 +- **[Standalone Contract]**: `scripts/scaffold-record-and-replay-skill-repo.py` 不再硬编码 `officialEvidence.hasSuccessfulRecordingGolden=false`，改为从 source baseline summary 派生，并让生成态 manifest verifier 校验该字段与 copied `evidence/source-baseline-summary.json` 一致。 +- **[Compatibility]**: Record & Replay Python helper 补齐 future annotations，避免 macOS system Python 3.9 直接运行 capture packet / ingest / readiness 脚本时因 PEP 604 annotation 解析失败。 +- **[Docs Sync]**: 更新 README、中文 README、handoff、replication、official capture checklist、architecture、reverse-engineering reference、references index 和 quality score，把状态从 required missing 调整为 required ready / recommended pending。 + +### 🔁 Follow-up (2026-07-01, action smoke scroll best-effort) + +- **[Gate Policy]**: 默认 `mixed-action-stop` 真实输入 smoke 的 release hard gate 收敛为 `mouse.click`、AX context、MCP transcript 和 recording-to-skill handoff；外部 Swift action 仍发送 best-effort scroll wheel。 +- **[Reasoning]**: 官方尚未确认独立 scroll event 名称，且 macOS 合成 scroll wheel 在部分环境会被 session event tap 过滤。捕获到的 `experimentalRawEvents` / `reason=scrollWheel` 仍进入 summary/scaffold 并渲染 Scroll replay step，但不再决定 official golden gate 是否通过。 +- **[Validation Target]**: 后续 strict audit 应以 click/AX/MCP/skill handoff 作为默认 action evidence；scroll raw-only 只作为额外可用证据和推荐校准项。 diff --git a/docs/histories/2026-07/20260701-1825-record-and-replay-official-golden.md b/docs/histories/2026-07/20260701-1825-record-and-replay-official-golden.md new file mode 100644 index 00000000..01d33be0 --- /dev/null +++ b/docs/histories/2026-07/20260701-1825-record-and-replay-official-golden.md @@ -0,0 +1,31 @@ +## [2026-07-01 18:25] | Task: Record & Replay official golden baseline + +### Execution Context +* **Agent ID**: `Codex` +* **Base Model**: `GPT-5` +* **Runtime**: `macOS local shell + Codex Record & Replay hosted MCP` + +### User Query +> 复刻官方 Record & Replay，并基于官方行为推进 OCU baseline；允许调用官方录制，但完成后要确保停止，不要留下后台录制。 + +### Changes Overview +**Scope:** Record & Replay fixture/readiness/capture-packet/docs + +**Key Actions:** +- **Official fixture**: Imported the required `simple-action-stop` official successful recording fixture after hosted capture and redaction. +- **Readiness compatibility**: Updated golden readiness and recording compare helpers to treat official top-level `ax.mode=fullTree` payloads as full AX evidence. +- **Candidate handoff**: Changed generated OCU candidate wrappers to import candidates without making same-scenario strong compare part of the required official golden gate. +- **Docs**: Updated Record & Replay design/capture docs and README commands to distinguish required official fixture readiness from optional candidate calibration. + +### Design Intent (Why) +The first official hosted recording showed that official sessions may include extra host/control events compared with synthetic OCU candidate recordings. The required official golden gate should therefore prove that the official fixture exists and passes scenario readiness, while OCU candidate comparison remains a calibration tool for finding gaps. + +### Files Modified +- `scripts/check-event-stream-golden-readiness.py` +- `scripts/compare-event-stream-recordings.py` +- `scripts/prepare-record-and-replay-official-golden-capture.py` +- `docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/` +- `docs/design-docs/record-and-replay-official-golden-capture.md` +- `docs/design-docs/record-and-replay-replication.md` +- `README.md` +- `README.zh-CN.md` diff --git a/docs/references/README.md b/docs/references/README.md index 4291d100..6b297b32 100644 --- a/docs/references/README.md +++ b/docs/references/README.md @@ -15,6 +15,7 @@ - `codex-computer-use-reverse-engineering/` - 官方 `Codex Computer Use.app` / `SkyComputerUseClient` 的持续逆向分析资料；大体积一次性分析产物默认在本地 `research/` 下重新生成，不提交进仓库。 + - Record & Replay 相关入口是 `codex-computer-use-reverse-engineering/record-and-replay-event-stream.md`，记录官方 `record-and-replay` 1.0.857 的插件包装、Codex.app asar 分工、Computer Use runtime 字符串、event-stream MCP surface、no-active / raw-timeout fixture、required `simple-action-stop` successful recording golden fixture，以及 recommended golden 场景的后续缺口。 - `codex-network-capture.md` - 用 `mitmdump` + `scripts/codex_dump.py` 抓 Codex 上游 HTTP / WebSocket 流量，并把对应 `session_id` 的本地 `function_call` / `function_call_output` 摘要一起沉淀到 `artifacts/codex-dumps/` 做持续分析。 - `codex-local-runtime-logs.md` diff --git a/docs/references/codex-computer-use-reverse-engineering/README.md b/docs/references/codex-computer-use-reverse-engineering/README.md index 45f09c6b..97be52c0 100644 --- a/docs/references/codex-computer-use-reverse-engineering/README.md +++ b/docs/references/codex-computer-use-reverse-engineering/README.md @@ -10,6 +10,16 @@ - 当前已观察到的运行时行为、宿主依赖、Inspector 直连失败现象和共享状态线索。 - `packaging-and-lifecycle-integration.md` - 当前已确认的插件打包结构、主 app 分发形态、CLI surface、`turn-ended` 生命周期接入方式。 +- `record-and-replay-event-stream.md` + - 对官方 `record-and-replay` 1.0.857 的插件包装、`event-stream mcp` surface、Codex.app asar 分工、录制控制条字符串、事件文件和 AX compact diff 线索的整理。 +- `fixtures/record-and-replay-event-stream-surface-1.0.857.json` + - 官方 Record & Replay `event-stream mcp` 的 normalized `initialize` / `tools/list` 非录制输出，用于对比 OCU surface 漂移。 +- `fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json` + - 官方 Record & Replay raw `event_stream_start` / `event_stream_status` / `event_stream_stop` 在宿主外超时的 sanitized transcript shape，用于记录当前采样边界；不含本机路径。 +- `fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json` + - 通过 Codex-hosted Record & Replay tool 观察到的 no-active `event_stream_status` / `event_stream_stop` text JSON shape；不启动录制，不含本机路径。 +- `../../../scripts/probe-event-stream-recording.py` + - 可复跑探测脚本：支持 local OCU 或官方 bundled Record & Replay client，记录 initialize/tools/start/status/stop transcript，处理 `elicitation/create`，并在显式 `--start-stop` 时尝试短录制、status 和自动 stop；官方 start/status/stop 当前仍用于边界观测，不进默认 CI。 - `internal-ipc-surface.md` - 当前已确认的 client-service 内部 IPC 类型、sender authorization、skyshot 模型和 service 生命周期线索。 - `tool-call-samples-2026-04-17.md` diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json new file mode 100644 index 00000000..19b396bf --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json @@ -0,0 +1,64 @@ +{ + "capturedAt": "2026-06-26", + "captureScope": "initialize and tools/list only; no recording was started", + "officialPluginVersion": "record-and-replay 1.0.857", + "initialize": { + "capabilities": { + "tools": { + "listChanged": false + } + }, + "protocolVersion": "2025-11-25", + "serverInfo": { + "name": "Record & Replay", + "version": "bb9a4422c62d1302e92ee60153d20c362229d5e0fa4731c4b9e90d969b245e7c" + } + }, + "tools": [ + { + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": false + }, + "description": "Start recording the user's actions for up to 30 minutes. If a recording is already active, return that active session instead of starting another one.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_start" + }, + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "description": "Get the current or most recent Record & Replay recording status including paths to metadata and events during the recording.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_status" + }, + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "description": "Stop the active event stream recording if one is running and return status including paths to metadata and events during the recording.", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_stop" + } + ] +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json new file mode 100644 index 00000000..82a1362a --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json @@ -0,0 +1,29 @@ +{ + "capturedAt": "2026-06-27", + "captureScope": "Codex-hosted Record & Replay tool calls with no active recording; no recording was started", + "officialPluginVersion": "record-and-replay 1.0.857", + "toolResponses": { + "event_stream_status": { + "content": [ + { + "textJSON": { + "isRecording": false, + "maxDurationSeconds": 1800 + }, + "type": "text" + } + ] + }, + "event_stream_stop": { + "content": [ + { + "textJSON": { + "isRecording": false, + "maxDurationSeconds": 1800 + }, + "type": "text" + } + ] + } + } +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json new file mode 100644 index 00000000..5853257f --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json @@ -0,0 +1,138 @@ +{ + "elicitationRequests": [], + "finalStatusResponseShape": null, + "fixtureFormatVersion": 1, + "kind": "record-and-replay-event-stream-probe", + "officialPluginVersion": "1.0.857", + "ok": true, + "process": { + "returncode": 0, + "stderr": "" + }, + "protocolVersion": "2025-11-25", + "repeatStartResponseShape": null, + "repeatStopResponseShape": null, + "serverInfo": { + "name": "Record & Replay", + "version": "bb9a4422c62d1302e92ee60153d20c362229d5e0fa4731c4b9e90d969b245e7c" + }, + "source": "official", + "startResponseShape": { + "timeout": 10.0 + }, + "startStopCompleted": false, + "startStopRequested": true, + "startTimedOut": true, + "statusResponseShape": { + "timeout": 10.0 + }, + "stopResponseShape": { + "timeout": 10.0 + }, + "toolNames": [ + "event_stream_start", + "event_stream_status", + "event_stream_stop" + ], + "transcriptShape": [ + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": 1, + "method": "initialize", + "timeout": null + }, + { + "direction": "receive", + "eof": null, + "hasError": false, + "hasResult": true, + "id": 1, + "method": null, + "timeout": null + }, + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": null, + "method": "notifications/initialized", + "timeout": null + }, + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": 2, + "method": "tools/list", + "timeout": null + }, + { + "direction": "receive", + "eof": null, + "hasError": false, + "hasResult": true, + "id": 2, + "method": null, + "timeout": null + }, + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": 3, + "method": "tools/call", + "timeout": null + }, + { + "direction": "receive", + "eof": null, + "hasError": false, + "hasResult": false, + "id": null, + "method": null, + "timeout": 10.0 + }, + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": 4, + "method": "tools/call", + "timeout": null + }, + { + "direction": "receive", + "eof": null, + "hasError": false, + "hasResult": false, + "id": null, + "method": null, + "timeout": 10.0 + }, + { + "direction": "send", + "eof": null, + "hasError": false, + "hasResult": false, + "id": 5, + "method": "tools/call", + "timeout": null + }, + { + "direction": "receive", + "eof": null, + "hasError": false, + "hasResult": false, + "id": null, + "method": null, + "timeout": 10.0 + } + ] +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/README.md b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/README.md new file mode 100644 index 00000000..cfd4dc0c --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/README.md @@ -0,0 +1,10 @@ +# ocu-simple-action-stop + +This is a sanitized Record & Replay event-stream fixture. + +- Source: `ocu` +- Screenshots are not copied. +- User text, window titles, timestamps, session ids, local paths, and app attribution are redacted by default. + +Use `open-computer-use event-stream validate --json ` or +`scripts/validate-event-stream-recording.py ` to check structure. diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/events.jsonl b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/events.jsonl new file mode 100644 index 00000000..08f150e1 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/events.jsonl @@ -0,0 +1,5 @@ +{"eventsPath":"events.jsonl","kind":"session.started","metadataPath":"metadata.json","sessionId":"fixture-session","suppressedEventsPath":"suppressed.jsonl","timestamp":"","type":"session.started"} +{"app":{"bundleIdentifier":"","name":"Feishu","pid":40061},"kind":"window.changed","reason":"initial","sessionId":"fixture-session","timestamp":"","type":"window.changed","window":{"bounds":{"Height":859,"Width":1239,"X":284,"Y":429},"id":687,"layer":0},"windowContextKey":"com.electron.lark\u001f"} +{"accessibilityInspectorPayload":{"cumulativeDiffFromInitial":false,"diffFromPrevious":false,"focusedSummary":"297 text entry area (settable, string) 沟通时请保持“公开可接受”\\n​\\n​\\n","fullTree":["","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""],"kind":"full","renderedText":"","screenshotAvailable":false,"screenshotNeededForContext":false,"treeLines":["","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","","",""],"windowBounds":{"height":859,"width":1239,"x":284,"y":429},"windowTitle":""},"app":{"bundleIdentifier":"","name":"Feishu","pid":40061},"diffFromPrevious":false,"kind":"AX.focusedWindowChanged","reason":"initial","screenshotNeededForContext":false,"sessionId":"fixture-session","timestamp":"","type":"AX.focusedWindowChanged","window":{"bounds":{"Height":859,"Width":1239,"X":284,"Y":429},"id":687,"layer":0}} +{"app":{"bundleIdentifier":"","name":"Feishu","pid":40061},"appKitLocation":{"x":1145.8125,"y":799.03515625},"button":"left","clickCount":1,"experimentalRawEvents":[{"appKitLocation":{"x":1145.8125,"y":799.03515625},"button":"left","clickCount":1,"eventType":"leftMouseDown","location":{"x":1145.8125,"y":640.96484375}},{"appKitLocation":{"x":1145.8125,"y":799.03515625},"button":"left","clickCount":1,"eventType":"leftMouseUp","location":{"x":1145.8125,"y":640.96484375}}],"kind":"mouse.click","location":{"x":1145.8125,"y":640.96484375},"sessionId":"fixture-session","targetAccessibilityElement":{"actions":["AXShowMenu","AXScrollToVisible"],"frame":{"height":48,"width":846,"x":671,"y":594},"role":"AXGroup","roleDescription":"group","windowFrame":{"height":48,"width":846,"x":387,"y":165}},"timestamp":"","type":"mouse.click","window":{"bounds":{"Height":859,"Width":1239,"X":284,"Y":429},"id":687,"layer":0}} +{"endReason":"recording_controls_stopped","kind":"session.ended","sessionId":"fixture-session","timestamp":"","type":"session.ended"} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/fixture-manifest.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/fixture-manifest.json new file mode 100644 index 00000000..2fe77f77 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/fixture-manifest.json @@ -0,0 +1,101 @@ +{ + "capturedAt": "2026-07-01", + "eventCount": 5, + "eventTypes": { + "AX.focusedWindowChanged": 1, + "mouse.click": 1, + "session.ended": 1, + "session.started": 1, + "window.changed": 1 + }, + "files": { + "events": "events.jsonl", + "mcpTranscript": "mcp-transcript.json", + "metadata": "metadata.json", + "session": "session.json", + "suppressed": "suppressed.jsonl" + }, + "fixtureFormatVersion": 1, + "importedAt": "2026-07-01", + "name": "ocu-simple-action-stop", + "officialPluginVersion": null, + "redaction": { + "mcpTranscriptSanitized": true, + "pathKeysRedacted": [ + "path", + "*Path" + ], + "pathKeysRewrittenRelative": [ + "currentSegmentEventsPath", + "currentSegmentMetadataPath", + "eventsPath", + "metadataPath", + "sessionPath", + "suppressedEventsPath" + ], + "preserveAppAttribution": false, + "screenshotsCopied": false, + "sessionIdKeys": [ + "id", + "sessionID", + "sessionId" + ], + "textKeys": [ + "AXDescription", + "AXSelectedText", + "AXTitle", + "AXValue", + "characters", + "cumulativeRenderedText", + "cumulativeTreeLines", + "description", + "diffRenderedText", + "fullRenderedText", + "fullTree", + "label", + "rawCharacters", + "renderedText", + "selectedText", + "text", + "title", + "treeLines", + "value", + "windowTitle" + ], + "timestampKeys": [ + "capturedAt", + "createdAt", + "endedAt", + "startTime", + "startedAt", + "timestamp", + "updatedAt" + ] + }, + "scenario": "simple-action-stop", + "scenarioRecipe": { + "captureGoal": "Record one low-risk left click and finish with the Record & Replay Done control.", + "expectedActionEvents": [ + "mouse.click" + ], + "expectedEndReason": "recording_controls_stopped", + "expectedEvidence": [ + "event_stream_start/status/stop hosted MCP response shape", + "metadata/session/events/suppressed handoff paths", + "one session.started as the first event", + "one session.ended as the final event", + "mouse.click action event", + "AX payload for the clicked target/window" + ], + "notes": [ + "Use this as the minimum official successful recording golden fixture.", + "Keep the clicked target harmless and easy to describe after redaction." + ], + "ocuCandidateSourceKind": "run-action-smoke", + "priority": "required", + "scenario": "simple-action-stop", + "userAction": "Click a stable, harmless UI target once, then click Done in the official recording controls." + }, + "source": "ocu", + "suppressedEventCount": 0 +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/mcp-transcript.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/mcp-transcript.json new file mode 100644 index 00000000..e1b1b1ac --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/mcp-transcript.json @@ -0,0 +1,587 @@ +{ + "finalStatusResponseShape": { + "id": 8, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "repeatStartResponseShape": { + "id": 4, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "repeatStopResponseShape": { + "id": 7, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "startResponseShape": { + "id": 3, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "statusResponseShape": { + "id": 5, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "stopResponseShape": { + "id": 6, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + }, + "target": "local-open-computer-use-action-smoke", + "transcript": [ + { + "direction": "send", + "message": { + "id": 1, + "jsonrpc": "2.0", + "method": "initialize", + "params": { + "capabilities": {}, + "clientInfo": { + "name": "event-stream-action-smoke", + "version": "0" + }, + "protocolVersion": "2025-11-25" + } + } + }, + { + "direction": "receive", + "message": { + "id": 1, + "jsonrpc": "2.0", + "result": { + "capabilities": { + "tools": { + "listChanged": false + } + }, + "protocolVersion": "2025-11-25", + "serverInfo": { + "name": "Record & Replay", + "version": "0.1.53" + } + } + } + }, + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {} + } + }, + { + "direction": "send", + "message": { + "id": 2, + "jsonrpc": "2.0", + "method": "tools/list", + "params": {} + } + }, + { + "direction": "receive", + "message": { + "id": 2, + "jsonrpc": "2.0", + "result": { + "tools": [ + { + "annotations": { + "destructiveHint": false, + "idempotentHint": false, + "openWorldHint": false, + "readOnlyHint": false + }, + "description": "", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_start" + }, + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": true + }, + "description": "", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_status" + }, + { + "annotations": { + "destructiveHint": false, + "idempotentHint": true, + "openWorldHint": false, + "readOnlyHint": false + }, + "description": "", + "inputSchema": { + "additionalProperties": false, + "properties": {}, + "type": "object" + }, + "name": "event_stream_stop" + } + ] + } + } + }, + { + "direction": "send", + "message": { + "id": 3, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_start" + } + } + }, + { + "direction": "receive", + "message": { + "id": 3, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + }, + { + "direction": "send", + "message": { + "id": 4, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_start" + } + } + }, + { + "direction": "receive", + "message": { + "id": 4, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + }, + { + "direction": "send", + "message": { + "id": 5, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_status" + } + } + }, + { + "direction": "receive", + "message": { + "id": 5, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": true, + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventCount": 3, + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "recording", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + }, + { + "direction": "send", + "message": { + "id": 6, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_stop" + } + } + }, + { + "direction": "receive", + "message": { + "id": 6, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + }, + { + "direction": "send", + "message": { + "id": 7, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_stop" + } + } + }, + { + "direction": "receive", + "message": { + "id": 7, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + }, + { + "direction": "send", + "message": { + "id": 8, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_status" + } + } + }, + { + "direction": "receive", + "message": { + "id": 8, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 97, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" + }, + "textRedacted": true, + "type": "text" + } + ], + "isError": false + } + } + } + ] +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/metadata.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/metadata.json new file mode 100644 index 00000000..baacc388 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/metadata.json @@ -0,0 +1,14 @@ +{ + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 5, + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/session.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/session.json new file mode 100644 index 00000000..715d2f86 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/session.json @@ -0,0 +1,7 @@ +{ + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "id": "fixture-session", + "startedAt": "" +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/suppressed.jsonl b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates/ocu-simple-action-stop/suppressed.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/README.md b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/README.md new file mode 100644 index 00000000..48b25940 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/README.md @@ -0,0 +1,11 @@ +# official-simple-action-stop-1.0.857 + +This is a sanitized Record & Replay event-stream fixture. + +- Source: `official` +- Official plugin version: `record-and-replay 1.0.857` +- Screenshots are not copied. +- User text, window titles, timestamps, session ids, local paths, and app attribution are redacted by default. + +Use `open-computer-use event-stream validate --json ` or +`scripts/validate-event-stream-recording.py ` to check structure. diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/events.jsonl b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/events.jsonl new file mode 100644 index 00000000..bb4a8531 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/events.jsonl @@ -0,0 +1,13 @@ +{"id":1,"kind":"session.started","timestamp":""} +{"app":{"name":"Sublime Text"},"ax":{"mode":"fullTree","textRedacted":true,"textLength":698},"id":2,"kind":"window.changed","timestamp":"","window":{"title":""}} +{"app":{"name":"Sublime Text"},"ax":{"mode":"diffFromPrevious","textRedacted":true,"textLength":9},"id":3,"kind":"mouse.click","mouse":{"button":"left","clickCount":0,"target":{"role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"Sublime Text"},"ax":{"mode":"diffFromPrevious","textRedacted":true,"textLength":9},"id":4,"kind":"mouse.click","mouse":{"button":"left","target":{"role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"Sublime Text"},"ax":{"mode":"diffFromPrevious","textRedacted":true,"textLength":9},"id":5,"kind":"mouse.click","mouse":{"button":"left","target":{"role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"Sublime Text"},"ax":{"mode":"diffFromPrevious","textRedacted":true,"textLength":9},"id":6,"kind":"mouse.click","mouse":{"button":"left","target":{"role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"TextEdit"},"ax":{"mode":"fullTree","textRedacted":true,"textLength":767},"id":9,"kind":"window.changed","timestamp":"","window":{"title":""}} +{"app":{"name":"Google Chrome"},"ax":{"mode":"fullTree","textRedacted":true,"textLength":49795},"id":7,"kind":"window.changed","timestamp":"","window":{"title":"","url":"https://bytedance.larkoffice.com/wiki/Eto0wVcw6igdCtkTVK4caIKnnDK"}} +{"app":{"name":"TextEdit"},"ax":{"mode":"fullTree","textRedacted":true,"textLength":767},"id":10,"kind":"mouse.click","mouse":{"button":"left","target":{"identifier":"_NS:34","role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"TextEdit"},"ax":{"mode":"fullTree","textRedacted":true,"textLength":767},"id":11,"kind":"mouse.click","mouse":{"button":"left","target":{"identifier":"_NS:34","role":"AXWindow","subrole":"AXStandardWindow","title":""}},"timestamp":"","window":{"title":""}} +{"app":{"name":"TextEdit"},"ax":{"mode":"diffFromPrevious","textRedacted":true,"textLength":9},"id":13,"keyboard":{"keyEquivalent":"a","modifiers":["command"],"target":{"identifier":"First Text View","role":"AXTextArea","value":""}},"kind":"keyboard.shortcut","timestamp":"","window":{"title":""}} +{"app":{"name":"TextEdit"},"id":14,"kind":"selection.changed","selection":{"selectedRange":{"length":49,"location":0},"selectedText":"","target":{"identifier":"First Text View","role":"AXTextArea","value":""}},"timestamp":"","window":{"title":""}} +{"id":15,"kind":"session.ended","timestamp":""} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/fixture-manifest.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/fixture-manifest.json new file mode 100644 index 00000000..f6b6bc96 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/fixture-manifest.json @@ -0,0 +1,102 @@ +{ + "capturedAt": "2026-07-01", + "eventCount": 13, + "eventTypes": { + "keyboard.shortcut": 1, + "mouse.click": 6, + "selection.changed": 1, + "session.ended": 1, + "session.started": 1, + "window.changed": 3 + }, + "files": { + "events": "events.jsonl", + "mcpTranscript": "mcp-transcript.json", + "metadata": "metadata.json", + "session": "session.json", + "suppressed": "suppressed.jsonl" + }, + "fixtureFormatVersion": 1, + "importedAt": "2026-07-01", + "name": "official-simple-action-stop-1.0.857", + "officialPluginVersion": "record-and-replay 1.0.857", + "redaction": { + "mcpTranscriptSanitized": true, + "pathKeysRedacted": [ + "path", + "*Path" + ], + "pathKeysRewrittenRelative": [ + "currentSegmentEventsPath", + "currentSegmentMetadataPath", + "eventsPath", + "metadataPath", + "sessionPath", + "suppressedEventsPath" + ], + "preserveAppAttribution": false, + "screenshotsCopied": false, + "sessionIdKeys": [ + "id", + "sessionID", + "sessionId" + ], + "textKeys": [ + "AXDescription", + "AXSelectedText", + "AXTitle", + "AXValue", + "characters", + "cumulativeRenderedText", + "cumulativeTreeLines", + "description", + "diffRenderedText", + "fullRenderedText", + "fullTree", + "label", + "rawCharacters", + "renderedText", + "selectedText", + "text", + "title", + "treeLines", + "value", + "windowTitle" + ], + "timestampKeys": [ + "capturedAt", + "createdAt", + "endedAt", + "startTime", + "startedAt", + "timestamp", + "updatedAt" + ] + }, + "scenario": "simple-action-stop", + "scenarioRecipe": { + "captureGoal": "Record one low-risk left click and finish with the Record & Replay Done control.", + "expectedActionEvents": [ + "mouse.click" + ], + "expectedEndReason": "recording_controls_stopped", + "expectedEvidence": [ + "event_stream_start/status/stop hosted MCP response shape", + "metadata/session/events/suppressed handoff paths", + "one session.started as the first event", + "one session.ended as the final event", + "mouse.click action event", + "AX payload for the clicked target/window" + ], + "notes": [ + "Use this as the minimum official successful recording golden fixture.", + "Keep the clicked target harmless and easy to describe after redaction." + ], + "ocuCandidateSourceKind": "run-action-smoke", + "priority": "required", + "scenario": "simple-action-stop", + "userAction": "Click a stable, harmless UI target once, then click Done in the official recording controls." + }, + "source": "official", + "suppressedEventCount": 0 +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/mcp-transcript.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/mcp-transcript.json new file mode 100644 index 00000000..d7ac8b8e --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/mcp-transcript.json @@ -0,0 +1,250 @@ +{ + "finalStatusResponseShape": { + "result": { + "content": [ + { + "decodedText": { + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + }, + "fixtureFormatVersion": 1, + "kind": "record-and-replay-hosted-capture-transcript", + "requiredHostedResponseShapes": [ + "startResponseShape", + "statusResponseShape", + "stopResponseShape", + "finalStatusResponseShape" + ], + "scenario": "simple-action-stop", + "source": "codex-hosted-record-and-replay", + "startResponseShape": { + "result": { + "content": [ + { + "decodedText": { + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + }, + "statusResponseShape": { + "result": { + "content": [ + { + "decodedText": { + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + }, + "stopResponseShape": { + "result": { + "content": [ + { + "decodedText": { + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + }, + "transcript": [ + { + "direction": "send", + "message": { + "id": 1, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_start" + } + } + }, + { + "direction": "receive", + "message": { + "id": 1, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + } + }, + { + "direction": "send", + "message": { + "id": 2, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_status" + } + } + }, + { + "direction": "receive", + "message": { + "id": 2, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "eventsPath": "events.jsonl", + "isRecording": true, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + } + }, + { + "direction": "send", + "message": { + "id": 3, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_stop" + } + } + }, + { + "direction": "receive", + "message": { + "id": 3, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + } + }, + { + "direction": "send", + "message": { + "id": 4, + "jsonrpc": "2.0", + "method": "tools/call", + "params": { + "arguments": {}, + "name": "event_stream_status" + } + } + }, + { + "direction": "receive", + "message": { + "id": 4, + "jsonrpc": "2.0", + "result": { + "content": [ + { + "decodedText": { + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "isRecording": false, + "maxDurationSeconds": 1800, + "metadataPath": "metadata.json", + "sessionDirectoryPathLength": 102, + "sessionDirectoryPathRedacted": true, + "sessionID": "fixture-session", + "startedAt": "" + }, + "textRedacted": true, + "type": "text" + } + ] + } + } + } + ] +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/metadata.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/metadata.json new file mode 100644 index 00000000..1e4ac93b --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/metadata.json @@ -0,0 +1,14 @@ +{ + "active": false, + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventCount": 13, + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionId": "fixture-session", + "sessionPath": "session.json", + "startedAt": "", + "state": "stopped", + "suppressedEventCount": 0, + "suppressedEventsPath": "suppressed.jsonl" +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/session.json b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/session.json new file mode 100644 index 00000000..715d2f86 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/session.json @@ -0,0 +1,7 @@ +{ + "endReason": "recording_controls_stopped", + "endedAt": "", + "eventsPath": "events.jsonl", + "id": "fixture-session", + "startedAt": "" +} diff --git a/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/suppressed.jsonl b/docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/official-simple-action-stop-1.0.857/suppressed.jsonl new file mode 100644 index 00000000..e69de29b diff --git a/docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md b/docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md new file mode 100644 index 00000000..6872bdd2 --- /dev/null +++ b/docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md @@ -0,0 +1,377 @@ +# Record & Replay Event Stream + +本文记录 2026-06-26 对官方 `record-and-replay` 1.0.857 与 `Codex Computer Use.app` 的本地逆向观察。目标是为 `open-computer-use` 复刻 Record & Replay event stream 提供可追溯上下文。 + +## 摘要 + +官方 Record & Replay 不是一个独立自动化 runtime。它仍然启动 `Codex Computer Use.app` 内的 `SkyComputerUseClient`，通过 `event-stream mcp` 子命令暴露一组单独的 MCP tools。普通 Computer Use 插件和 Record & Replay 插件共用同一个 client，差别主要在启动参数、插件元数据和 skill。 + +`Codex.app` 侧目前只观察到插件可见性、`recordAndReplay` feature flag 和通用 MCP elicitation/approval 处理。录制控制条、事件文件、AX diff、截图上下文和 `event_stream_*` tools 的字符串均在 Computer Use 相关 app / service 二进制中观察到。 + +这份证据支撑当前实现判断：OCU 应直接复刻官方 `event-stream mcp` 的可观察协议和 recording 文件行为，同时自己补齐脱离 Codex.app 后必须拥有的开始确认、录制控制条、Done / Discard、wait 唤醒和 notify callback。Codex.app 可以作为官方宿主参与 feature gate 或通用 MCP elicitation，但不能成为 OCU 独立使用时的必需依赖。 + +仍未被本页证据完整证明的内容包括：官方 successful recording 的跨场景完整 `metadata.json` / `events.jsonl` schema、AX compact diff 的算法级输出、截图触发阈值、timeout endReason、以及 Record & Replay start elicitation 的业务化 `message` / `_meta`。Required `simple-action-stop` 已有 official golden fixture，但 keyboard、drag、cancel、timeout 和算法级策略仍必须通过更多 official golden recordings 或正常 Codex 宿主流程继续采集，不能只靠字符串推断。 + +## 官方插件包装 + +官方插件在 Codex plugin cache 内的相对位置： + +```text +openai-bundled/record-and-replay/1.0.857/ +``` + +`record-and-replay` 插件的 `.mcp.json` 启动方式： + +```json +{ + "mcpServers": { + "event-stream": { + "command": "./Codex Computer Use.app/Contents/SharedSupport/SkyComputerUseClient.app/Contents/MacOS/SkyComputerUseClient", + "args": ["event-stream", "mcp"], + "cwd": "." + } + } +} +``` + +对比普通 `computer-use` 插件，它也启动同一个 `SkyComputerUseClient`，但参数是： + +```json +["mcp"] +``` + +这说明官方把 Record & Replay 实现为同一 runtime 下的另一个 MCP mode，而不是另一个宿主能力。 + +## MCP Surface + +直接探测官方 `SkyComputerUseClient event-stream mcp` 的 `tools/list`，serverInfo 为： + +```text +name: Record & Replay +protocolVersion: 2025-11-25 +``` + +2026-06-26 复核时，官方 initialize result 只包含 `capabilities`、`protocolVersion` 和 `serverInfo`，没有 `instructions` 字段。OCU event-stream MCP initialize 应保持这一点；普通 Computer Use MCP 的 instructions 不适用于 Record & Replay surface。 + +对应的 normalized 非录制 fixture 已保存到 `fixtures/record-and-replay-event-stream-surface-1.0.857.json`。该 fixture 只包含 `initialize` 和 `tools/list`，没有调用 `event_stream_start`，因此不包含用户录制内容或 session 文件。 + +同日尝试用裸 `SkyComputerUseClient event-stream mcp` 调用官方 `event_stream_status` 时，stdio client 未在数秒内返回响应，手动中断。该现象说明 lifecycle tools 可能依赖官方 runtime / 宿主状态，不适合作为无宿主 smoke 的稳定非录制样本；当前自动化官方对比因此只覆盖 `initialize` 和 `tools/list`，且带读取超时。 + +2026-06-27 通过 Codex-hosted Record & Replay MCP tool 观察到 no-active 状态：在未启动录制且 `event_stream_status` 显示无 active recording 后，`event_stream_status` 与 `event_stream_stop` 都返回 text JSON `{"isRecording": false, "maxDurationSeconds": 1800}`，没有 `state`、`active`、`eventCount`、`eventsPath` 或 `metadataPath`。该观察已保存为 `fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json`。这只校准 no-active lifecycle response shape；active / completed session 的官方字段仍需要 successful recording golden 校准。仓库内 `scripts/compare-event-stream-no-active.py` 会启动本地 OCU `event-stream mcp`，只调用 `event_stream_status` / `event_stream_stop`，和该 fixture 比对 text JSON，并断言不会创建 session 文件；该脚本不调用 `event_stream_start`。 + +2026-06-29 通过 Codex-hosted Record & Replay 正常 `event_stream_start` / `event_stream_status` / `event_stream_stop` 观察到一个 completed session，最终 `endReason=recording_controls_stopped`，`isRecording=false`，`maxDurationSeconds=1800`。这次录制约 5 分钟，期间用户没有执行预期的低风险单击场景，实际捕获了多应用真实工作流和私有可见内容，因此不入库、不脱敏导入为 fixture，也不能替代 required `simple-action-stop` official successful recording golden。只保留脱敏结构观察： + +- active start response 包含 `eventsPath`、`isRecording=true`、`maxDurationSeconds=1800`、`metadataPath`、`sessionDirectoryPath`、`sessionID` 和 `startedAt`。 +- completed status / stop response 包含 `endReason`、`endedAt`、`eventsPath`、`isRecording=false`、`maxDurationSeconds=1800`、`metadataPath`、`sessionDirectoryPath`、`sessionID` 和 `startedAt`。 +- `metadataPath` 实际指向 `session.json`；该 session 文件只包含 `id`、`startedAt`、`endedAt`、`endReason` 和 `eventsPath`。 +- session 目录中只观察到 `events.jsonl` 和 `session.json`，没有 `metadata.json`、`suppressed.jsonl` 或截图目录。 +- `events.jsonl` 共 434 行，事件使用顶层 `kind` 字段；脱敏 kind 计数为 `session.started=1`、`session.ended=1`、`window.changed=16`、`selection.changed=124`、`mouse.click=34`、`mouse.context_menu=2`、`mouse.drag=4`、`keyboard.text_input=88`、`keyboard.submit=21`、`keyboard.shortcut=143`。 +- AX payload 中能观察到 `fullTree` 与 `diffFromPrevious` 形态；该样本只证明官方 completed session 会写 AX full / diff 结构，不足以校准 compact diff 算法、截图触发策略或 OCU same-scenario 对比。 + +这个观察把 official hosted successful recording 的最小输入合同收敛为 `events.jsonl + session.json`：后续正式导入官方 golden 时，不能假设原始目录一定有 OCU 风格 `metadata.json`、`suppressed.jsonl` 或截图目录。仓库 fixture 层会在脱敏导入时补齐自己的可验证 metadata 和空 suppressed 文件；这只是入库 normalization，不代表官方原始包一定写这些文件。 + +2026-07-01 通过正常 Codex-hosted Record & Replay 流程采集并脱敏导入了 required `simple-action-stop` official fixture：`fixtures/recordings/official-simple-action-stop-1.0.857/`。该 fixture 来自 official `record-and-replay 1.0.857`，manifest 标记 `source=official`、`scenario=simple-action-stop`，脱敏后事件统计为 13 条：`session.started=1`、`window.changed=3`、`mouse.click=6`、`keyboard.shortcut=1`、`selection.changed=1`、`session.ended=1`，最终 `endReason=recording_controls_stopped`，并带 hosted `event_stream_start/status/stop/final status` response-shape evidence。`scripts/check-event-stream-golden-readiness.py` 用 `--require-source official`、`--require-official-plugin-version 'record-and-replay 1.0.857'`、`--require-event-type mouse.click`、`--require-end-reason recording_controls_stopped`、`--require-session-alias`、`--require-metadata-counts`、`--require-handoff-paths` 和 MCP transcript/response-shape requirements 可通过；`scripts/check-event-stream-official-fixture-coverage.py --check-readiness --require-readiness` 与 `make record-and-replay-official-golden-fixture-gate` 也可通过。这满足最小 required official successful recording golden 门槛，但该样本不是 keyboard / drag / cancel / timeout 校准样本，也不能单独证明 AX compact diff 算法、截图触发阈值或所有事件字段 schema 已等价。 + +仓库内 `scripts/compare-event-stream-surface.py` 是 non-recording surface fixture 的可复跑 drift check：默认比较 OCU `event-stream mcp` 与 `fixtures/record-and-replay-event-stream-surface-1.0.857.json`，`--use-default-official` 会额外探测本机默认官方 plugin cache 下的 `SkyComputerUseClient event-stream mcp` 并和同一 fixture 比较。2026-06-26 本机复核时，官方默认 cache 的 non-recording surface 仍匹配该 fixture；该脚本不调用 `event_stream_start`，不会启动真实录制。 + +`scripts/run-record-and-replay-baseline-smoke.sh` 也会消费 `scripts/compare-event-stream-no-active.py` 的结果，并在最终摘要写入 `evidence.officialNoActiveResponse`。该 evidence 让 opt-in baseline 同时证明官方 no-active response shape 和无 session 文件副作用，但仍只属于 non-recording 证据。 + +2026-06-27 再次运行 `scripts/compare-event-stream-surface.py --use-default-official`，本机 OCU `event-stream mcp` 与官方默认 cache 中的 `record-and-replay/1.0.857` non-recording surface 仍同时匹配该 fixture。复核范围仍只包括 initialize / `tools/list`，确认 server name 为 `Record & Replay`、protocol version 为 `2025-11-25`，tools 仍为 `event_stream_start`、`event_stream_status`、`event_stream_stop`。该复核没有调用 `event_stream_start`，没有产生官方 recording。 + +仓库内 `scripts/probe-event-stream-recording.py` 是后续采集 official golden recording 前的 raw MCP 探测脚手架：它支持 `--target local|official`，记录 initialize / tools-list / start / repeat start / status / stop / repeat stop / final status transcript，处理 server-initiated `elicitation/create`，并在显式 `--start-stop` 时尝试短录制、重复 start、status、stop、重复 stop 和最终 status。probe 的 elicitation response shape 按 Codex host 观察结果处理：accept 发送 `content: {}`，decline / cancel 发送 `content: null`；`--elicitation-action accept|decline|cancel` 可显式选择 action，旧的 `--decline-elicitation` 仍作为 decline 兼容别名。`--fixture-output` 会把 MCP text content 中可解析的 JSON 转成脱敏 `textJSON`，避免 start/status/stop response text 内部保留本机绝对路径。`scripts/test-event-stream-recording-probe.py` 用 fake MCP server 固定 elicitation 行为、repeat start、repeat stop、final status 和 text JSON 脱敏，`scripts/test-event-stream-local-probe.py` 用真实本地 OCU start/repeat start/status/stop/repeat stop/final status 固定 local response shape、one-active、idempotent stop、completed status 语义和最终 session validator 路径，并验证 generated fixture 不含本机路径。2026-06-26 本机复核结果： + +- `scripts/probe-event-stream-recording.py --target official` 可正常返回官方 `Record & Replay` initialize / tools-list。 +- `scripts/probe-event-stream-recording.py --target official --start-stop --timeout 5` 中，官方 raw `event_stream_start` 在宿主外 5 秒超时，随后 raw `event_stream_stop` 也超时；没有返回 `metadataPath` / `eventsPath`，也没有可导入的官方 recording fixture。 +- `scripts/probe-event-stream-recording.py --target local --start-stop` 可对 OCU baseline 完成 start / status / stop，并返回 `recording_controls_stopped`，用于验证 probe 脚手架自身。 + +2026-06-27 再次运行 `make event-stream-official-start-probe`，官方 bundled `record-and-replay/1.0.857` 仍只在 initialize / tools-list 阶段正常返回；`event_stream_start`、随后 `event_stream_status` 和 `event_stream_stop` 都在 10 秒超时，没有返回 `metadataPath`、`eventsPath`、`sessionPath` 或任何可导入 recording。该复核确认当前裸官方 MCP client 仍缺少 Codex 宿主侧录制状态 / UI choreography，不能单独产出 official successful recording fixture。 + +因此 official golden recordings 仍需要通过正常 Codex 宿主 Record & Replay 流程、或后续找到官方 runtime 所需的宿主状态后采集；raw 官方 client 只适合 surface 和边界观测，不能产出 required/recommended successful recording fixture。 + +上述官方 raw start/status/stop timeout 已保存为 sanitized fixture： + +- `fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json` + +该 fixture 只保留 transcript shape、server info、tool names、start/status/stop 三次 `tools/call` timeout 事实和进程返回形态，不包含本机绝对路径或用户录制内容。`scripts/test-event-stream-probe-fixtures.py` 会在默认 CI 中校验它的基本结构，并要求 `startResponseShape`、`statusResponseShape` 和 `stopResponseShape` 都是 timeout；`scripts/test-event-stream-recording-probe.py` 则校验 probe 自身的 elicitation 应答和 request 摘要提取逻辑。若把这类 timeout transcript 作为 `mcp-transcript.json` 交给 golden readiness gate 并要求对应 tool response，gate 会明确报 `MCP tool timed out`；它只能证明当前宿主外采样边界，不能替代成功的 official response golden。 + +官方 successful recording 后续入库时，优先用 `scripts/ingest-official-record-and-replay-fixture.py --status-json --name --scenario --inspect-only` 从 hosted `event_stream_stop` / `event_stream_status` 返回 JSON 里递归提取 `metadataPath` / `sessionPath` / `eventsPath` / `sessionDirectoryPath`，先确认 recording input、handoff path 和可选 MCP transcript 证据都能解析且不会创建 fixture；检查通过后去掉 `--inspect-only` 正式导入。也可以用 `--status-json -` 从 stdin 直接读取刚复制出的 hosted JSON；如果 hosted JSON 中的 handoff path 是相对路径，传 `--status-json-base-dir ` 固定解析基准。若 hosted JSON 只给 `sessionDirectoryPath`，导入器会直接把该目录作为 recording input，再读取其中的 `metadata.json` 或官方最小 `session.json`。随后脚本会调用 `scripts/import-event-stream-fixture.py` 生成脱敏 fixture 并立即跑 golden readiness。导入器现在接受 official hosted 最小目录，也就是只有 `events.jsonl` 和 `session.json` 的 recording；它会把 session `id` 当作 session id 脱敏，生成 fixture 侧 `metadata.json`，补齐 `eventCount` / `suppressedEventCount` / `eventsPath` / `metadataPath` / `sessionPath` / `suppressedEventsPath`，并在原始包没有 suppressed stream 时写空 `suppressed.jsonl`，以满足后续 readiness / compare gate 的 handoff path 合同。若同时有 raw MCP transcript，可加 `--mcp-transcript --require-mcp-transcript-evidence --check-fixture-set`，或者在 status JSON 已经来自文件时用 `--mcp-transcript -` 从 stdin 读取单独 transcript，让导入后继续跑 response-shape readiness 和集合级 official fixture set gate；`--status-json -` 和 `--mcp-transcript -` 不能同时使用，若同一 status JSON 已含 transcript / response-shape evidence，可加 `--use-status-json-as-transcript` 复用。`make event-stream-official-fixture-ingest-smoke` 用合成 hosted response 覆盖这条路径。 + +底层仍可直接用 `scripts/import-event-stream-fixture.py --mcp-transcript ` 生成脱敏 fixture，再跑 `make event-stream-fixture-smoke` 复核导入链路。该 smoke 不只检查脱敏：它还会把导入后的 fixture 继续喂给 `check-event-stream-golden-readiness.py --require-mcp-response-shape repeatStartResponseShape --require-mcp-response-shape repeatStopResponseShape --require-mcp-response-shape finalStatusResponseShape`，并在样本包含 fallback 证据时要求 `--require-suppressed-events` / `--require-suppressed-event-type`；同时用 `compare-event-stream-recordings.py --require-mcp-response-shapes --require-same-mcp-response-schema` 自比。这样能固定 sanitized `mcp-transcript.json` 和 `suppressed.jsonl` 被 readiness/compare 校准 gate 正确消费；真实 successful recording 覆盖仍以 `fixtures/recordings/*/fixture-manifest.json` 的 official scenario 和 coverage/readiness report 为准。 + +已观察到 3 个 tools： + +| Tool | 参数 | 读写语义 | 说明 | +| --- | --- | --- | --- | +| `event_stream_start` | 无 | non-readonly, non-idempotent | 开始录制用户操作，最长 30 分钟；已有 active recording 时返回当前 session。 | +| `event_stream_status` | 无 | readonly, idempotent | 返回当前或最近一次录制状态，包括 metadata 和 events 路径。 | +| `event_stream_stop` | 无 | non-readonly, idempotent | 停止 active recording；返回 metadata 和 events 路径。 | + +这个 surface 应作为 OCU 官方兼容模式的第一约束。扩展能力应放在 OCU CLI 或独立 runtime API 中，不应修改这 3 个 tool 的参数。 + +## Skill 行为 + +官方 `record-and-replay` skill 的关键流程： + +- 用户确认准备好后才调用 `event_stream_start`。 +- start 后结束当前 turn，请用户完成操作后回复。 +- 用户完成后调用 `event_stream_stop`。 +- 读取返回的 `metadataPath` 和 `eventsPath`。 +- `events.jsonl` 是主要输入，`session.json` / metadata 只提供 timing 和路径。 +- cancellation 的 `endReason` 为 `recording_controls_cancelled`。 +- AX payload 可能是 full tree，也可能是 diff。 +- AX compact diff 使用 `~`、`+`、`-` 表示 changed / added / removed。 + +这意味着复刻工作不仅要产出事件，还要保留能被 skill 消费的文件布局和 endReason。 + +## Codex.app Asar 观察 + +本地解包命令示例： + +```bash +npx --yes @electron/asar extract /Contents/Resources/app.asar /codex-app-asar-record-replay +``` + +在解包后的 asar 中观察到： + +- `recordAndReplay` feature flag。 +- bundled plugin eligibility 中对 `record-and-replay` 的特殊处理。 +- 插件 UI 中对 `record-and-replay` 的 icon / name 特殊展示。 +- 通用 MCP elicitation/approval 事件：`reply-with-mcp-server-elicitation-response`。 +- pending request panel 处理 MCP server elicitation response。 +- `elicitation/create` params schema 中观察到 `mode: "form"`、`message`、`requestedSchema`；`requestedSchema` 只接受 object schema，`properties` 可为空，`required` 可选。Codex host 的 UI 分流逻辑会检查 `mode === "form"` 后才进入普通表单 elicitation path。 +- elicitation response schema 中观察到 `action: "accept" | "decline" | "cancel"`；`accept` 可带 `content` 对象，`decline` / `cancel` 会带 `content: null`。Codex host 还会把用户操作通过 `reply-with-mcp-server-elicitation-response` 发回 runtime。 + +未在 Codex.app asar 中观察到： + +- `event_stream_start` +- `event_stream_status` +- `event_stream_stop` +- 录制控制条文案 +- `events.jsonl` +- `metadata.json` +- `recording_controls_cancelled` + +推断：Codex.app 作为宿主只负责 feature gate、插件可见性和通用 MCP elicitation；Record & Replay 的核心录制逻辑和控制条在 Computer Use runtime 内部。 + +## Computer Use 二进制字符串 + +在官方 `Codex Computer Use.app` / `SkyComputerUseClient` / `SkyComputerUseService` 相关二进制中观察到下列字符串。 + +录制控制条和结束原因： + +```text +Record & Replay Recording Controls +Record & Replay is recording your actions +Open Record & Replay recording controls +Move Record & Replay recording controls +Discard Recording +I'm done recording. +I've cancelled recording. +recording_controls_stopped +recording_controls_cancelled +``` + +MCP 与文件产物： + +```text +Runs the Record & Replay client as an MCP server +event_stream_start +event_stream_status +event_stream_stop +events.jsonl - append-only activity events captured during the segment +metadata.json - segment timestamps and event counts +eventsPath +metadataPath +suppressedEventsPath +currentSegmentEventsPath +currentSegmentMetadataPath +suppressed.jsonl +session.json +``` + +权限和 gating： + +```text +Record & Replay is not enabled for this user. +Record & Replay approval cancelled via MCP elicitation. +Record & Replay approval denied via MCP elicitation. +``` + +这些字符串支持两个判断： + +- 官方控制条不是 Codex.app 直接绘制的 Electron UI。 +- 官方 event stream 至少包含主事件流、metadata 和 suppressed events 三类文件。 +- 官方 start tool description 和控制条确认录制最长 30 分钟；当前字符串样本未确认超时结束时对外写入的 endReason。 +- 官方 start 前 approval 至少有 cancelled / denied 两种对外错误语义；`Codex.app` 可提供通用 MCP elicitation panel，但 Record & Replay 录制逻辑仍在 Computer Use runtime。 + +## Start Approval 观察 + +当前证据分两类： + +- `Codex.app` asar：能看到通用 MCP elicitation/approval request 和 pending request panel 处理逻辑。 +- Computer Use 相关二进制：能看到 `Record & Replay approval cancelled via MCP elicitation.`、`Record & Replay approval denied via MCP elicitation.`、录制控制条文案和 `recording_controls_*` endReason。 + +因此更稳妥的推断是：官方宿主可以承载 MCP elicitation UI，但 Record & Replay start approval 的业务语义属于 event-stream runtime。OCU 复刻时不能要求独立 plugin / skill 一定运行在 Codex.app 里，也不能把 approval 做成新的 MCP tool 参数。 + +后续实现和校准应遵守： + +- `event_stream_start` 保持无参数。 +- approval approved 后才创建 session 和写 `session.started`。 +- approval denied / cancelled 不创建 session，并返回官方风格错误。 +- OCU 独立模式需要自有开始确认 UI 或明确的 policy fallback。 +- MCP elicitation 只能作为 host 支持时的内部通道，不能成为 OCU 独立使用的硬依赖，也不能改变 `tools/list` 或 `event_stream_start` 的 input schema。 +- OCU 当前 baseline 已接入 `OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp|elicitation`：event-stream MCP server 只在 host initialize 声明 `capabilities.elicitation` 后发送 `elicitation/create`，accept 后创建 session，decline / cancel / unsupported 不创建 session。OCU 发送显式 `mode: "form"` 和空对象 `requestedSchema`，以匹配 Codex.app host 的可渲染 form elicitation schema；真实官方 Record & Replay start request 的最终 message / `_meta` / business-specific schema 仍待 official golden 校准。 + +## Event Stream 与 AX Diff + +已观察到的事件和 AX diff 相关字符串： + +```text +session.started +mouse.click +mouse.context_menu +mouse.drag +keyboard.text_input +keyboard.submit +keyboard.shortcut +terminal.value_changed +selection.changed +AX.focusedWindowChanged +window.changed +debug.error +diffFromPrevious +suppressedEventCount +isAXTreeDiffingEnabled +lastAXTree +AX tree unexpectedly missing. +screenshotNeededForContext +accessibilityInspectorPayload +experimentalRawEvents +secureInput +The following is a diff from the previous accessibility tree +The following is a cumulative diff from the initial accessibility tree +Child page depth differs +values differ from index +timediff +feature/axTreeDiffing +feature/axTreeDiffingRemovedElementIDRanges +DifferenceBaseline +UIElementRenderDifference +UIElementRenderDifferenceBuffer +enableAXDiffing +difference +disableDiff +Skip AX tree diff render because removed element summary exceeded full-tree line budget (%ld). +Skip AX tree diff render because difference exceeded full-tree line budget (%ld). +``` + +官方 skill 还明确提到 compact render 语法： + +- `~`：changed +- `+`：added +- `-`：removed + +因此 OCU 如果要“直接复刻官方”，不能只记录 raw AX tree 或每次 full tree。需要实现官方风格的 compact AX diff，并在 diff 超预算或缺少 baseline 时回退 full tree。OCU 当前已在 full AX payload 中双写官方观察到的 `fullTree` 和既有 `treeLines`，并保留 previous diff 作为主 `renderedText/treeLines`，额外写入 `cumulativeRenderedText/cumulativeTreeLines` 表示从 initial tree 到 current tree 的累计 diff；`fullTree` 当前采用与 `treeLines` 同源的行数组，字段精确结构和 diff 算法仍待 official golden recording 校准。 + +输入事件方面，当前字符串样本能确认 `mouse.context_menu`、`mouse.drag`、`keyboard.submit`、`keyboard.shortcut`、`terminal.value_changed`、`selection.changed`、`window.changed` 和 `debug.error` 这些官方事件名，也能看到 `secureInput` 线索。OCU baseline 已实现 `mouse.context_menu`、`mouse.drag`、`keyboard.submit`、`keyboard.shortcut`、轻量 `terminal.value_changed`、轻量 `selection.changed` / selection clear、顶层 `window.changed` 和低敏 `debug.error`；生产路径现在优先安装 listen-only `CGEvent` tap 捕获 session 级输入，失败时回退到 `NSEvent.addGlobalMonitorForEvents`，并新增 `make event-stream-action-smoke` 验证外部 HID click 和 scroll wheel 能分别写出 `mouse.click` 与 scroll raw-only `experimentalRawEvents`，且同一份真实录制产物能继续生成含 Scroll replay step 的 `SKILL.md` 草稿。`keyboard-input-stop` 和 `drag-stop` 场景通过同一 action smoke 的显式 scenario 分别验证 `keyboard.text_input` 和 `mouse.drag`；`keyboard.shortcut` 仍是实现支持的官方字符串事件名，但默认真实输入 smoke 不再依赖低影响功能键 shortcut 合成，后续等 official golden recording 入库后再校准 shortcut 字段和采样策略。其中 `mouse.drag` 按 mouse down / dragged / up 的起止位置、距离和时长记录，字段结构仍需 official golden recording 校准。`terminal.value_changed` 当前只在明显终端上下文且 focused AX value 变化时记录，payload 只写 value hash / length 和脱敏 focused element，不写终端缓冲区全文；字段结构、触发时机和官方是否记录明文仍需官方样本校准。`selection.changed` 当前在 focused AX element 暴露非空 `selectedText` 且 selection signature 变化时记录；如果此前已记录过选择而当前选择为空，会写 `selectionCleared=true` 的清空事件，避免后续 skill 误用 stale selection，字段和触发时机仍需官方样本校准。`window.changed` 当前在录制开始和输入前 app/window context 变化时写入，并继续补 `AX.focusedWindowChanged` 作为 AX 上下文；字段、初始帧语义和事件顺序仍需官方样本校准。`debug.error` 当前在 AX snapshot、cumulative AX diff fallback、screenshot 写入失败和 Input Monitoring monitor 全部安装失败时写入主事件流，payload 只保留 subsystem / reason / errorType / 低敏 context，完整错误细节继续写入 `suppressed.jsonl`。Input Monitoring 不可用的 OCU baseline 使用 `subsystem=inputMonitoring` 和 `reason=inputMonitorsUnavailable`，这是为了避免独立集成方拿到无输入事件的空录制却没有可诊断信号；字段仍待官方样本校准。`secureInput` 当前作为安全 baseline 接入：focused AX element 暴露 protected/secure/password 信号时，键盘文本事件不写明文 `text`，raw key event 不写 `characters`，只保留 redaction 标记和字符数。终端上下文下，OCU 也会脱敏键盘 raw characters 和 focused element value，避免把终端缓冲区写入输入事件。`experimentalRawEvents` 已作为低层 NSEvent 摘要接入点击、拖拽、键盘和 scroll wheel；当前字符串样本没有确认独立 scroll event 名称，因此 OCU 对 scroll 只写 raw-only `experimentalRawEvents` 事件，避免臆造官方事件名。为了让 recording-to-skill handoff 不漏掉滚动动作，summary/scaffold 会把 `kind=experimentalRawEvents` / `type=experimentalRawEvents` 且 `reason=scrollWheel` 的事件纳入 `actionSequence` 并渲染成 Scroll replay step。 + +## 截图上下文 + +当前没有观察到“每个事件都截图”的证据。已观察到的线索是： + +- `screenshotNeededForContext` +- `accessibilityInspectorPayload` +- `experimentalRawEvents` + +更合理的实现方向是：事件默认以 AX 和输入事件为主，只有在上下文不足或官方 schema 要求时标记并采集截图。截图路径、字段名和采集频率仍需要官方样本校准。 + +OCU 当前采用的可校准策略： + +- 默认策略为 `auto`，通过 `OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS=auto|always|never` 覆盖。 +- `auto` 只在 AX payload 上下文明显不足时尝试写截图；`always` / `never` 用于验证和调试。 +- 截图写入 session 目录下的 `screenshots/`，payload 包含 `screenshotNeededForContext`、`screenshotAvailable`，写入成功时包含 `screenshotPath`。 +- 这属于对官方线索的开源实现策略，不代表已经确认官方触发阈值或字段全集；后续仍需 official golden recording 校准。 + +## 对 OCU 的实现影响 + +OCU 应分成两层： + +- 官方兼容层：`event-stream mcp` 暴露无参数 3 tools，文件布局和 JSON schema 尽量对齐官方。 +- OCU 扩展层：CLI `wait`、自有录制控制条、独立通知机制、后续单独 plugin/skill repo。 + +自有控制条是必要扩展，因为 OCU 不能依赖 Codex.app 帮忙绘制 recording bar，也不能依赖 Codex.app 把用户点击回传给 Computer Use。控制条交互应直接落到 OCU runtime，并把 stop/cancel 状态写回 session。 + +## 兼容边界 + +后续实现和评审应按下表区分“官方复刻”和“OCU 扩展”，避免把集成便利性误认为官方 surface。 + +| 领域 | 官方兼容要求 | OCU 扩展 | +| --- | --- | --- | +| MCP tools | 只暴露 `event_stream_start`、`event_stream_status`、`event_stream_stop`，三者均无参数。 | 不在 MCP surface 增加 `cancel`、`wait` 或 callback 参数。 | +| CLI / runtime API | 官方插件通过 `SkyComputerUseClient event-stream mcp` 间接使用。 | `open-computer-use event-stream start/status/stop/cancel/wait --json` 供独立集成方使用；`wait --notify-command ''` 提供本地 callback。 | +| 录制 UI | 官方控制条在 Computer Use runtime 内部，Codex.app 只做宿主 gating / elicitation。 | OCU 自己绘制浮动控制条，Done / Discard 交互直接落到 OCU runtime。 | +| 通知模型 | 官方 skill 通过下一轮调用 stop/status 读取路径。 | 已提供 blocking `wait` listener 和同步 `--notify-command` callback；后续如需长期订阅，可继续在 OCU 扩展层增加 webhook/socket。 | +| Start approval | 官方存在 MCP elicitation cancelled / denied 错误语义，tool surface 仍无参数。 | OCU 已有自有开始确认 UI / policy，并接入 MCP elicitation baseline 作为 host-capability 内部 gate；host form schema 已按 Codex.app asar 校准，真实官方业务 metadata / message 仍待 golden 校准。 | +| 事件文件 | `events.jsonl` 是主输入，metadata/session 提供 timing、路径和状态。 | 可增加 `latest-session.json` / `active-session.json` 这类索引文件，但不要让上层 skill 必须依赖它们。 | +| AX / screenshot | 以官方样本确认 payload schema、compact diff 和截图触发条件。 | 在校准前允许保留 OCU baseline 字段，但必须在 docs 标记待官方校准。 | + +## 未决问题 + +后续需要用官方 runtime 采集短录制样本来确认： + +- `event_stream_start/status/stop` 的完整 response JSON schema。 +- `events.jsonl` 每类事件的字段、顺序和 timing 格式。 +- `metadata.json` 或 `session.json` 的实际命名、字段和状态流转。 +- cancel 与 stop 的 metadata 差异。 +- start approval approved / denied / cancelled 的 MCP response、错误文本和是否写任何 session 文件。 +- `suppressed.jsonl` 写入条件和字段。 +- 截图路径、payload 字段和触发策略。 +- AX diff 的 element identity、line budget、removed range summary 和 fallback 规则。 + +这些样本应按版本保存，至少记录官方插件版本、采集日期、系统版本和脱敏方式。 + +## OCU 当前复刻状态 + +截至 2026-06-26，OCU 已具备可继续校准的 macOS baseline： + +- `open-computer-use event-stream mcp` 暴露官方兼容的 3 个无参数 tools。 +- `event-stream start/status/stop/cancel/wait --json` 作为 OCU CLI 扩展，便于独立插件或 skill 集成；`wait --notify-command ''` 在结束唤醒时把最终 status JSON 写给本地 callback，超时 wait 不触发 callback。 +- session 产物包含 `events.jsonl`、OCU 完整状态 `metadata.json`、官方风格最小 handoff `session.json`、`suppressed.jsonl`、`latest-session.json`、录制中的 `active-session.json`，以及按需写入的 `screenshots/`。 +- OCU 的 `session.json` 不再是 `metadata.json` 的字节相等 alias；它写 `id`、`startedAt`、结束后的 `endedAt` / `endReason` 和 `eventsPath`，贴近 2026-06-29 Codex-hosted completed session 观察。Swift / Python validator 与 golden readiness 仍要求 `metadata.json` 与 `session.json` 语义兼容，并在直接传入 `session.json` 且同目录存在 `metadata.json` 时优先用完整 metadata 做 OCU strict 校验。 +- `AX.focusedWindowChanged` 会尽量写入 `accessibilityInspectorPayload`，首帧为 full tree，并同时保留官方兼容 `fullTree` 与 OCU `treeLines`；同 app/window 后续帧使用轻量 `~` / `+` / `-` compact diff，diff payload 同时包含 previous diff 和 cumulative diff baseline。 +- `mouse.click` / `mouse.context_menu` 和 `keyboard.text_input` / `keyboard.submit` / `keyboard.shortcut` 会补充轻量 AX 目标上下文：点击事件尽量记录 `targetAccessibilityElement`，键盘事件尽量记录 `focusedAccessibilityElement`。 +- OCU 自有浮动控制条负责录制状态、计时、Done / Discard 和拖动，不依赖 Codex.app。 +- OCU 已接入 MCP elicitation approval baseline；`OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp|elicitation` 时，支持 elicitation 的 MCP host 会收到内部 `elicitation/create` approval request，accept 后才创建 session，decline / cancel / unsupported 不创建 session。 +- OCU 已按官方 “up to 30 minutes” 行为增加自动结束；当前 endReason 使用 `recording_time_limit_reached` 作为开源 baseline，后续需用 official golden recording 校准官方实际值。 +- OCU 已按官方已确认事件名补齐 `mouse.drag` baseline，mouse down 时只记录 pending pointer，mouse up 后按距离阈值决定写 click/context menu 还是 drag，避免拖拽被误记为点击。 +- OCU 已按官方字符串补轻量 `selection.changed` baseline；输入事件后如果当前 focused AX element 有非空 `selectedText` 且 signature 变化，会写入 selected text、focused element summary 和 app/window attribution；如果此前有选择而当前选择为空，会写 `selectionCleared=true` 清空事件。 +- OCU 已按官方字符串补轻量 `terminal.value_changed` baseline；明显终端上下文下 focused AX value 变化时写入 value hash / length 和脱敏 focused element，不写终端缓冲区全文。 +- OCU 已按官方字符串补顶层 `window.changed` baseline；录制开始和输入前 app/window context 变化时写入 window context，并随后补 `AX.focusedWindowChanged`。 +- OCU 已按官方字符串补轻量 `debug.error` baseline；AX / screenshot 上下文采集降级以及 Input Monitoring monitor 全部安装失败时写主事件低敏 debug metadata，完整细节仍写 `suppressed.jsonl`。 +- OCU 已按官方 `secureInput` 线索补键盘文本脱敏 baseline；secure focused element 下不记录明文 text/raw characters。 +- OCU 已按官方字符串里的 `experimentalRawEvents` 补低层输入事件 baseline，默认开启，可用 `OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS=0` 关闭；scroll wheel 暂时只作为 raw-only 事件记录。 +- OCU 已补 input 前 window context 检测；如果 app bundle + frontmost window title 变化，会在输入事件前先写 `AX.focusedWindowChanged`，以覆盖同一 app 内窗口切换后的上下文缺口。 +- OCU 已新增 runtime CLI `open-computer-use event-stream validate --json` 和源码脚本 `scripts/validate-event-stream-recording.py` 作为录制产物结构校验器；OCU smoke 使用 `--strict-ocu` 校验 `metadata.json` / `session.json` 语义兼容、count、final event、endReason、截图路径，以及 OCU active recording 的 `currentSegmentEventsPath` / `currentSegmentMetadataPath` 存在性、final recording 不残留 current segment 字段，并要求完整 OCU session 声明且可解析 `metadataPath` / `sessionPath` / `eventsPath` / `suppressedEventsPath`。runtime Swift validator 与源码 Python validator 都会输出 `declaredPaths` 作为 handoff 证据。安装后的独立 skill/plugin 应优先调用 runtime CLI，源码脚本用于仓库内交叉检查和后续官方 golden recordings 非 strict 校准；current segment 与 declared path strict gate 是 OCU 产物一致性约束，字段最终仍待官方 successful recording 校准。 +- OCU 已新增 runtime CLI `open-computer-use event-stream summarize --json` 和源码脚本 `scripts/summarize-event-stream-recording.py` 作为录制后可消费性审计工具，会提取 event type、window、action sequence、target/focused element、selection、debug、redaction evidence 和 OCU 扩展 `skillReadiness`；默认只输出 AX `value` / `selectedText` 长度，不输出原文。这是生成 skill 前的 OCU 辅助入口，不代表已确认官方 summary schema。 +- OCU 已新增 runtime CLI `open-computer-use event-stream scaffold-skill --json`，可在安装态把 summary 生成第一版带 workflow readiness 的 `SKILL.md` 草稿和脱敏 `references/recording-summary.json`；源码 checkout 里的 `scripts/scaffold-event-stream-skill.py` 保留为开发交叉检查。这是独立 repo / skill 集成的辅助脚手架，不属于官方 `event_stream_*` MCP surface，也不替代 agent 对 `events.jsonl` 的人工审查和业务语义整理。 +- OCU 已新增 `scripts/import-event-stream-fixture.py` 作为官方 golden recording 入库前的脱敏导入入口，默认改写本机路径、session id、timestamp、app attribution、窗口标题、AX value / selectedText，并移除 screenshot path；事件和 suppressed event 中的通用 `path` / 未知 `*Path` 字符串会改成 redacted marker 和长度，避免本机路径通过非标准字段漏进 fixture；如果输入是 official hosted 最小 `events.jsonl + session.json` 目录，导入器会把 `session.json.id` 脱敏为 fixture session id，合成 fixture `metadata.json` 和空 `suppressed.jsonl`，并保留官方风格 `session.json` handoff，保证后续 `--require-session-alias` / `--require-metadata-counts` / `--require-handoff-paths` readiness 可以直接消费；`--mcp-transcript ` 会额外写入脱敏 `mcp-transcript.json`，把 MCP response text JSON 解码后只保留脱敏 shape；`--scenario ` 会写入 fixture manifest，供 official baseline set gate 与 OCU candidate 按同 scenario 配对；`make event-stream-fixture-smoke` 覆盖这条净化路径。 +- OCU 已新增 `scripts/ingest-official-record-and-replay-fixture.py` 作为 hosted official recording 入库包装入口；它可从 `event_stream_stop` / `event_stream_status` 返回 JSON、JSON-RPC `result.content[].text`、`--status-json -` stdin 或直接 `metadata.json` / `session.json` 里提取 handoff path，调用 import 脚本脱敏入库，并按 scenario 运行 golden readiness。传入 `--mcp-transcript` 和 `--require-mcp-transcript-evidence` 时，会进一步要求 response-shape 证据；传入 `--use-status-json-as-transcript` 时，可把同一 status JSON 临时复用成 transcript；传入 `--check-fixture-set` 时，会用当前 output root 运行集合级 official fixture set gate。`make event-stream-official-fixture-ingest-smoke` 覆盖 nested hosted response、stdin 输入、脱敏、readiness 和 fixture set gate。 +- OCU 已新增 `scripts/ingest-ocu-record-and-replay-candidate.py` 作为 same-scenario candidate 入库入口；它可导入已有 OCU `--recording`，也可从 action smoke JSONL 的 `recordingsRoot` / `sessionId` 解析真实 session 目录并自动消费同一 JSON 中的 `mcpTranscriptPath`，或显式 `--run-action-smoke` 采集一次 opt-in 真实输入 smoke 后导入。action smoke 当前通过 proxied `event-stream mcp` start/repeat start/status/stop/repeat stop/final status 录制同一 session，因此输出的 transcript 可作为 candidate response-shape evidence；手动保存 action smoke stdout 给 `--smoke-json` 时需要保留 smoke 临时目录，或直接使用 importer 的 `--run-action-smoke` 路径。`--run-action-smoke` 会按 `--scenario simple-action-stop|drag-stop` 选择同名 action smoke 场景，分别产生 click 或 drag candidate；其它非 keyboard scenario 回退到默认 `mixed-action-stop`，只作为通用真实监听链路证据。`keyboard-input-stop` 当前通过已有 recording 或保留的 smoke JSON 导入，因为 macOS 对 synthetic keyboard event 的 session tap 过滤在不同环境下不稳定。脚本会先跑 strict OCU validation，再调用 fixture import 生成 `source=ocu` fixture，随后按 scenario 跑 readiness；提供 `--official-root --check-fixture-set` 时，会把新 candidate 直接交给 official fixture set gate。若 official 样本要求 MCP response shape evidence，candidate 应传入或自动消费 `mcp-transcript.json` 并加 `--require-mcp-transcript-evidence`。 +- OCU 已新增 `scripts/check-event-stream-golden-readiness.py` 作为源码侧 golden gate，检查脱敏 fixture 的 manifest/source/version、final state、metadata/session handoff 兼容性、metadata event/suppressed counts、declared handoff path evidence、`session.started` 首事件 / 唯一性、`session.ended` 最终事件 / 唯一性、动作事件、AX payload，以及按需要求的 expected `endReason`、full/diff/cumulative AX payload、suppressed fallback evidence、`event_stream_start/status/stop` MCP response 证据和 `repeatStartResponseShape` / `repeatStopResponseShape` / `finalStatusResponseShape` 等 raw probe lifecycle response shape 证据；`--require-end-reason ` 会从 metadata/session handoff 和 `session.ended.endReason` 汇总校验，用于后续 stop / cancel / timeout 官方样本入库。gate 输出 `metadataSessionAliasComplete`、`metadataSessionAliasMatches`、`metadataEventCountMatches`、`metadataSuppressedEventCountMatches`、`declaredPaths`、`lastEventType`、`sessionEndedCount`、`sessionEndedIsFinal` 和 `hasFullTreePayload`，默认拒绝同时存在但语义不兼容的 `metadata.json` / `session.json`，默认拒绝已声明但与文件行数不一致的 `eventCount` / `suppressedEventCount`，默认拒绝已声明但不存在的 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath`，也默认拒绝多个 `session.ended` 或 `session.ended` 后继续写入事件的 malformed fixture；`--require-session-alias` 会进一步要求两份 handoff 文件都存在，`--require-metadata-counts` 会进一步要求 metadata count 字段存在，`--require-handoff-paths` 会进一步要求四个 handoff path 字段都存在且可解析，`--require-suppressed-events` 要求 `suppressed.jsonl` 非空，`--require-suppressed-event-type ` 要求指定 suppressed event type 存在。required MCP tool 的 timeout transcript 会明确失败为 `MCP tool timed out`，required response shape 的 timeout 会失败为 `MCP response shape timed out`，no-active status/stop 这类生命周期 fixture 必须显式放宽 action / AX 要求，避免结构合法但信息不足或只有 timeout 边界的样本被当成事件语义 golden。 +- OCU 已新增 `scripts/compare-event-stream-recordings.py` 作为官方 fixture 与 OCU candidate 的事件序列、metadata key diff、stable metadata value diff、declared handoff path evidence、final session evidence、JSON schema、suppressed stream、replay-critical semantic field、AX diff evidence 和可选 MCP response shape evidence 对比入口；`--require-same-metadata-keys` 可要求 candidate metadata key set 与 baseline 一致，`--require-same-metadata-values` 可要求 candidate 的 `state`、`active`、`endReason`、`eventCount`、`suppressedEventCount` 与 baseline 一致，避免 final state / endReason / count 漂移只因 key set 一致而漏掉；`--require-handoff-paths` 可要求 baseline 与 candidate 都声明 `metadataPath` / `eventsPath` / `sessionPath` / `suppressedEventsPath` 且解析后文件存在，避免 candidate 只保留 key set 却丢掉 skill 可消费路径；`--require-final-session-evidence` 可要求 candidate 保留 baseline 的 `session.ended` 个数、最后事件位置和 endReason 集合，避免录制闭环丢失；`--require-same-suppressed-event-sequence` 和 `--require-same-suppressed-schema` 可要求 candidate 保留 baseline `suppressed.jsonl` 中出现的 AX / screenshot / 压缩降级证据顺序和 schema；`--require-semantic-fields` 可要求 candidate 保留 baseline 中出现的 target AX、keyboard focused AX、drag 起止点、selection / terminal 摘要字段、window attribution、AX `fullTree`、AX `treeLines` / `cumulativeTreeLines`；当 baseline 与 candidate 都带 `mcp-transcript.json` 时，`--require-mcp-response-shapes` 可要求 candidate 保留 baseline 中出现的 start/repeat/status/stop/final response shape result/error/timeout 状态，`--require-same-mcp-response-schema` 可进一步要求 response schema path/type 不丢失。`make event-stream-compare-smoke` 覆盖相同样本通过、缺 metadata key、metadata value drift、缺 handoff path、handoff path 指向不存在文件、缺 final session evidence、缺 suppressed event/schema evidence、缺 semantic field、事件顺序漂移、缺 AX diff/cumulative evidence、缺 diff marker、缺 MCP response shape、response timeout 状态漂移和 response schema 缺失失败路径。 +- OCU 已新增 `scripts/check-event-stream-official-fixture-set.py` 作为 official golden fixture 集合级 gate；它要求 official fixture manifest 带 `scenario`，默认至少要求 `simple-action-stop` official successful recording，通过 `check-event-stream-golden-readiness.py` 跑 strict readiness。`simple-action-stop` 会要求 `mouse.click` 和 `recording_controls_stopped` endReason，`keyboard-input-stop` 会要求 `keyboard.text_input` 和 `recording_controls_stopped` endReason，`drag-stop` 会要求 `mouse.drag` 和 `recording_controls_stopped` endReason，`cancel` 会要求 `recording_controls_cancelled` endReason，`timeout` 在官方 endReason 未确认前只跑生命周期 readiness、不固定 endReason，并允许 no action / no AX payload。传入 `--candidate-root` 后，它会按同 scenario 找 OCU candidate，先用 `source=ocu` 跑同 scenario readiness，再调用 `compare-event-stream-recordings.py` 要求 event sequence、schema、metadata keys/values、handoff paths、final session evidence、semantic fields 和 MCP response shape 对齐。`make event-stream-official-fixture-set-smoke` 用临时 official / OCU fixtures 覆盖成功、缺 candidate、candidate alias 漂移、缺 scenario、stop endReason mismatch、keyboard / drag 成功录制策略、keyboard 缺失必需事件失败、cancel 成功、timeout lifecycle-only 样本和导入 `--scenario` 写 manifest；当前仓库的 required `simple-action-stop` official fixture 可通过该 gate，recommended 场景继续作为后续入库后的机械验收入口。 +- OCU candidate ingest 入口 `scripts/ingest-ocu-record-and-replay-candidate.py` 现在与集合级 gate 的 stopped action scenario 保持一致：`simple-action-stop` 要求 `mouse.click`，`keyboard-input-stop` 要求 `keyboard.text_input`，`drag-stop` 要求 `mouse.drag`；三者都会要求 stopped endReason、full AX payload，并在 `--require-mcp-transcript-evidence` 时要求 repeat start / repeat stop / final status response-shape evidence。这让 OCU candidate 在 official fixture 入库前也能先通过同 scenario readiness，而不是只等 official-vs-candidate gate 再发现 keyboard/drag 录制缺关键事件。 +- OCU 已新增 `scripts/check-event-stream-official-fixture-coverage.py` 作为仓库真实 official fixture 覆盖率 report；它默认要求 `simple-action-stop` official successful recording，缺失时返回非零，`--allow-missing` 会改成只输出 `coverageOk=false` / `missingOfficialScenarios` 供默认 CI 和 baseline smoke 记录当前缺口。该 report 同时输出 recommended coverage，默认清单是 `simple-action-stop`、`keyboard-input-stop`、`drag-stop`、`cancel`、`timeout`，并通过 `missingRecommendedOfficialScenarios` 指引下一批 official golden 采集；recommended coverage 不影响退出码。传入 `--check-readiness` 时，required scenario 存在后还会跑 official fixture set readiness 并输出 `requiredOfficialReadinessOk`；如果 scenario manifest 已存在但 readiness 不合格，会在顶层 `notReadyOfficialScenarios` 里列出对应场景。传入 `--require-readiness` 时，只有 required scenario 覆盖和 readiness 都通过才算 `coverageOk=true` / `hasRequiredOfficialSuccessfulFixture=true`。 +- OCU baseline 聚合入口 `scripts/run-record-and-replay-baseline-smoke.sh` 现在同时运行 hosted official fixture ingest smoke 与 OCU candidate ingest smoke，并把结果写入 `evidence.fixtureIngestPipelines`；该 evidence 也要求 official hosted JSON 只给 `sessionDirectoryPath` 时可导入官方最小 session 目录。这证明导入/脱敏/配对/候选 readiness 管线可复跑；真实 official recording 是否齐备以 coverage report 的 required/recommended scenario 覆盖、required readiness 以及 official fixture set gate 为准。 +- OCU baseline 聚合入口现在也运行截图上下文 smoke，并把结果写入 `evidence.screenshotContextSmoke`；这证明 `always` policy 会让 AX payload 带 `screenshotNeededForContext`，且当前环境返回截图数据时 `screenshotPath` 会被校验存在。它仍不是官方截图触发阈值或 suppressed 行为的 golden 校准，后续仍需 official recording fixture。 +- OCU 已新增 `scripts/probe-event-stream-recording.py` 作为 official golden recording 采集前的 raw MCP 探测入口；local OCU start/repeat start/status/stop/repeat stop/final status probe 可用，并由 `scripts/test-event-stream-local-probe.py` 固定 generated fixture 脱敏、one-active response、idempotent stop、completed status、active current segment response 和 final cleanup response；官方 bundled client 在宿主外只读 surface probe 可用，raw start/status/stop 当前超时。 + +仍未完成的官方级校准： + +- start approval gate、OCU 自有开始确认弹窗和 MCP elicitation approval 已有 baseline；Codex.app host form schema 和 response action 已确认，官方 Record & Replay 真实业务 message / `_meta` 仍待样本确认。 +- 需要继续通过正常 Codex 宿主流程或已满足官方 runtime 宿主状态的环境采集 recommended official golden recordings，确认 keyboard、drag、cancel、timeout 等场景的完整 JSON schema 和字段顺序；当前 raw 官方 client start/status/stop probe 仍超时。 +- 需要采集达到或模拟最长时长的官方样本，确认 timeout completion 的 state / endReason / metadata 差异。 +- AX compact diff 当前是官方风格的轻量 previous / cumulative 实现，还不是已确认等价的 `UIElementRenderDifference` 算法。 +- 点击/键盘目标摘要字段名和内容仍是 OCU baseline，需要 official golden recordings 校准官方是否使用同名字段、是否有更完整的 raw event 结构。 +- `mouse.drag` 字段名、距离阈值、`terminal.value_changed` 字段与触发时机、`selection.changed` 字段与清空选择语义、`window.changed` 字段与初始帧语义、`debug.error` 字段语义、Input Monitoring failure debug payload、`secureInput` 字段语义、raw payload 结构，以及可能存在的独立 scroll event schema 仍待官方样本校准。 +- 截图上下文触发条件、路径字段和 suppressed 行为仍只基于已观察字符串与 OCU smoke 验证。 diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift index 8b7b6d92..e390465a 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/AccessibilitySnapshot.swift @@ -95,7 +95,11 @@ public enum SnapshotTextStyle { } enum SnapshotBuilder { - static func build(for app: RunningAppDescriptor, showFullText: Bool = false) throws -> AppSnapshot { + static func build( + for app: RunningAppDescriptor, + showFullText: Bool = false, + includeScreenshot: Bool = true + ) throws -> AppSnapshot { if app.name == FixtureBridge.appName, let fixtureState = try FixtureBridge.readState() { return buildFixtureSnapshot(app: app, state: fixtureState) } @@ -122,13 +126,13 @@ enum SnapshotBuilder { rootWindow = resolvedFocusedWindow var windowTitle = stringValue(of: rootWindow, attribute: kAXTitleAttribute) - var windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle) + var windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle, includeScreenshot: includeScreenshot) if windowCapture == nil, recoverVisibleWindow(for: app, appElement: appElement, preferredWindow: rootWindow) { focusedApplication = copyElement(systemWide, attribute: kAXFocusedApplicationAttribute) if let recoveredWindow = preferredFocusedWindow(appElement: appElement, appPID: app.pid, focusedApplication: focusedApplication, systemWide: systemWide) { rootWindow = recoveredWindow windowTitle = stringValue(of: recoveredWindow, attribute: kAXTitleAttribute) - windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle) + windowCapture = WindowCapture.resolve(for: app.pid, titleHint: windowTitle, includeScreenshot: includeScreenshot) } } @@ -363,7 +367,7 @@ private struct WindowCapture { let bounds: CGRect let image: CGImage? - static func resolve(for pid: pid_t, titleHint: String?) -> WindowCapture? { + static func resolve(for pid: pid_t, titleHint: String?, includeScreenshot: Bool = true) -> WindowCapture? { guard let infoList = CGWindowListCopyWindowInfo([.optionOnScreenOnly], kCGNullWindowID) as? [[String: Any]] else { return nil } @@ -396,7 +400,7 @@ private struct WindowCapture { return nil } - let image = captureImage(windowID: best.windowID, bounds: best.bounds) + let image = includeScreenshot ? captureImage(windowID: best.windowID, bounds: best.bounds) : nil return WindowCapture(windowID: best.windowID, layer: best.layer, bounds: best.bounds, image: image) } diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamMCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamMCPServer.swift new file mode 100644 index 00000000..76c29e5d --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamMCPServer.swift @@ -0,0 +1,306 @@ +import Foundation + +public typealias EventStreamMCPStartApprovalRequester = () throws -> EventStreamStartApprovalDecision + +public final class EventStreamMCPServer: LineBasedMCPServer { + private let dispatcher: EventStreamToolDispatcher + private var clientSupportsElicitation = false + private var nextRequestID = 1000 + + public init(service: EventStreamService = .shared) { + self.dispatcher = EventStreamToolDispatcher(service: service) + } + + public func run() throws { + while let line = readLine(strippingNewline: true) { + guard !line.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { + continue + } + + if let response = handle(line: line, startApprovalRequester: requestStartApprovalViaMCP) { + FileHandle.standardOutput.write((response + "\n").data(using: .utf8)!) + } + } + } + + public func handle(line: String) -> String? { + handle(line: line, startApprovalRequester: nil) + } + + public func handle(line: String, startApprovalRequester: EventStreamMCPStartApprovalRequester?) -> String? { + do { + guard let payload = try JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] else { + return try encodeJSONRPCError(id: nil, code: -32700, message: "Invalid JSON-RPC payload") + } + + let method = payload["method"] as? String + let id = payload["id"] + let params = payload["params"] as? [String: Any] ?? [:] + + switch method { + case "initialize": + let capabilities = params["capabilities"] as? [String: Any] ?? [:] + clientSupportsElicitation = capabilities["elicitation"] is [String: Any] + return try encodeJSONRPCResult( + id: id, + result: [ + "protocolVersion": "2025-11-25", + "serverInfo": [ + "name": "Record & Replay", + "version": openComputerUseVersion, + ], + "capabilities": [ + "tools": [ + "listChanged": false, + ], + ], + ] + ) + case "notifications/initialized": + return nil + case "ping": + return try encodeJSONRPCResult(id: id, result: [:]) + case "tools/list": + return try encodeJSONRPCResult( + id: id, + result: [ + "tools": EventStreamToolDefinitions.all.map(\.asDictionary), + ] + ) + case "tools/call": + let toolParams = try parseToolCallParams(payload: payload) + let name = try parseToolName(params: toolParams) + let arguments = try parseToolArguments(params: toolParams) + let approvalDecision = try startApprovalDecisionIfNeeded( + toolName: name, + requester: startApprovalRequester + ) + let result = try dispatcher.callTool( + name: name, + arguments: arguments, + startApprovalDecision: approvalDecision + ) + return try encodeJSONRPCResult( + id: id, + result: result.asDictionary + ) + default: + if method == nil { + return nil + } + + return try encodeJSONRPCError(id: id, code: -32601, message: "Method not found: \(method ?? "")") + } + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error) + let payload = try? JSONSerialization.jsonObject(with: Data(line.utf8)) as? [String: Any] + let id = payload?["id"] + return try? encodeJSONRPCResult( + id: id, + result: ToolCallResult.text(message, isError: true).asDictionary + ) + } + } + + private func parseToolCallParams(payload: [String: Any]) throws -> [String: Any] { + guard let params = payload["params"] as? [String: Any] else { + throw ComputerUseError.invalidArguments("tools/call params must be an object") + } + return params + } + + private func parseToolName(params: [String: Any]) throws -> String { + guard let name = params["name"] as? String, !name.isEmpty else { + throw ComputerUseError.invalidArguments("tools/call params.name must be a non-empty string") + } + return name + } + + private func parseToolArguments(params: [String: Any]) throws -> [String: Any] { + guard let rawArguments = params["arguments"] else { + return [:] + } + guard let arguments = rawArguments as? [String: Any] else { + throw ComputerUseError.invalidArguments("tools/call arguments must be an object") + } + return arguments + } + + private func startApprovalDecisionIfNeeded( + toolName: String, + requester: EventStreamMCPStartApprovalRequester? + ) throws -> EventStreamStartApprovalDecision? { + guard toolName == "event_stream_start" else { + return nil + } + guard dispatcher.requiresMCPStartApproval else { + return nil + } + guard !dispatcher.status().isActive else { + return nil + } + guard clientSupportsElicitation, let requester else { + return .cancelled + } + return try requester() + } + + private func requestStartApprovalViaMCP() throws -> EventStreamStartApprovalDecision { + let requestID = "event-stream-start-approval-\(nextRequestID)" + nextRequestID += 1 + let request = try encode(Self.startApprovalElicitationRequest(id: requestID)) + FileHandle.standardOutput.write((request + "\n").data(using: .utf8)!) + + guard let responseLine = readLine(strippingNewline: true), + let response = try JSONSerialization.jsonObject(with: Data(responseLine.utf8)) as? [String: Any] else { + return .cancelled + } + if response["error"] != nil { + return .cancelled + } + guard let result = response["result"] as? [String: Any], + let action = result["action"] as? String else { + return .cancelled + } + switch action { + case "accept": + return .approved + case "decline", "reject": + return .denied + case "cancel": + return .cancelled + default: + return .cancelled + } + } + + private func encodeJSONRPCResult(id: Any?, result: [String: Any]) throws -> String { + try encode([ + "jsonrpc": "2.0", + "id": id ?? NSNull(), + "result": result, + ]) + } + + private func encodeJSONRPCError(id: Any?, code: Int, message: String) throws -> String { + try encode([ + "jsonrpc": "2.0", + "id": id ?? NSNull(), + "error": [ + "code": code, + "message": message, + ], + ]) + } + + private func encode(_ object: [String: Any]) throws -> String { + let data = try JSONSerialization.data(withJSONObject: object, options: [.withoutEscapingSlashes]) + guard let text = String(data: data, encoding: .utf8) else { + throw ComputerUseError.message("Failed to encode JSON-RPC response.") + } + + return text + } + + static func startApprovalElicitationRequest(id requestID: String) -> [String: Any] { + [ + "jsonrpc": "2.0", + "id": requestID, + "method": "elicitation/create", + "params": [ + "mode": "form", + "message": "Open Computer Use wants to start Record & Replay and record your actions.", + "requestedSchema": [ + "type": "object", + "properties": [:], + "required": [], + ], + ], + ] + } +} + +public final class EventStreamToolDispatcher { + private let service: EventStreamService + + public init(service: EventStreamService = .shared) { + self.service = service + } + + public var requiresMCPStartApproval: Bool { + service.configuredStartApprovalPolicy == .mcpElicitation + } + + public func status() -> EventStreamRecordingStatus { + service.status() + } + + public func callTool( + name: String, + arguments: [String: Any], + startApprovalDecision: EventStreamStartApprovalDecision? = nil + ) throws -> ToolCallResult { + try validateNoArguments(toolName: name, arguments: arguments) + + let status: EventStreamRecordingStatus + switch name { + case "event_stream_start": + status = try service.start(approvalDecision: startApprovalDecision) + case "event_stream_status": + status = service.status() + case "event_stream_stop": + status = try service.stop() + default: + throw ComputerUseError.unsupportedTool(name) + } + + return ToolCallResult.text(try mcpStatusText(status)) + } + + private func validateNoArguments(toolName: String, arguments: [String: Any]) throws { + guard !arguments.isEmpty else { + return + } + + switch toolName { + case "event_stream_start", "event_stream_status", "event_stream_stop": + throw ComputerUseError.invalidArguments("\(toolName) does not accept arguments") + default: + return + } + } + + private func mcpStatusText(_ status: EventStreamRecordingStatus) throws -> String { + let data = try JSONSerialization.data( + withJSONObject: mcpStatusDictionary(status), + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + guard let text = String(data: data, encoding: .utf8) else { + throw ComputerUseError.message("Failed to encode event stream status as JSON.") + } + return text + } + + private func mcpStatusDictionary(_ status: EventStreamRecordingStatus) -> [String: Any] { + let maximumDurationSeconds = service.configuredMaximumDurationSeconds + if status.state == .idle, status.sessionID == nil { + return [ + "isRecording": false, + "maxDurationSeconds": maximumDurationSeconds, + ] + } + + var dictionary = status.asDictionary + dictionary["isRecording"] = status.isActive + dictionary["maxDurationSeconds"] = maximumDurationSeconds + if let metadataPath = status.metadataPath { + let metadataURL = URL(fileURLWithPath: metadataPath) + let sessionDirectoryURL = metadataURL.deletingLastPathComponent() + let sessionURL = sessionDirectoryURL.appendingPathComponent("session.json") + dictionary["metadataPath"] = sessionURL.path + dictionary["sessionPath"] = sessionURL.path + dictionary["sessionDirectoryPath"] = sessionDirectoryURL.path + } + return dictionary + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingControls.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingControls.swift new file mode 100644 index 00000000..97e4f12b --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingControls.swift @@ -0,0 +1,190 @@ +import AppKit +import Foundation + +@MainActor +final class EventStreamRecordingControls: NSObject { + private let panel: NSPanel + private let timerLabel: NSTextField + private let startedAt: Date + private let onStop: () -> Void + private let onCancel: () -> Void + private var timer: Timer? + + init(startedAt: Date, onStop: @escaping () -> Void, onCancel: @escaping () -> Void) { + self.startedAt = startedAt + self.onStop = onStop + self.onCancel = onCancel + + let size = NSSize(width: 520, height: 54) + self.panel = NSPanel( + contentRect: NSRect(origin: .zero, size: size), + styleMask: [.titled, .fullSizeContentView, .nonactivatingPanel], + backing: .buffered, + defer: false + ) + self.timerLabel = NSTextField(labelWithString: "00:00") + + super.init() + + configurePanel(size: size) + configureContent(size: size) + } + + func show() { + updateElapsedTime() + timer = Timer.scheduledTimer( + timeInterval: 0.5, + target: self, + selector: #selector(updateElapsedTime), + userInfo: nil, + repeats: true + ) + panel.orderFrontRegardless() + } + + func close() { + timer?.invalidate() + timer = nil + panel.orderOut(nil) + } + + func performControlButtonForTesting(title: String) -> Bool { + guard let contentView = panel.contentView, + let button = firstButton(in: contentView, title: title) + else { + return false + } + + button.performClick(nil) + return true + } + + private func configurePanel(size: NSSize) { + panel.title = "Record & Replay Recording Controls" + panel.titleVisibility = .hidden + panel.titlebarAppearsTransparent = true + panel.isOpaque = false + panel.backgroundColor = .clear + panel.hasShadow = true + panel.isMovableByWindowBackground = true + panel.hidesOnDeactivate = false + panel.level = .statusBar + panel.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .transient] + panel.standardWindowButton(.closeButton)?.isHidden = true + panel.standardWindowButton(.miniaturizeButton)?.isHidden = true + panel.standardWindowButton(.zoomButton)?.isHidden = true + + let screenFrame = (NSScreen.main ?? NSScreen.screens.first)?.visibleFrame ?? .zero + panel.setFrame(eventStreamRecordingControlsFrame(panelSize: size, visibleFrame: screenFrame), display: false) + } + + private func configureContent(size: NSSize) { + let material = NSVisualEffectView(frame: NSRect(origin: .zero, size: size)) + material.material = .hudWindow + material.blendingMode = .behindWindow + material.state = .active + material.wantsLayer = true + material.layer?.cornerRadius = 14 + material.layer?.masksToBounds = true + + let dot = NSView() + dot.translatesAutoresizingMaskIntoConstraints = false + dot.wantsLayer = true + dot.layer?.backgroundColor = NSColor.systemRed.cgColor + dot.layer?.cornerRadius = 5 + dot.setContentHuggingPriority(.required, for: .horizontal) + dot.setContentHuggingPriority(.required, for: .vertical) + + let titleLabel = NSTextField(labelWithString: "Record & Replay is recording your actions") + titleLabel.translatesAutoresizingMaskIntoConstraints = false + titleLabel.font = NSFont.systemFont(ofSize: 13, weight: .semibold) + titleLabel.textColor = .labelColor + titleLabel.lineBreakMode = .byTruncatingTail + + timerLabel.translatesAutoresizingMaskIntoConstraints = false + timerLabel.font = NSFont.monospacedDigitSystemFont(ofSize: 12, weight: .medium) + timerLabel.textColor = .secondaryLabelColor + timerLabel.alignment = .right + timerLabel.setContentHuggingPriority(.required, for: .horizontal) + + let doneButton = NSButton(title: "Done", target: self, action: #selector(handleStop)) + doneButton.translatesAutoresizingMaskIntoConstraints = false + doneButton.bezelStyle = .rounded + doneButton.controlSize = .small + doneButton.toolTip = "Stop recording" + + let discardButton = NSButton(title: "Discard", target: self, action: #selector(handleCancel)) + discardButton.translatesAutoresizingMaskIntoConstraints = false + discardButton.bezelStyle = .rounded + discardButton.controlSize = .small + discardButton.toolTip = "Discard recording" + + let stack = NSStackView(views: [dot, titleLabel, timerLabel, doneButton, discardButton]) + stack.translatesAutoresizingMaskIntoConstraints = false + stack.orientation = .horizontal + stack.alignment = .centerY + stack.spacing = 10 + stack.edgeInsets = NSEdgeInsets(top: 0, left: 16, bottom: 0, right: 16) + + material.addSubview(stack) + panel.contentView = material + + NSLayoutConstraint.activate([ + dot.widthAnchor.constraint(equalToConstant: 10), + dot.heightAnchor.constraint(equalToConstant: 10), + timerLabel.widthAnchor.constraint(equalToConstant: 48), + doneButton.widthAnchor.constraint(equalToConstant: 74), + discardButton.widthAnchor.constraint(equalToConstant: 86), + stack.leadingAnchor.constraint(equalTo: material.leadingAnchor), + stack.trailingAnchor.constraint(equalTo: material.trailingAnchor), + stack.topAnchor.constraint(equalTo: material.topAnchor), + stack.bottomAnchor.constraint(equalTo: material.bottomAnchor), + ]) + } + + private func firstButton(in view: NSView, title: String) -> NSButton? { + if let button = view as? NSButton, button.title == title { + return button + } + + for subview in view.subviews { + if let button = firstButton(in: subview, title: title) { + return button + } + } + + return nil + } + + @objc + private func handleStop() { + onStop() + } + + @objc + private func handleCancel() { + onCancel() + } + + @objc + private func updateElapsedTime() { + let elapsed = max(Int(Date().timeIntervalSince(startedAt)), 0) + let minutes = elapsed / 60 + let seconds = elapsed % 60 + timerLabel.stringValue = String(format: "%02d:%02d", minutes, seconds) + } +} + +func eventStreamRecordingControlsFrame( + panelSize: NSSize, + visibleFrame: NSRect, + topInset: CGFloat = 18 +) -> NSRect { + let proposedX = visibleFrame.midX - panelSize.width / 2 + let maxX = max(visibleFrame.minX, visibleFrame.maxX - panelSize.width) + let originX = min(max(proposedX, visibleFrame.minX), maxX) + let proposedY = visibleFrame.maxY - panelSize.height - topInset + let maxY = max(visibleFrame.minY, visibleFrame.maxY - panelSize.height) + let originY = min(max(proposedY, visibleFrame.minY), maxY) + return NSRect(origin: NSPoint(x: originX, y: originY), size: panelSize) +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingSummary.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingSummary.swift new file mode 100644 index 00000000..9a5c9697 --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingSummary.swift @@ -0,0 +1,1006 @@ +import Foundation + +public struct EventStreamRecordingSummaryOptions: Equatable, Sendable { + public let inputPath: String + public let includeText: Bool + public let requireAction: Bool + + public init(inputPath: String, includeText: Bool = false, requireAction: Bool = false) { + self.inputPath = inputPath + self.includeText = includeText + self.requireAction = requireAction + } +} + +private let eventStreamSummaryActionEventTypes: Set = [ + "mouse.click", + "mouse.context_menu", + "mouse.drag", + "keyboard.text_input", + "keyboard.submit", + "keyboard.shortcut", + "terminal.value_changed", + "selection.changed", +] + +private let eventStreamSummaryContextEventTypes: Set = [ + "session.started", + "session.ended", + "window.changed", + "AX.focusedWindowChanged", + "debug.error", + "experimentalRawEvents", +] + +private let eventStreamSummarySafetyKeywords: [(String, String)] = [ + ("send", "sendAction"), + ("delete", "deleteAction"), + ("remove", "deleteAction"), + ("trash", "deleteAction"), + ("archive", "archiveAction"), + ("purchase", "purchaseAction"), + ("buy", "purchaseAction"), + ("pay", "paymentAction"), + ("approve", "approvalAction"), + ("upload", "uploadAction"), + ("publish", "publishAction"), + ("share", "shareAction"), + ("invite", "inviteAction"), + ("submit", "submitAction"), + ("save", "saveAction"), +] + +private let eventStreamSummaryBlockingDiagnosticReasons: Set = [ + "inputMonitorsUnavailable", +] + +private let eventStreamSummaryActionSequenceLimit = 50 +private let eventStreamSummaryRuntimeInputsLimit = 50 +private let eventStreamSummarySafetySignalsLimit = 50 +private let eventStreamSummaryElementListLimit = 25 +private let eventStreamSummaryDiagnosticsLimit = 25 +private let eventStreamSummaryScreenshotPathsLimit = 25 + +public func summarizeEventStreamRecording( + options: EventStreamRecordingSummaryOptions +) throws -> [String: Any] { + let resolved = try resolveEventStreamSummaryInput(path: options.inputPath) + let events = try readEventStreamSummaryJSONL(resolved.eventsURL) + + var eventTypeCounts: [String: Int] = [:] + var actionEvents: [[String: Any]] = [] + var contextEventCount = 0 + var windows: [[String: Any]] = [] + var seenWindowKeys: Set = [] + var actionSequence: [[String: Any]] = [] + var targetElements: [[String: Any]] = [] + var focusedElements: [[String: Any]] = [] + var selectionEvents: [[String: Any]] = [] + var debugErrors: [[String: Any]] = [] + var blockingDiagnostics: [[String: Any]] = [] + var redactionEvents: [[String: Any]] = [] + var screenshotPaths: [String] = [] + var runtimeInputs: [[String: Any]] = [] + var safetySignals: [[String: Any]] = [] + var sessionEndReasons: Set = [] + var targetElementCount = 0 + var focusedElementCount = 0 + var selectionEventCount = 0 + var debugErrorCount = 0 + var redactionEventCount = 0 + var screenshotPathCount = 0 + var runtimeInputCount = 0 + var safetySignalCount = 0 + + for (offset, event) in events.enumerated() { + let line = offset + 1 + let eventType = eventStreamSummaryEventKind(event) + if let eventType { + eventTypeCounts[eventType, default: 0] += 1 + if eventStreamSummaryContextEventTypes.contains(eventType) { + contextEventCount += 1 + } + } + + if let window = eventStreamSummaryWindow(from: event) { + let key = [ + window["bundleIdentifier"] as? String ?? "", + window["appName"] as? String ?? "", + window["windowTitle"] as? String ?? "", + ].joined(separator: "|") + if !seenWindowKeys.contains(key) { + seenWindowKeys.insert(key) + windows.append(window) + } + } + + if let actionType = eventStreamSummaryActionType(for: event) { + actionEvents.append(event) + var action: [String: Any] = [ + "line": line, + "type": actionType, + ] + if let timestamp = event["timestamp"] as? String { + action["timestamp"] = timestamp + } + if let window = eventStreamSummaryWindow(from: event) { + action["window"] = window + } + for key in ["location", "startLocation", "endLocation", "key", "modifiers", "selectionCleared", "reason"] { + if let value = event[key] { + action[key] = value + } + } + if let rawEvents = event["experimentalRawEvents"] as? [[String: Any]] { + let rawEventTypes = rawEvents.compactMap { $0["eventType"] as? String } + if !rawEventTypes.isEmpty { + action["rawEventTypes"] = Array(rawEventTypes.prefix(8)) + } + if let scroll = rawEvents.first(where: { $0["eventType"] as? String == "scrollWheel" }) { + for key in ["scrollingDeltaX", "scrollingDeltaY", "hasPreciseScrollingDeltas"] { + if let value = scroll[key] { + action[key] = value + } + } + } + } + if let text = event["text"] as? String { + action["textLength"] = text.count + } else if let textLength = event["textLength"] { + action["textLength"] = textLength + } + if let element = eventStreamSummaryElement(from: event, includeText: options.includeText) { + action["element"] = element + } + if let runtimeInput = eventStreamSummaryRuntimeInput( + line: line, + event: event, + action: action + ) { + runtimeInputCount += 1 + if runtimeInputs.count < eventStreamSummaryRuntimeInputsLimit { + runtimeInputs.append(runtimeInput) + } + } + if let safetySignal = eventStreamSummarySafetySignal( + line: line, + event: event, + action: action + ) { + safetySignalCount += 1 + if safetySignals.count < eventStreamSummarySafetySignalsLimit { + safetySignals.append(safetySignal) + } + } + if actionSequence.count < eventStreamSummaryActionSequenceLimit { + actionSequence.append(action) + } + } + + if let target = event["targetAccessibilityElement"] as? [String: Any] { + targetElementCount += 1 + let wrapper = ["targetAccessibilityElement": target] + if targetElements.count < eventStreamSummaryElementListLimit { + targetElements.append(eventStreamSummaryElement(from: wrapper, includeText: options.includeText) ?? [:]) + } + } + if let focused = event["focusedAccessibilityElement"] as? [String: Any] { + focusedElementCount += 1 + let wrapper = ["focusedAccessibilityElement": focused] + if focusedElements.count < eventStreamSummaryElementListLimit { + focusedElements.append(eventStreamSummaryElement(from: wrapper, includeText: options.includeText) ?? [:]) + } + } + + if eventType == "selection.changed" { + selectionEventCount += 1 + let selectedText = event["selectedText"] as? String + if selectionEvents.count < eventStreamSummaryElementListLimit { + selectionEvents.append([ + "line": line, + "selectedTextLength": selectedText?.count ?? 0, + "selectionCleared": event["selectionCleared"] as? Bool == true, + ]) + } + } + + if eventType == "session.ended", let endReason = event["endReason"] as? String, !endReason.isEmpty { + sessionEndReasons.insert(endReason) + } + + if eventType == "debug.error" { + if eventStreamSummaryIsBlockingDiagnostic(event) { + blockingDiagnostics.append(eventStreamSummaryFilteredEvent( + line: line, + event: event, + keys: ["subsystem", "reason", "errorType"] + )) + } + debugErrorCount += 1 + if debugErrors.count < eventStreamSummaryDiagnosticsLimit { + debugErrors.append(eventStreamSummaryFilteredEvent( + line: line, + event: event, + keys: ["subsystem", "reason", "errorType"] + )) + } + } + + if event["secureInput"] as? Bool == true || event["redacted"] as? Bool == true { + redactionEventCount += 1 + if redactionEvents.count < eventStreamSummaryDiagnosticsLimit { + var redaction: [String: Any] = [ + "line": line, + "secureInput": event["secureInput"] as? Bool == true, + "redacted": event["redacted"] as? Bool == true, + ] + if let eventType { + redaction["type"] = eventType + } + redactionEvents.append(redaction) + } + } + + if let payload = event["accessibilityInspectorPayload"] as? [String: Any], + let screenshotPath = payload["screenshotPath"] as? String, + !screenshotPath.isEmpty + { + screenshotPathCount += 1 + if screenshotPaths.count < eventStreamSummaryScreenshotPathsLimit { + screenshotPaths.append(screenshotPath) + } + } + } + + let summaryLimits = eventStreamSummaryLimits( + actionEventCount: actionEvents.count, + actionSequenceStored: actionSequence.count, + runtimeInputCount: runtimeInputCount, + runtimeInputsStored: runtimeInputs.count, + safetySignalCount: safetySignalCount, + safetySignalsStored: safetySignals.count, + targetElementCount: targetElementCount, + targetElementsStored: targetElements.count, + focusedElementCount: focusedElementCount, + focusedElementsStored: focusedElements.count, + selectionEventCount: selectionEventCount, + selectionEventsStored: selectionEvents.count, + debugErrorCount: debugErrorCount, + debugErrorsStored: debugErrors.count, + redactionEventCount: redactionEventCount, + redactionEventsStored: redactionEvents.count, + screenshotPathCount: screenshotPathCount, + screenshotPathsStored: screenshotPaths.count + ) + let hasTruncatedSummary = summaryLimits["hasTruncatedSummary"] as? Bool == true + + let inferredEndReason = (resolved.metadata["endReason"] as? String) + ?? (sessionEndReasons.count == 1 ? sessionEndReasons.first : nil) + let recordingWasCancelled = (resolved.metadata["state"] as? String) == "cancelled" + || inferredEndReason == "recording_controls_cancelled" + let recordingIncomplete = eventStreamSummaryRecordingIncomplete( + metadata: resolved.metadata, + eventTypeCounts: eventTypeCounts + ) + let firstEventType = events.first.flatMap(eventStreamSummaryEventKind) + let sessionStartedIsFirst = firstEventType == "session.started" + let sessionStartedNotFirst = eventTypeCounts["session.started", default: 0] > 0 && !sessionStartedIsFirst + let sessionStartedCountInvalid = eventTypeCounts["session.started", default: 0] != 1 + let finalEventType = events.last.flatMap(eventStreamSummaryEventKind) + let sessionEndedIsFinal = finalEventType == "session.ended" + let sessionEndedNotFinal = eventTypeCounts["session.ended", default: 0] > 0 && !sessionEndedIsFinal + let sessionEndedCountInvalid = eventTypeCounts["session.ended", default: 0] > 1 + + var warnings: [String] = [] + if actionEvents.isEmpty { + warnings.append("recording has no high-level user action events") + } + if recordingWasCancelled { + warnings.append("recording was cancelled; do not create or update a skill from this event stream") + } + if recordingIncomplete { + warnings.append("recording is not complete; stop the recording before creating a skill") + } + if sessionStartedCountInvalid { + warnings.append("recording must contain exactly one session.started event before creating a skill") + } + if sessionStartedNotFirst { + warnings.append("recording has events before session.started; start must be the first event before creating a skill") + } + if sessionEndedNotFinal { + warnings.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + } + if sessionEndedCountInvalid { + warnings.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + } + if eventTypeCounts["AX.focusedWindowChanged", default: 0] == 0 { + warnings.append("recording has no AX focused window context") + } + if !debugErrors.isEmpty { + warnings.append("recording includes debug.error events; inspect diagnostics before creating a skill") + } + if !blockingDiagnostics.isEmpty { + warnings.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + } + if !redactionEvents.isEmpty { + warnings.append("recording includes redaction or secureInput signals; avoid copying sensitive values") + } + if !safetySignals.isEmpty { + warnings.append("recording includes actions that may require explicit user confirmation") + } + if hasTruncatedSummary { + warnings.append("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill") + } + + var errors: [String] = [] + if options.requireAction, actionEvents.isEmpty { + errors.append("required at least one high-level user action event") + } + if options.requireAction, recordingWasCancelled { + errors.append("recording was cancelled; do not create or update a skill from this event stream") + } + if options.requireAction, recordingIncomplete { + errors.append("recording is not complete; stop the recording before creating a skill") + } + if options.requireAction, sessionStartedCountInvalid { + errors.append("recording must contain exactly one session.started event before creating a skill") + } + if options.requireAction, sessionStartedNotFirst { + errors.append("recording has events before session.started; start must be the first event before creating a skill") + } + if options.requireAction, sessionEndedNotFinal { + errors.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + } + if options.requireAction, sessionEndedCountInvalid { + errors.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + } + if options.requireAction, !blockingDiagnostics.isEmpty { + errors.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + } + + let skillEvidence = eventStreamSummarySkillEvidence( + eventTypeCounts: eventTypeCounts, + hasActionEvents: !actionEvents.isEmpty, + hasTargetElements: !targetElements.isEmpty, + hasFocusedElements: !focusedElements.isEmpty, + hasSelectionSignals: !selectionEvents.isEmpty, + hasScreenshots: !screenshotPaths.isEmpty, + hasDebugErrors: !debugErrors.isEmpty, + hasBlockingDiagnostics: !blockingDiagnostics.isEmpty, + recordingIncomplete: recordingIncomplete, + sessionStartedNotFirst: sessionStartedNotFirst, + sessionStartedCountInvalid: sessionStartedCountInvalid, + sessionEndedNotFinal: sessionEndedNotFinal, + sessionEndedCountInvalid: sessionEndedCountInvalid, + hasRedactionSignals: !redactionEvents.isEmpty, + hasSafetySignals: !safetySignals.isEmpty, + hasTruncatedSummary: hasTruncatedSummary + ) + + var result: [String: Any] = [ + "ok": errors.isEmpty, + "sessionDir": resolved.sessionDirectory.path, + "eventsPath": resolved.eventsURL.path, + "eventCount": events.count, + "eventTypes": eventTypeCounts.sortedDictionary(), + "actionEventCount": actionEvents.count, + "contextEventCount": contextEventCount, + "windows": windows, + "skillEvidence": skillEvidence, + "skillReadiness": eventStreamSummarySkillReadiness( + evidence: skillEvidence, + hasWindowContext: !windows.isEmpty, + recordingWasCancelled: recordingWasCancelled, + recordingIncomplete: recordingIncomplete, + sessionStartedNotFirst: sessionStartedNotFirst, + sessionStartedCountInvalid: sessionStartedCountInvalid, + sessionEndedNotFinal: sessionEndedNotFinal, + sessionEndedCountInvalid: sessionEndedCountInvalid + ), + "summaryLimits": summaryLimits, + "runtimeInputs": runtimeInputs, + "safetySignals": safetySignals, + "actionSequence": actionSequence, + "targetElements": targetElements, + "focusedElements": focusedElements, + "selectionEvents": selectionEvents, + "debugErrors": debugErrors, + "blockingDiagnostics": blockingDiagnostics, + "redactionEvents": redactionEvents, + "screenshotPaths": screenshotPaths, + "includesRawText": options.includeText, + "warnings": warnings, + "errors": errors, + "sessionStartedCount": eventTypeCounts["session.started", default: 0], + "sessionEndedCount": eventTypeCounts["session.ended", default: 0], + ] + if let firstEventType { + result["firstEventType"] = firstEventType + result["sessionStartedIsFirst"] = sessionStartedIsFirst + } + if let finalEventType { + result["finalEventType"] = finalEventType + result["sessionEndedIsFinal"] = sessionEndedIsFinal + } + + if let metadataURL = resolved.metadataURL { + result["metadataPath"] = metadataURL.path + } + if let sessionID = resolved.metadata["sessionId"] ?? resolved.metadata["sessionID"] { + result["sessionId"] = sessionID + } + if let state = resolved.metadata["state"] { + result["state"] = state + } + if let endReason = inferredEndReason { + result["endReason"] = endReason + } + + return result +} + +private struct EventStreamSummaryInput { + let sessionDirectory: URL + let metadataURL: URL? + let metadata: [String: Any] + let eventsURL: URL +} + +private func resolveEventStreamSummaryInput(path: String) throws -> EventStreamSummaryInput { + let url = URL(fileURLWithPath: path) + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { + throw ComputerUseError.message("Path does not exist: \(path)") + } + + if isDirectory.boolValue { + let metadataURL = url.appendingPathComponent("metadata.json") + let sessionURL = url.appendingPathComponent("session.json") + let resolvedMetadataURL: URL? + let metadata: [String: Any] + if FileManager.default.fileExists(atPath: metadataURL.path) { + resolvedMetadataURL = metadataURL + metadata = try readEventStreamSummaryJSONObject(metadataURL) + } else if FileManager.default.fileExists(atPath: sessionURL.path) { + resolvedMetadataURL = sessionURL + metadata = try readEventStreamSummaryJSONObject(sessionURL) + } else { + resolvedMetadataURL = nil + metadata = [:] + } + let eventsURL = eventStreamSummaryPath( + metadata["eventsPath"], + baseDirectory: url, + fallback: url.appendingPathComponent("events.jsonl") + ) + return EventStreamSummaryInput( + sessionDirectory: url, + metadataURL: resolvedMetadataURL, + metadata: metadata, + eventsURL: eventsURL + ) + } + + switch url.lastPathComponent { + case "metadata.json", "session.json": + let metadata = try readEventStreamSummaryJSONObject(url) + let sessionDirectory = url.deletingLastPathComponent() + let eventsURL = eventStreamSummaryPath( + metadata["eventsPath"], + baseDirectory: sessionDirectory, + fallback: sessionDirectory.appendingPathComponent("events.jsonl") + ) + return EventStreamSummaryInput( + sessionDirectory: sessionDirectory, + metadataURL: url, + metadata: metadata, + eventsURL: eventsURL + ) + case "events.jsonl": + return EventStreamSummaryInput( + sessionDirectory: url.deletingLastPathComponent(), + metadataURL: nil, + metadata: [:], + eventsURL: url + ) + default: + throw ComputerUseError.message("Input must be a session directory, metadata.json, session.json, or events.jsonl.") + } +} + +private func readEventStreamSummaryJSONObject(_ url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + guard let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ComputerUseError.message("Expected JSON object in \(url.path).") + } + return object +} + +private func readEventStreamSummaryJSONL(_ url: URL) throws -> [[String: Any]] { + let text = try String(contentsOf: url, encoding: .utf8) + var events: [[String: Any]] = [] + for (index, line) in text.split(separator: "\n", omittingEmptySubsequences: true).enumerated() { + let data = Data(line.utf8) + guard let event = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ComputerUseError.message("Expected JSON object in \(url.path):\(index + 1).") + } + events.append(event) + } + return events +} + +private func eventStreamSummaryPath(_ value: Any?, baseDirectory: URL, fallback: URL) -> URL { + guard let path = value as? String, !path.isEmpty else { + return fallback + } + if path.hasPrefix("/") { + return URL(fileURLWithPath: path) + } + return baseDirectory.appendingPathComponent(path) +} + +private func eventStreamSummaryWindow(from event: [String: Any]) -> [String: Any]? { + let app = firstNestedDictionary(event, keys: ["app", "application", "appContext"]) + let window = firstNestedDictionary(event, keys: ["window", "windowContext", "targetWindow"]) + let appName = firstString(event, keys: ["appName", "applicationName", "localizedName"]) + ?? firstString(app, keys: ["name", "appName", "localizedName"]) + let bundleIdentifier = firstString(event, keys: ["bundleIdentifier", "bundleId", "appBundleIdentifier"]) + ?? firstString(app, keys: ["bundleIdentifier", "bundleId"]) + let windowTitle = firstString(event, keys: ["windowTitle", "title"]) + ?? firstString(window, keys: ["title", "windowTitle"]) + + var result: [String: Any] = [:] + if let appName { + result["appName"] = appName + } + if let bundleIdentifier { + result["bundleIdentifier"] = bundleIdentifier + } + if let windowTitle { + result["windowTitle"] = windowTitle + } + return result.isEmpty ? nil : result +} + +private func eventStreamSummaryActionType(for event: [String: Any]) -> String? { + guard let eventType = eventStreamSummaryEventKind(event) else { + return nil + } + if eventStreamSummaryActionEventTypes.contains(eventType) { + return eventType + } + if eventType == "experimentalRawEvents", + event["reason"] as? String == "scrollWheel" + { + return eventType + } + return nil +} + +private func eventStreamSummaryElement( + from event: [String: Any], + includeText: Bool +) -> [String: Any]? { + guard let element = firstNestedDictionary(event, keys: ["targetAccessibilityElement", "focusedAccessibilityElement"]) else { + return nil + } + + var result: [String: Any] = [:] + for key in ["role", "subrole", "title", "label", "description"] { + if let value = element[key] as? String, !value.isEmpty { + result[key] = value + } else if let value = element[key] as? Bool { + result[key] = value + } + } + for key in ["value", "selectedText"] { + if let value = element[key] as? String, !value.isEmpty { + if includeText { + result[key] = value + } + result["\(key)Length"] = value.count + } else if let value = element[key] as? Bool { + result[key] = value + } + } + if let actions = element["actions"] as? [String], !actions.isEmpty { + result["actions"] = Array(actions.prefix(8)) + } + if element["secureInput"] as? Bool == true { + result["secureInput"] = true + } + return result.isEmpty ? nil : result +} + +private func eventStreamSummaryFilteredEvent( + line: Int, + event: [String: Any], + keys: [String] +) -> [String: Any] { + var result: [String: Any] = ["line": line] + for key in keys { + if let value = event[key] { + result[key] = value + } + } + return result +} + +private func eventStreamSummaryRuntimeInput( + line: Int, + event: [String: Any], + action: [String: Any] +) -> [String: Any]? { + guard let eventType = action["type"] as? String else { + return nil + } + + var result: [String: Any] = [ + "line": line, + "sourceEventType": eventType, + "requiresUserValue": true, + ] + if let element = action["element"] as? [String: Any], !element.isEmpty { + result["target"] = element + } + + switch eventType { + case "keyboard.text_input": + result["kind"] = "text" + result["description"] = "Runtime text to enter into the observed target." + if let textLength = action["textLength"] { + result["textLength"] = textLength + } + if event["secureInput"] as? Bool == true || event["redacted"] as? Bool == true { + result["sensitive"] = true + } + return result + case "selection.changed": + result["kind"] = "selection" + result["description"] = "Current selection or selected-content semantics if the workflow depends on it." + if let selectedText = event["selectedText"] as? String { + result["selectedTextLength"] = selectedText.count + } + if let selectionCleared = action["selectionCleared"] { + result["selectionCleared"] = selectionCleared + } + return result + default: + return nil + } +} + +private func eventStreamSummarySafetySignal( + line: Int, + event: [String: Any], + action: [String: Any] +) -> [String: Any]? { + guard let eventType = action["type"] as? String else { + return nil + } + + var reason: String? + if eventType == "keyboard.submit" { + reason = "submitAction" + } + + let element = action["element"] as? [String: Any] ?? [:] + if reason == nil, let matchedReason = eventStreamSummarySafetyKeywordReason(element: element) { + reason = matchedReason + } + guard let reason else { + return nil + } + + var result: [String: Any] = [ + "line": line, + "sourceEventType": eventType, + "reason": reason, + "confirmationRequired": true, + ] + if let timestamp = event["timestamp"] as? String { + result["timestamp"] = timestamp + } + if let window = action["window"] as? [String: Any], !window.isEmpty { + result["window"] = window + } + if !element.isEmpty { + result["target"] = element + } + return result +} + +private func eventStreamSummarySafetyKeywordReason(element: [String: Any]) -> String? { + let text = ["role", "title", "label", "description", "subrole"] + .compactMap { element[$0] as? String } + .joined(separator: " ") + .lowercased() + guard !text.isEmpty else { + return nil + } + for (keyword, reason) in eventStreamSummarySafetyKeywords where text.contains(keyword) { + return reason + } + return nil +} + +private func eventStreamSummaryIsBlockingDiagnostic(_ event: [String: Any]) -> Bool { + guard eventStreamSummaryEventKind(event) == "debug.error", + let reason = event["reason"] as? String + else { + return false + } + return eventStreamSummaryBlockingDiagnosticReasons.contains(reason) +} + +private func eventStreamSummaryRecordingIncomplete( + metadata: [String: Any], + eventTypeCounts: [String: Int] +) -> Bool { + if metadata["state"] as? String == "recording" || metadata["active"] as? Bool == true { + return true + } + return eventTypeCounts["session.ended", default: 0] == 0 +} + +private func eventStreamSummaryLimits( + actionEventCount: Int, + actionSequenceStored: Int, + runtimeInputCount: Int, + runtimeInputsStored: Int, + safetySignalCount: Int, + safetySignalsStored: Int, + targetElementCount: Int, + targetElementsStored: Int, + focusedElementCount: Int, + focusedElementsStored: Int, + selectionEventCount: Int, + selectionEventsStored: Int, + debugErrorCount: Int, + debugErrorsStored: Int, + redactionEventCount: Int, + redactionEventsStored: Int, + screenshotPathCount: Int, + screenshotPathsStored: Int +) -> [String: Any] { + var omittedCounts: [String: Int] = [:] + omittedCounts["actionSequence"] = max(0, actionEventCount - actionSequenceStored) + omittedCounts["runtimeInputs"] = max(0, runtimeInputCount - runtimeInputsStored) + omittedCounts["safetySignals"] = max(0, safetySignalCount - safetySignalsStored) + omittedCounts["targetElements"] = max(0, targetElementCount - targetElementsStored) + omittedCounts["focusedElements"] = max(0, focusedElementCount - focusedElementsStored) + omittedCounts["selectionEvents"] = max(0, selectionEventCount - selectionEventsStored) + omittedCounts["debugErrors"] = max(0, debugErrorCount - debugErrorsStored) + omittedCounts["redactionEvents"] = max(0, redactionEventCount - redactionEventsStored) + omittedCounts["screenshotPaths"] = max(0, screenshotPathCount - screenshotPathsStored) + + let hasTruncatedSummary = omittedCounts.values.contains { $0 > 0 } + return [ + "hasTruncatedSummary": hasTruncatedSummary, + "limits": [ + "actionSequence": eventStreamSummaryActionSequenceLimit, + "runtimeInputs": eventStreamSummaryRuntimeInputsLimit, + "safetySignals": eventStreamSummarySafetySignalsLimit, + "targetElements": eventStreamSummaryElementListLimit, + "focusedElements": eventStreamSummaryElementListLimit, + "selectionEvents": eventStreamSummaryElementListLimit, + "debugErrors": eventStreamSummaryDiagnosticsLimit, + "redactionEvents": eventStreamSummaryDiagnosticsLimit, + "screenshotPaths": eventStreamSummaryScreenshotPathsLimit, + ], + "storedCounts": [ + "actionSequence": actionSequenceStored, + "runtimeInputs": runtimeInputsStored, + "safetySignals": safetySignalsStored, + "targetElements": targetElementsStored, + "focusedElements": focusedElementsStored, + "selectionEvents": selectionEventsStored, + "debugErrors": debugErrorsStored, + "redactionEvents": redactionEventsStored, + "screenshotPaths": screenshotPathsStored, + ], + "sourceCounts": [ + "actionSequence": actionEventCount, + "runtimeInputs": runtimeInputCount, + "safetySignals": safetySignalCount, + "targetElements": targetElementCount, + "focusedElements": focusedElementCount, + "selectionEvents": selectionEventCount, + "debugErrors": debugErrorCount, + "redactionEvents": redactionEventCount, + "screenshotPaths": screenshotPathCount, + ], + "omittedCounts": omittedCounts, + ] +} + +private func eventStreamSummarySkillEvidence( + eventTypeCounts: [String: Int], + hasActionEvents: Bool, + hasTargetElements: Bool, + hasFocusedElements: Bool, + hasSelectionSignals: Bool, + hasScreenshots: Bool, + hasDebugErrors: Bool, + hasBlockingDiagnostics: Bool, + recordingIncomplete: Bool, + sessionStartedNotFirst: Bool, + sessionStartedCountInvalid: Bool, + sessionEndedNotFinal: Bool, + sessionEndedCountInvalid: Bool, + hasRedactionSignals: Bool, + hasSafetySignals: Bool, + hasTruncatedSummary: Bool +) -> [String: Any] { + [ + "hasActionEvents": hasActionEvents, + "hasInputEvents": eventTypeCounts.keys.contains { $0.hasPrefix("keyboard.") }, + "hasPointerEvents": eventTypeCounts.keys.contains { $0.hasPrefix("mouse.") }, + "hasAXContext": eventTypeCounts["AX.focusedWindowChanged", default: 0] > 0, + "hasTargetElements": hasTargetElements, + "hasFocusedElements": hasFocusedElements, + "hasSelectionSignals": hasSelectionSignals, + "hasTerminalSignals": eventTypeCounts["terminal.value_changed", default: 0] > 0, + "hasScreenshots": hasScreenshots, + "hasDebugErrors": hasDebugErrors, + "hasBlockingDiagnostics": hasBlockingDiagnostics, + "recordingIncomplete": recordingIncomplete, + "sessionStartedNotFirst": sessionStartedNotFirst, + "sessionStartedCountInvalid": sessionStartedCountInvalid, + "sessionEndedNotFinal": sessionEndedNotFinal, + "sessionEndedCountInvalid": sessionEndedCountInvalid, + "hasRedactionSignals": hasRedactionSignals, + "hasSafetySignals": hasSafetySignals, + "hasTruncatedSummary": hasTruncatedSummary, + ] +} + +private func eventStreamSummarySkillReadiness( + evidence: [String: Any], + hasWindowContext: Bool, + recordingWasCancelled: Bool = false, + recordingIncomplete: Bool = false, + sessionStartedNotFirst: Bool = false, + sessionStartedCountInvalid: Bool = false, + sessionEndedNotFinal: Bool = false, + sessionEndedCountInvalid: Bool = false +) -> [String: Any] { + let hasActionEvents = evidence["hasActionEvents"] as? Bool == true + let hasAXContext = evidence["hasAXContext"] as? Bool == true + let hasTargetElements = evidence["hasTargetElements"] as? Bool == true + let hasFocusedElements = evidence["hasFocusedElements"] as? Bool == true + let hasSelectionSignals = evidence["hasSelectionSignals"] as? Bool == true + let hasDebugErrors = evidence["hasDebugErrors"] as? Bool == true + let hasBlockingDiagnostics = evidence["hasBlockingDiagnostics"] as? Bool == true + let isRecordingIncomplete = recordingIncomplete || evidence["recordingIncomplete"] as? Bool == true + let hasEventsBeforeSessionStarted = sessionStartedNotFirst || evidence["sessionStartedNotFirst"] as? Bool == true + let hasInvalidSessionStartedCount = sessionStartedCountInvalid || evidence["sessionStartedCountInvalid"] as? Bool == true + let hasEventsAfterSessionEnded = sessionEndedNotFinal || evidence["sessionEndedNotFinal"] as? Bool == true + let hasInvalidSessionEndedCount = sessionEndedCountInvalid || evidence["sessionEndedCountInvalid"] as? Bool == true + let hasRedactionSignals = evidence["hasRedactionSignals"] as? Bool == true + let hasSafetySignals = evidence["hasSafetySignals"] as? Bool == true + let hasTruncatedSummary = evidence["hasTruncatedSummary"] as? Bool == true + + var reasons: [String] = [] + if recordingWasCancelled { + reasons.append("recording was cancelled; do not create or update a skill from this event stream") + } + if isRecordingIncomplete { + reasons.append("recording is not complete; stop the recording before creating a skill") + } + if hasInvalidSessionStartedCount { + reasons.append("recording must contain exactly one session.started event before creating a skill") + } + if hasEventsBeforeSessionStarted { + reasons.append("recording has events before session.started; start must be the first event before creating a skill") + } + if hasEventsAfterSessionEnded { + reasons.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + } + if hasInvalidSessionEndedCount { + reasons.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + } + if !hasActionEvents { + reasons.append("recording has no high-level user action events") + } + if !hasWindowContext { + reasons.append("recording has no stable app/window context") + } + if !hasAXContext { + reasons.append("recording has no AX focused window context") + } + if !(hasTargetElements || hasFocusedElements || hasSelectionSignals) { + reasons.append("recording has no semantic target, focused element, or selection evidence") + } + if hasDebugErrors { + reasons.append("recording includes debug.error diagnostics") + } + if hasBlockingDiagnostics { + reasons.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + } + if hasRedactionSignals { + reasons.append("recording includes redaction or secureInput signals") + } + if hasSafetySignals { + reasons.append("recording includes actions that may require explicit user confirmation") + } + if hasTruncatedSummary { + reasons.append("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill") + } + + let status: String + let recommendedNextStep: String + if recordingWasCancelled { + status = "insufficient" + recommendedNextStep = "Acknowledge the cancellation and ask the user to re-record when they want to create a skill." + } else if isRecordingIncomplete { + status = "insufficient" + recommendedNextStep = "Stop the recording, then inspect the completed events before creating a skill." + } else if hasInvalidSessionStartedCount { + status = "insufficient" + recommendedNextStep = "Discard this malformed recording and re-record so exactly one start event opens the event stream." + } else if hasEventsBeforeSessionStarted { + status = "insufficient" + recommendedNextStep = "Discard this malformed recording and re-record so session.started is the first event." + } else if hasEventsAfterSessionEnded { + status = "insufficient" + recommendedNextStep = "Discard this malformed recording and re-record so the stop or cancel event closes the event stream." + } else if hasInvalidSessionEndedCount { + status = "insufficient" + recommendedNextStep = "Discard this malformed recording and re-record so the stop or cancel event closes the event stream exactly once." + } else if hasBlockingDiagnostics { + status = "insufficient" + recommendedNextStep = "Fix the recording permissions or input monitoring issue, then ask the user to re-record the workflow." + } else if !hasActionEvents { + status = "insufficient" + recommendedNextStep = "Ask the user to re-record after confirming permissions and demonstrating at least one workflow action." + } else if reasons.isEmpty { + status = "ready" + recommendedNextStep = "Create or refine a reusable skill from the recording, then validate the final skill package." + } else { + status = "needsReview" + recommendedNextStep = "Inspect events.jsonl, replace user-specific values with inputs, and refine the generated skill before packaging." + } + + return [ + "status": status, + "canCreateSkillDraft": hasActionEvents && !recordingWasCancelled && !isRecordingIncomplete && !hasInvalidSessionStartedCount && !hasEventsBeforeSessionStarted && !hasEventsAfterSessionEnded && !hasInvalidSessionEndedCount && !hasBlockingDiagnostics, + "requiresHumanReview": status != "ready", + "reasons": reasons, + "recommendedNextStep": recommendedNextStep, + ] +} + +private func firstNestedDictionary(_ dictionary: [String: Any]?, keys: [String]) -> [String: Any]? { + guard let dictionary else { + return nil + } + for key in keys { + if let value = dictionary[key] as? [String: Any] { + return value + } + } + return nil +} + +private func firstString(_ dictionary: [String: Any]?, keys: [String]) -> String? { + guard let dictionary else { + return nil + } + for key in keys { + if let value = dictionary[key] as? String, !value.isEmpty { + return value + } + } + return nil +} + +private func eventStreamSummaryEventKind(_ event: [String: Any]) -> String? { + (event["kind"] as? String) ?? (event["type"] as? String) +} + +private extension Dictionary where Key == String, Value == Int { + func sortedDictionary() -> [String: Int] { + Dictionary(uniqueKeysWithValues: sorted { $0.key < $1.key }) + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingValidation.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingValidation.swift new file mode 100644 index 00000000..b17cc122 --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamRecordingValidation.swift @@ -0,0 +1,696 @@ +import Foundation + +public struct EventStreamRecordingValidationOptions: Equatable, Sendable { + public let inputPath: String + public let strictOCU: Bool + public let requiredEventTypes: [String] + public let requireSkillDraft: Bool + + public init( + inputPath: String, + strictOCU: Bool = false, + requiredEventTypes: [String] = [], + requireSkillDraft: Bool = false + ) { + self.inputPath = inputPath + self.strictOCU = strictOCU + self.requiredEventTypes = requiredEventTypes + self.requireSkillDraft = requireSkillDraft + } +} + +private let eventStreamValidationSkillDraftActionEventTypes: Set = [ + "mouse.click", + "mouse.context_menu", + "mouse.drag", + "keyboard.text_input", + "keyboard.submit", + "keyboard.shortcut", + "terminal.value_changed", + "selection.changed", +] + +private let eventStreamValidationBlockingDiagnosticReasons: Set = [ + "inputMonitorsUnavailable", +] + +private let eventStreamValidationHandoffPathKeys = [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", +] + +public func validateEventStreamRecording( + options: EventStreamRecordingValidationOptions +) -> [String: Any] { + do { + return try validateEventStreamRecordingImpl(options: options) + } catch { + let message = (error as? LocalizedError)?.errorDescription ?? String(describing: error) + return [ + "ok": false, + "errors": [message], + "warnings": [], + ] + } +} + +private func validateEventStreamRecordingImpl( + options: EventStreamRecordingValidationOptions +) throws -> [String: Any] { + let resolved = try resolveEventStreamValidationInput(path: options.inputPath) + var warnings: [String] = [] + var errors: [String] = [] + + let metadata = try resolved.metadataURL.map(readEventStreamValidationJSONObject) ?? [:] + let metadataJSONURL = resolved.sessionDirectory.appendingPathComponent("metadata.json") + let sessionAliasURL = resolved.sessionDirectory.appendingPathComponent("session.json") + if resolved.metadataURL == nil { + if options.strictOCU { + errors.append("strict OCU validation requires metadata.json or session.json") + } else { + warnings.append("metadata/session files not available; validating events.jsonl only") + } + } else if FileManager.default.fileExists(atPath: metadataJSONURL.path), + FileManager.default.fileExists(atPath: sessionAliasURL.path) + { + do { + let metadataJSON = try readEventStreamValidationJSONObject(metadataJSONURL) + let sessionJSON = try readEventStreamValidationJSONObject(sessionAliasURL) + if !eventStreamValidationSessionAliasCompatible(metadata: metadataJSON, sessionAlias: sessionJSON) { + errors.append("metadata.json and session.json differ") + } + } catch { + errors.append((error as? LocalizedError)?.errorDescription ?? String(describing: error)) + } + } else if options.strictOCU { + errors.append("strict OCU validation requires both metadata.json and session.json") + } else { + warnings.append("metadata/session alias pair is incomplete") + } + + let declaredPaths = eventStreamValidationHandoffPathEvidence( + metadata: metadata, + baseDirectory: resolved.sessionDirectory + ) + errors.append( + contentsOf: eventStreamValidationHandoffPathErrors( + evidence: declaredPaths, + requireAllPaths: options.strictOCU + ) + ) + + let state = metadata["state"] as? String + let currentSegmentEventsURL = eventStreamValidationOptionalPath( + metadata["currentSegmentEventsPath"], + baseDirectory: resolved.sessionDirectory + ) + let currentSegmentMetadataURL = eventStreamValidationOptionalPath( + metadata["currentSegmentMetadataPath"], + baseDirectory: resolved.sessionDirectory + ) + if let currentSegmentEventsURL, + !FileManager.default.fileExists(atPath: currentSegmentEventsURL.path) + { + errors.append("currentSegmentEventsPath does not exist: \(metadata["currentSegmentEventsPath"] ?? currentSegmentEventsURL.path)") + } + if let currentSegmentMetadataURL, + !FileManager.default.fileExists(atPath: currentSegmentMetadataURL.path) + { + errors.append("currentSegmentMetadataPath does not exist: \(metadata["currentSegmentMetadataPath"] ?? currentSegmentMetadataURL.path)") + } + if options.strictOCU { + if state == "recording" { + if currentSegmentEventsURL == nil { + errors.append("recording state requires currentSegmentEventsPath") + } + if currentSegmentMetadataURL == nil { + errors.append("recording state requires currentSegmentMetadataPath") + } + } else if state == "stopped" || state == "cancelled" { + if currentSegmentEventsURL != nil { + errors.append("final state must not include currentSegmentEventsPath") + } + if currentSegmentMetadataURL != nil { + errors.append("final state must not include currentSegmentMetadataPath") + } + } + } + + let eventsURL = resolved.eventsURL ?? eventStreamValidationPath( + metadata["eventsPath"], + baseDirectory: resolved.sessionDirectory, + fallback: resolved.sessionDirectory.appendingPathComponent("events.jsonl") + ) + let suppressedURL = eventStreamValidationPath( + metadata["suppressedEventsPath"], + baseDirectory: resolved.sessionDirectory, + fallback: resolved.sessionDirectory.appendingPathComponent("suppressed.jsonl") + ) + + let events: [[String: Any]] + do { + events = try readEventStreamValidationJSONL(eventsURL) + } catch { + errors.append((error as? LocalizedError)?.errorDescription ?? String(describing: error)) + events = [] + } + + let suppressedEvents: [[String: Any]] + if FileManager.default.fileExists(atPath: suppressedURL.path) { + do { + suppressedEvents = try readEventStreamValidationJSONL(suppressedURL) + } catch { + errors.append((error as? LocalizedError)?.errorDescription ?? String(describing: error)) + suppressedEvents = [] + } + } else if options.strictOCU || metadata["suppressedEventsPath"] != nil { + errors.append("suppressedEventsPath does not exist: \(suppressedURL.path)") + suppressedEvents = [] + } else { + warnings.append("suppressed events file not found: \(suppressedURL.path)") + suppressedEvents = [] + } + + if let eventCount = metadata["eventCount"] as? Int { + if eventCount != events.count { + errors.append("eventCount=\(eventCount) does not match events.jsonl lines=\(events.count)") + } + } else { + warnings.append("metadata has no eventCount") + } + + if let suppressedEventCount = metadata["suppressedEventCount"] as? Int { + if suppressedEventCount != suppressedEvents.count { + errors.append("suppressedEventCount=\(suppressedEventCount) does not match suppressed.jsonl lines=\(suppressedEvents.count)") + } + } else { + warnings.append("metadata has no suppressedEventCount") + } + + let eventTypeCounts = eventStreamValidationEventTypeCounts(events) + for requiredEventType in options.requiredEventTypes where eventTypeCounts[requiredEventType, default: 0] == 0 { + errors.append("missing required event type: \(requiredEventType)") + } + + let active = metadata["active"] as? Bool + let endReason = metadata["endReason"] as? String + let startedEvents = events.filter { eventStreamValidationEventKind($0) == "session.started" } + let sessionStartedCountInvalid = startedEvents.count != 1 + let firstEventType = events.first.flatMap(eventStreamValidationEventKind) + let sessionStartedIsFirst = firstEventType == "session.started" + let sessionStartedNotFirst = !startedEvents.isEmpty && !sessionStartedIsFirst + let endedEvents = events.filter { eventStreamValidationEventKind($0) == "session.ended" } + let endedReasons = Set(endedEvents.compactMap { $0["endReason"] as? String }) + let sessionEndedCountInvalid = endedEvents.count > 1 + let finalEventType = events.last.flatMap(eventStreamValidationEventKind) + let sessionEndedIsFinal = finalEventType == "session.ended" + let sessionEndedNotFinal = !endedEvents.isEmpty && !sessionEndedIsFinal + let inferredEndReason = endReason ?? (endedReasons.count == 1 ? endedReasons.first : nil) + let recordingWasCancelled = state == "cancelled" || inferredEndReason == "recording_controls_cancelled" + let recordingIncomplete = eventStreamValidationRecordingIncomplete( + state: state, + active: active, + endedEvents: endedEvents + ) + let blockingDiagnostics = eventStreamValidationBlockingDiagnostics(events) + let hasSkillDraftAction = eventStreamValidationSkillDraftActionEventTypes.contains { + eventTypeCounts[$0, default: 0] > 0 + } + let skillDraftReasons = eventStreamValidationSkillDraftBlockingReasons( + hasAction: hasSkillDraftAction, + recordingWasCancelled: recordingWasCancelled, + recordingIncomplete: recordingIncomplete, + blockingDiagnostics: blockingDiagnostics, + sessionStartedNotFirst: sessionStartedNotFirst, + sessionStartedCountInvalid: sessionStartedCountInvalid, + sessionEndedNotFinal: sessionEndedNotFinal, + sessionEndedCountInvalid: sessionEndedCountInvalid + ) + if options.requireSkillDraft { + errors.append(contentsOf: skillDraftReasons) + } + + if sessionStartedCountInvalid { + let message = startedEvents.isEmpty + ? "recording has no session.started event" + : "recording has multiple session.started events" + if options.strictOCU { + errors.append(message) + } else if !options.requireSkillDraft { + warnings.append(message) + } + } + + if sessionStartedNotFirst { + let message = "session.started is not the first event" + if options.strictOCU { + errors.append(message) + } else if !options.requireSkillDraft { + warnings.append(message) + } + } + + if sessionEndedCountInvalid { + let message = "recording has multiple session.ended events" + if options.strictOCU { + errors.append(message) + } else if !options.requireSkillDraft { + warnings.append(message) + } + } + + if sessionEndedNotFinal { + let message = "session.ended is not the final event" + if options.strictOCU { + errors.append(message) + } else if !options.requireSkillDraft { + warnings.append(message) + } + } + + if state == "stopped" || state == "cancelled" { + if endedEvents.isEmpty { + let message = "final state \(state ?? "") has no session.ended event" + if options.strictOCU { + errors.append(message) + } else { + warnings.append(message) + } + } + if let endReason { + if !endedReasons.isEmpty, !endedReasons.contains(endReason) { + errors.append("metadata endReason=\(endReason) not present in session.ended events: \(Array(endedReasons).sorted())") + } + } + } + + let sessionID = (metadata["sessionId"] as? String) ?? (metadata["sessionID"] as? String) + if let sessionID { + let mismatchedLines = events.enumerated().compactMap { index, event -> Int? in + guard let eventSessionID = event["sessionId"] as? String else { + return nil + } + return eventSessionID == sessionID ? nil : index + 1 + } + if !mismatchedLines.isEmpty { + errors.append("event sessionId mismatch at JSONL lines: \(Array(mismatchedLines.prefix(10)))") + } + } + + for (index, event) in events.enumerated() { + for screenshotPath in collectEventStreamValidationScreenshotPaths(event) { + let screenshotURL = eventStreamValidationURL(path: screenshotPath, baseDirectory: resolved.sessionDirectory) + if !eventStreamValidationURL(screenshotURL, isInside: resolved.sessionDirectory) { + errors.append("screenshotPath from event line \(index + 1) must stay inside session directory: \(screenshotPath)") + continue + } + if !FileManager.default.fileExists(atPath: screenshotURL.path) { + errors.append("screenshotPath from event line \(index + 1) does not exist: \(screenshotPath)") + } + } + } + + if options.strictOCU { + if metadata["active"] as? Bool != (state == "recording") { + errors.append("metadata active flag does not match state") + } + if state != "recording" { + let activeSessionURL = resolved.sessionDirectory + .deletingLastPathComponent() + .appendingPathComponent("active-session.json") + if FileManager.default.fileExists(atPath: activeSessionURL.path) { + errors.append("active-session.json exists after final session state") + } + } + } + + var result: [String: Any] = [ + "ok": errors.isEmpty, + "sessionDir": resolved.sessionDirectory.path, + "sessionPath": sessionAliasURL.path, + "eventsPath": eventsURL.path, + "suppressedEventsPath": suppressedURL.path, + "eventCount": events.count, + "suppressedEventCount": suppressedEvents.count, + "eventTypes": eventTypeCounts.sortedDictionary(), + "sessionStartedCount": startedEvents.count, + "sessionStartedCountInvalid": sessionStartedCountInvalid, + "sessionEndedCount": endedEvents.count, + "sessionEndedCountInvalid": sessionEndedCountInvalid, + "declaredPaths": declaredPaths, + "warnings": warnings, + "errors": errors, + ] + if let metadataURL = resolved.metadataURL { + result["metadataPath"] = metadataURL.path + } + if let currentSegmentEventsURL { + result["currentSegmentEventsPath"] = currentSegmentEventsURL.path + } + if let currentSegmentMetadataURL { + result["currentSegmentMetadataPath"] = currentSegmentMetadataURL.path + } + if let state { + result["state"] = state + } + if let active { + result["active"] = active + } + if let inferredEndReason { + result["endReason"] = inferredEndReason + } + if let firstEventType { + result["firstEventType"] = firstEventType + result["sessionStartedIsFirst"] = sessionStartedIsFirst + } + if let finalEventType { + result["finalEventType"] = finalEventType + result["sessionEndedIsFinal"] = sessionEndedIsFinal + } + if let sessionID { + result["sessionId"] = sessionID + } + if let metadataEventCount = metadata["eventCount"] as? Int { + result["metadataEventCount"] = metadataEventCount + } + if let metadataSuppressedEventCount = metadata["suppressedEventCount"] as? Int { + result["metadataSuppressedEventCount"] = metadataSuppressedEventCount + } + result["skillDraftReady"] = skillDraftReasons.isEmpty + result["skillDraftReasons"] = skillDraftReasons + result["recordingIncomplete"] = recordingIncomplete + result["blockingDiagnostics"] = blockingDiagnostics + result["requireSkillDraft"] = options.requireSkillDraft + return result +} + +private func eventStreamValidationSkillDraftBlockingReasons( + hasAction: Bool, + recordingWasCancelled: Bool, + recordingIncomplete: Bool, + blockingDiagnostics: [[String: Any]], + sessionStartedNotFirst: Bool, + sessionStartedCountInvalid: Bool, + sessionEndedNotFinal: Bool, + sessionEndedCountInvalid: Bool +) -> [String] { + var reasons: [String] = [] + if !hasAction { + reasons.append("recording has no high-level user action events") + } + if recordingWasCancelled { + reasons.append("recording was cancelled; do not create or update a skill from this event stream") + } + if recordingIncomplete { + reasons.append("recording is not complete; stop the recording before creating a skill") + } + if !blockingDiagnostics.isEmpty { + reasons.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + } + if sessionStartedCountInvalid { + reasons.append("recording must contain exactly one session.started event before creating a skill") + } + if sessionStartedNotFirst { + reasons.append("recording has events before session.started; start must be the first event before creating a skill") + } + if sessionEndedNotFinal { + reasons.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + } + if sessionEndedCountInvalid { + reasons.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + } + return reasons +} + +private func eventStreamValidationRecordingIncomplete( + state: String?, + active: Bool?, + endedEvents: [[String: Any]] +) -> Bool { + if state == "recording" || active == true { + return true + } + return endedEvents.isEmpty +} + +private func eventStreamValidationBlockingDiagnostics(_ events: [[String: Any]]) -> [[String: Any]] { + var diagnostics: [[String: Any]] = [] + for (index, event) in events.enumerated() where eventStreamValidationEventKind(event) == "debug.error" { + let reason = event["reason"] as? String + guard reason.map(eventStreamValidationBlockingDiagnosticReasons.contains) == true else { + continue + } + var diagnostic: [String: Any] = ["line": index + 1] + for key in ["subsystem", "reason", "errorType"] { + if let value = event[key] { + diagnostic[key] = value + } + } + diagnostics.append(diagnostic) + } + return diagnostics +} + +private struct EventStreamValidationInput { + let sessionDirectory: URL + let metadataURL: URL? + let eventsURL: URL? +} + +private func resolveEventStreamValidationInput(path: String) throws -> EventStreamValidationInput { + let url = URL(fileURLWithPath: path) + var isDirectory: ObjCBool = false + guard FileManager.default.fileExists(atPath: url.path, isDirectory: &isDirectory) else { + throw ComputerUseError.message("Path does not exist: \(path)") + } + + if isDirectory.boolValue { + let metadataURL = url.appendingPathComponent("metadata.json") + let sessionURL = url.appendingPathComponent("session.json") + if FileManager.default.fileExists(atPath: metadataURL.path) { + return EventStreamValidationInput(sessionDirectory: url, metadataURL: metadataURL, eventsURL: nil) + } + if FileManager.default.fileExists(atPath: sessionURL.path) { + return EventStreamValidationInput(sessionDirectory: url, metadataURL: sessionURL, eventsURL: nil) + } + throw ComputerUseError.message("Session directory has no metadata.json or session.json: \(path)") + } + + switch url.lastPathComponent { + case "metadata.json": + return EventStreamValidationInput( + sessionDirectory: url.deletingLastPathComponent(), + metadataURL: url, + eventsURL: nil + ) + case "session.json": + let metadataURL = url.deletingLastPathComponent().appendingPathComponent("metadata.json") + let statusURL = FileManager.default.fileExists(atPath: metadataURL.path) ? metadataURL : url + return EventStreamValidationInput( + sessionDirectory: url.deletingLastPathComponent(), + metadataURL: statusURL, + eventsURL: nil + ) + case "events.jsonl": + return EventStreamValidationInput( + sessionDirectory: url.deletingLastPathComponent(), + metadataURL: nil, + eventsURL: url + ) + default: + throw ComputerUseError.message("Input must be a session directory, metadata.json, session.json, or events.jsonl.") + } +} + +private func readEventStreamValidationJSONObject(_ url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + guard let object = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ComputerUseError.message("Expected JSON object in \(url.path).") + } + return object +} + +private func readEventStreamValidationJSONL(_ url: URL) throws -> [[String: Any]] { + let text = try String(contentsOf: url, encoding: .utf8) + var events: [[String: Any]] = [] + for (index, line) in text.split(separator: "\n", omittingEmptySubsequences: true).enumerated() { + let data = Data(line.utf8) + guard let event = try JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw ComputerUseError.message("Expected JSON object in \(url.path):\(index + 1).") + } + events.append(event) + } + return events +} + +private func eventStreamValidationPath(_ value: Any?, baseDirectory: URL, fallback: URL) -> URL { + guard let path = value as? String, !path.isEmpty else { + return fallback + } + return eventStreamValidationURL(path: path, baseDirectory: baseDirectory) +} + +private func eventStreamValidationOptionalPath(_ value: Any?, baseDirectory: URL) -> URL? { + guard let path = value as? String, !path.isEmpty else { + return nil + } + return eventStreamValidationURL(path: path, baseDirectory: baseDirectory) +} + +private func eventStreamValidationURL(path: String, baseDirectory: URL) -> URL { + if path.hasPrefix("/") { + return URL(fileURLWithPath: path) + } + return baseDirectory.appendingPathComponent(path) +} + +private func eventStreamValidationHandoffPathEvidence( + metadata: [String: Any], + baseDirectory: URL +) -> [String: [String: Any]] { + var evidence: [String: [String: Any]] = [:] + for key in eventStreamValidationHandoffPathKeys { + let value = metadata[key] + var entry: [String: Any] = [ + "value": value ?? NSNull(), + "resolvedPath": NSNull(), + "exists": NSNull(), + ] + if let path = value as? String, !path.isEmpty { + let resolvedURL = eventStreamValidationURL(path: path, baseDirectory: baseDirectory) + entry["resolvedPath"] = resolvedURL.path + entry["exists"] = FileManager.default.fileExists(atPath: resolvedURL.path) + } + evidence[key] = entry + } + return evidence +} + +private func eventStreamValidationHandoffPathErrors( + evidence: [String: [String: Any]], + requireAllPaths: Bool +) -> [String] { + var errors: [String] = [] + for key in eventStreamValidationHandoffPathKeys { + let entry = evidence[key] ?? [:] + let value = entry["value"] + if let path = value as? String, !path.isEmpty { + if entry["exists"] as? Bool == false { + errors.append("\(key) does not exist: \(path)") + } + continue + } + if value != nil, !(value is NSNull), !((value as? String)?.isEmpty == true) { + errors.append("\(key) must be a non-empty string") + } else if requireAllPaths { + errors.append("strict OCU validation requires \(key)") + } + } + return errors +} + +private func eventStreamValidationSessionAliasCompatible( + metadata: [String: Any], + sessionAlias: [String: Any] +) -> Bool { + if jsonObjectsEqual(metadata, sessionAlias) { + return true + } + + let metadataSessionID = eventStreamValidationFirstString(metadata, keys: ["sessionId", "sessionID", "id"]) + let sessionID = eventStreamValidationFirstString(sessionAlias, keys: ["id", "sessionId", "sessionID"]) + guard let metadataSessionID, let sessionID, metadataSessionID == sessionID else { + return false + } + + if let metadataStartedAt = metadata["startedAt"] as? String, + metadataStartedAt != sessionAlias["startedAt"] as? String + { + return false + } + + for key in ["endedAt", "endReason"] { + if let sessionValue = sessionAlias[key] as? String, + sessionValue != metadata[key] as? String + { + return false + } + } + + guard let sessionEventsPath = sessionAlias["eventsPath"] as? String, + let metadataEventsPath = metadata["eventsPath"] as? String + else { + return false + } + if sessionEventsPath != metadataEventsPath, + URL(fileURLWithPath: sessionEventsPath).lastPathComponent != URL(fileURLWithPath: metadataEventsPath).lastPathComponent + { + return false + } + + return true +} + +private func eventStreamValidationFirstString(_ dictionary: [String: Any], keys: [String]) -> String? { + for key in keys { + if let value = dictionary[key] as? String, !value.isEmpty { + return value + } + } + return nil +} + +private func eventStreamValidationEventTypeCounts(_ events: [[String: Any]]) -> [String: Int] { + var counts: [String: Int] = [:] + for event in events { + if let type = eventStreamValidationEventKind(event) { + counts[type, default: 0] += 1 + } + } + return counts +} + +private func eventStreamValidationEventKind(_ event: [String: Any]) -> String? { + (event["kind"] as? String) ?? (event["type"] as? String) +} + +private func collectEventStreamValidationScreenshotPaths(_ value: Any) -> [String] { + if let dictionary = value as? [String: Any] { + var paths: [String] = [] + for (key, child) in dictionary { + if key == "screenshotPath", let path = child as? String, !path.isEmpty { + paths.append(path) + } else { + paths.append(contentsOf: collectEventStreamValidationScreenshotPaths(child)) + } + } + return paths + } + if let array = value as? [Any] { + return array.flatMap { collectEventStreamValidationScreenshotPaths($0) } + } + return [] +} + +private func eventStreamValidationURL(_ url: URL, isInside directory: URL) -> Bool { + let candidatePath = url.standardizedFileURL.path + let directoryPath = directory.standardizedFileURL.path + return candidatePath == directoryPath || candidatePath.hasPrefix(directoryPath + "/") +} + +private func jsonObjectsEqual(_ lhs: [String: Any], _ rhs: [String: Any]) -> Bool { + guard JSONSerialization.isValidJSONObject(lhs), JSONSerialization.isValidJSONObject(rhs) else { + return false + } + let lhsData = try? JSONSerialization.data(withJSONObject: lhs, options: [.sortedKeys]) + let rhsData = try? JSONSerialization.data(withJSONObject: rhs, options: [.sortedKeys]) + return lhsData == rhsData +} + +private extension Dictionary where Key == String, Value == Int { + func sortedDictionary() -> [String: Int] { + Dictionary(uniqueKeysWithValues: sorted { $0.key < $1.key }) + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamService.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamService.swift new file mode 100644 index 00000000..09d554e9 --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamService.swift @@ -0,0 +1,2799 @@ +import AppKit +import CoreGraphics +import Foundation + +public enum EventStreamRecordingState: String, Codable, Equatable, Sendable { + case idle + case recording + case stopped + case cancelled +} + +public enum EventStreamScreenshotPolicy: String, Equatable { + case auto + case always + case never +} + +public enum EventStreamStartApprovalPolicy: String, Equatable, Sendable { + case automatic + case interactive + case mcpElicitation + case approve + case deny + case cancel +} + +public enum EventStreamStartApprovalDecision: Equatable, Sendable { + case approved + case denied + case cancelled +} + +public struct EventStreamStartApprovalError: LocalizedError, Equatable { + public let decision: EventStreamStartApprovalDecision + + public var errorDescription: String? { + switch decision { + case .approved: + return nil + case .denied: + return eventStreamStartApprovalDeniedMessage + case .cancelled: + return eventStreamStartApprovalCancelledMessage + } + } +} + +public let eventStreamMaximumRecordingDuration: TimeInterval = 30 * 60 +public let eventStreamMaximumDurationEndReason = "recording_time_limit_reached" +public let eventStreamStartApprovalCancelledMessage = "Record & Replay approval cancelled via MCP elicitation." +public let eventStreamStartApprovalDeniedMessage = "Record & Replay approval denied via MCP elicitation." +let eventStreamMouseDragDistanceThreshold: CGFloat = 3 +let eventStreamMaximumRawEventsPerInputEvent = 32 +public typealias EventStreamInputMonitorInstaller = @MainActor (NSEvent.EventTypeMask, @escaping (NSEvent) -> Void) -> Any? + +func eventStreamRecordingDurationLimitDescription(_ duration: TimeInterval) -> String { + guard duration > 0 else { + return "until stopped" + } + + let totalSeconds = max(Int(duration.rounded()), 1) + if totalSeconds % 60 == 0 { + let minutes = totalSeconds / 60 + return "up to \(minutes) minute\(minutes == 1 ? "" : "s")" + } + return "up to \(totalSeconds) second\(totalSeconds == 1 ? "" : "s")" +} + +private final class EventStreamInputMonitorToken { + private let context: Any? + private let removeHandler: () -> Void + + init(context: Any? = nil, removeHandler: @escaping () -> Void) { + self.context = context + self.removeHandler = removeHandler + } + + func remove() { + _ = context + removeHandler() + } +} + +private final class EventStreamCGEventTapContext { + weak var service: EventStreamService? + + init(service: EventStreamService) { + self.service = service + } +} + +private let eventStreamCGEventTapCallback: CGEventTapCallBack = { _proxy, type, cgEvent, userInfo in + if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput { + return Unmanaged.passUnretained(cgEvent) + } + + guard let userInfo, + let event = NSEvent(cgEvent: cgEvent) + else { + return Unmanaged.passUnretained(cgEvent) + } + + let context = Unmanaged + .fromOpaque(userInfo) + .takeUnretainedValue() + context.service?.recordInputEventFromCGEventTap(event, type: type) + return Unmanaged.passUnretained(cgEvent) +} + +public struct EventStreamRecordingStatus: Codable, Equatable, Sendable { + public let sessionID: String? + public let state: EventStreamRecordingState + public let startedAt: String? + public let endedAt: String? + public let endReason: String? + public let eventsPath: String? + public let metadataPath: String? + public let suppressedEventsPath: String? + public let currentSegmentEventsPath: String? + public let currentSegmentMetadataPath: String? + public let eventCount: Int + public let suppressedEventCount: Int + + public var isActive: Bool { + state == .recording + } + + public var asDictionary: [String: Any] { + var dictionary: [String: Any] = [ + "state": state.rawValue, + "active": isActive, + "eventCount": eventCount, + "suppressedEventCount": suppressedEventCount, + ] + + if let sessionID { + dictionary["sessionId"] = sessionID + dictionary["sessionID"] = sessionID + } + if let startedAt { + dictionary["startedAt"] = startedAt + } + if let endedAt { + dictionary["endedAt"] = endedAt + } + if let endReason { + dictionary["endReason"] = endReason + } + if let eventsPath { + dictionary["eventsPath"] = eventsPath + } + if let metadataPath { + dictionary["metadataPath"] = metadataPath + dictionary["sessionPath"] = URL(fileURLWithPath: metadataPath) + .deletingLastPathComponent() + .appendingPathComponent("session.json") + .path + } + if let suppressedEventsPath { + dictionary["suppressedEventsPath"] = suppressedEventsPath + } + if let currentSegmentEventsPath { + dictionary["currentSegmentEventsPath"] = currentSegmentEventsPath + } + if let currentSegmentMetadataPath { + dictionary["currentSegmentMetadataPath"] = currentSegmentMetadataPath + } + + return dictionary + } + + public var sessionHandoffDictionary: [String: Any] { + var dictionary: [String: Any] = [:] + if let sessionID { + dictionary["id"] = sessionID + } + if let startedAt { + dictionary["startedAt"] = startedAt + } + if let endedAt { + dictionary["endedAt"] = endedAt + } + if let endReason { + dictionary["endReason"] = endReason + } + if let eventsPath { + dictionary["eventsPath"] = eventsPath + } + return dictionary + } + + public func jsonText(prettyPrinted: Bool = true) throws -> String { + let options: JSONSerialization.WritingOptions = prettyPrinted + ? [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + : [.sortedKeys, .withoutEscapingSlashes] + let data = try JSONSerialization.data(withJSONObject: asDictionary, options: options) + guard let text = String(data: data, encoding: .utf8) else { + throw ComputerUseError.message("Failed to encode event stream status as JSON.") + } + return text + } + + static var idle: EventStreamRecordingStatus { + EventStreamRecordingStatus( + sessionID: nil, + state: .idle, + startedAt: nil, + endedAt: nil, + endReason: nil, + eventsPath: nil, + metadataPath: nil, + suppressedEventsPath: nil, + currentSegmentEventsPath: nil, + currentSegmentMetadataPath: nil, + eventCount: 0, + suppressedEventCount: 0 + ) + } +} + +public struct EventStreamWaitResult: Equatable, Sendable { + public let status: EventStreamRecordingStatus + public let timedOut: Bool + public let sessionMatched: Bool +} + +public final class EventStreamService: @unchecked Sendable { + public static let shared = EventStreamService() + + private let rootDirectory: URL + private let installsInputMonitors: Bool + private let showsRecordingControls: Bool + private let screenshotPolicy: EventStreamScreenshotPolicy + private let recordsRawEvents: Bool + private let maximumDuration: TimeInterval + private let startApprovalPolicy: EventStreamStartApprovalPolicy + private let installsCGEventTap: Bool + private let inputMonitorInstaller: EventStreamInputMonitorInstaller + private let lock = NSLock() + private var activeSession: EventStreamSession? + private var inputMonitorTokens: [EventStreamInputMonitorToken] = [] + private var workspaceObserver: NSObjectProtocol? + private var recordingControls: EventStreamRecordingControls? + private var initialAXTreeLines: [String]? + private var initialAXContextKey: String? + private var lastAXTreeLines: [String]? + private var lastAXContextKey: String? + private var lastObservedWindowContextKey: String? + private var lastSelectionSignature: String? + private var lastTerminalValueSignature: String? + private var pendingPointerEvent: EventStreamPendingPointerEvent? + + public init( + rootDirectory: URL? = nil, + installsInputMonitors: Bool = true, + showsRecordingControls: Bool = eventStreamRecordingControlsEnabled(), + screenshotPolicy: EventStreamScreenshotPolicy = eventStreamScreenshotPolicy(), + recordsRawEvents: Bool = eventStreamRawEventsEnabled(), + maximumDuration: TimeInterval = eventStreamRecordingMaximumDuration(), + startApprovalPolicy: EventStreamStartApprovalPolicy = eventStreamStartApprovalPolicy(), + installsCGEventTap: Bool = true, + inputMonitorInstaller: @escaping EventStreamInputMonitorInstaller = { matching, handler in + NSEvent.addGlobalMonitorForEvents(matching: matching, handler: handler) + } + ) { + self.rootDirectory = rootDirectory ?? defaultEventStreamRootDirectory() + self.installsInputMonitors = installsInputMonitors + self.showsRecordingControls = showsRecordingControls + self.screenshotPolicy = screenshotPolicy + self.recordsRawEvents = recordsRawEvents + self.maximumDuration = maximumDuration + self.startApprovalPolicy = startApprovalPolicy + self.installsCGEventTap = installsCGEventTap + self.inputMonitorInstaller = inputMonitorInstaller + } + + public var configuredStartApprovalPolicy: EventStreamStartApprovalPolicy { + startApprovalPolicy + } + + public var configuredMaximumDurationSeconds: Int { + max(Int(maximumDuration.rounded()), 0) + } + + public func start(approvalDecision: EventStreamStartApprovalDecision? = nil) throws -> EventStreamRecordingStatus { + lock.lock() + if let activeSession { + let status = activeSession.status + lock.unlock() + return status + } + lock.unlock() + + try requestStartApprovalIfNeeded(approvalDecision: approvalDecision) + try FileManager.default.createDirectory(at: rootDirectory, withIntermediateDirectories: true) + + let now = Date() + let sessionID = makeEventStreamSessionID(date: now) + let sessionDirectory = rootDirectory.appendingPathComponent(sessionID, isDirectory: true) + try FileManager.default.createDirectory(at: sessionDirectory, withIntermediateDirectories: true) + + let session = EventStreamSession( + id: sessionID, + directory: sessionDirectory, + startedAtDate: now + ) + try writeEmptyFileIfNeeded(session.suppressedEventsURL) + + lock.lock() + if let activeSession { + let status = activeSession.status + lock.unlock() + return status + } + activeSession = session + lock.unlock() + + try appendEvent([ + "type": "session.started", + "timestamp": eventStreamTimestamp(now), + "sessionId": sessionID, + "metadataPath": session.metadataURL.path, + "eventsPath": session.eventsURL.path, + "suppressedEventsPath": session.suppressedEventsURL.path, + ]) + try writeStatus(session.status) + scheduleMaximumDurationStopIfNeeded(sessionID: sessionID) + installInputMonitorsIfNeeded() + recordWindowChanged(reason: "initial") + recordFocusedWindowChanged(reason: "initial") + showRecordingControlsIfNeeded(startedAt: now) + return session.status + } + + private func requestStartApprovalIfNeeded(approvalDecision: EventStreamStartApprovalDecision? = nil) throws { + if let approvalDecision { + switch approvalDecision { + case .approved: + return + case .denied, .cancelled: + throw EventStreamStartApprovalError(decision: approvalDecision) + } + } + + let policy: EventStreamStartApprovalPolicy + switch startApprovalPolicy { + case .automatic: + policy = showsRecordingControls && installsInputMonitors ? .interactive : .approve + default: + policy = startApprovalPolicy + } + + switch policy { + case .automatic, .approve: + return + case .mcpElicitation: + throw EventStreamStartApprovalError(decision: .cancelled) + case .deny: + throw EventStreamStartApprovalError(decision: .denied) + case .cancel: + throw EventStreamStartApprovalError(decision: .cancelled) + case .interactive: + let decision = requestInteractiveStartApproval(allowHeadlessAutoApprove: startApprovalPolicy == .automatic) + switch decision { + case .approved: + return + case .denied, .cancelled: + throw EventStreamStartApprovalError(decision: decision) + } + } + } + + private func requestInteractiveStartApproval(allowHeadlessAutoApprove: Bool) -> EventStreamStartApprovalDecision { + performEventStreamOnMainSync { + guard NSApp.isRunning else { + return allowHeadlessAutoApprove ? .approved : .cancelled + } + + let alert = NSAlert() + alert.messageText = "Record & Replay wants to record your actions" + alert.informativeText = "Open Computer Use will capture mouse, keyboard, window, and accessibility context until you click Done or Discard. Recording can last \(eventStreamRecordingDurationLimitDescription(self.maximumDuration))." + alert.alertStyle = .informational + alert.addButton(withTitle: "Start Recording") + alert.addButton(withTitle: "Cancel") + let response = alert.runModal() + return response == .alertFirstButtonReturn ? .approved : .cancelled + } + } + + public func status() -> EventStreamRecordingStatus { + lock.lock() + if let activeSession { + let status = activeSession.status + lock.unlock() + return status + } + lock.unlock() + + return (try? readLatestStatus()) ?? .idle + } + + public func stop(endReason: String = "recording_controls_stopped") throws -> EventStreamRecordingStatus { + try finish(state: .stopped, endReason: endReason) + } + + public func cancel() throws -> EventStreamRecordingStatus { + try finish(state: .cancelled, endReason: "recording_controls_cancelled") + } + + public func wait(sessionID: String? = nil, timeout: TimeInterval? = nil) -> EventStreamRecordingStatus { + waitResult(sessionID: sessionID, timeout: timeout).status + } + + public func waitResult(sessionID: String? = nil, timeout: TimeInterval? = nil) -> EventStreamWaitResult { + let deadline = timeout.map { Date().addingTimeInterval($0) } + while true { + if let sessionID, + let requestedStatus = try? readStoredStatus(sessionID: sessionID), + !requestedStatus.isActive { + return EventStreamWaitResult(status: requestedStatus, timedOut: false, sessionMatched: true) + } + + let current = status() + let matchesSession = sessionID == nil || current.sessionID == sessionID + if matchesSession, !current.isActive { + return EventStreamWaitResult(status: current, timedOut: false, sessionMatched: true) + } + if sessionID != nil, !matchesSession { + return EventStreamWaitResult(status: current, timedOut: true, sessionMatched: false) + } + if let deadline, Date() >= deadline { + return EventStreamWaitResult(status: current, timedOut: true, sessionMatched: matchesSession) + } + Thread.sleep(forTimeInterval: 0.1) + } + } + + public func appendSyntheticEventForTesting(_ event: [String: Any]) throws { + try appendEvent(event) + } + + public func appendSuppressedEventForTesting(_ event: [String: Any]) throws { + try appendSuppressedEvent(event) + } + + public func appendDebugErrorForTesting( + subsystem: String, + reason: String, + error: Error? = nil, + context: [String: Any] = [:] + ) throws { + try recordDebugError(subsystem: subsystem, reason: reason, error: error, context: context) + } + + func appendFocusedWindowChangedForTesting( + reason: String, + appName: String, + bundleIdentifier: String?, + snapshot: AppSnapshot + ) throws { + var payload: [String: Any] = [ + "type": "AX.focusedWindowChanged", + "timestamp": eventStreamTimestamp(Date()), + "reason": reason, + "screenshotNeededForContext": false, + "app": eventStreamApplicationContext( + appName: appName, + bundleIdentifier: bundleIdentifier, + pid: snapshot.app.pid + ), + ] + if let window = eventStreamWindowContext(snapshot: snapshot) { + payload["window"] = window + } + if let accessibilityPayload = recordAccessibilityPayload( + snapshot: snapshot, + appName: appName, + bundleIdentifier: bundleIdentifier + ) { + payload["accessibilityInspectorPayload"] = accessibilityPayload + payload["diffFromPrevious"] = accessibilityPayload["diffFromPrevious"] as? Bool ?? false + payload["screenshotNeededForContext"] = accessibilityPayload["screenshotNeededForContext"] as? Bool ?? false + } + try appendEvent(payload) + } + + private func finish(state: EventStreamRecordingState, endReason: String) throws -> EventStreamRecordingStatus { + try finish(state: state, endReason: endReason, onlySessionID: nil) + } + + private func finish( + state: EventStreamRecordingState, + endReason: String, + onlySessionID: String? + ) throws -> EventStreamRecordingStatus { + lock.lock() + guard let session = activeSession else { + lock.unlock() + return (try? readLatestStatus()) ?? .idle + } + if let onlySessionID, session.id != onlySessionID { + let status = session.status + lock.unlock() + return status + } + if session.isFinishing { + let status = session.status + lock.unlock() + return status + } + + let endedAt = Date() + session.endedAt = eventStreamTimestamp(endedAt) + session.endReason = endReason + session.isFinishing = true + lock.unlock() + + removeInputMonitors() + dismissRecordingControls() + try appendEvent([ + "type": "session.ended", + "timestamp": eventStreamTimestamp(endedAt), + "sessionId": session.id, + "endReason": endReason, + ], sessionOverride: session) + lock.lock() + session.state = state + session.isFinishing = false + let finalStatus = session.status + lock.unlock() + try writeStatus(finalStatus) + lock.lock() + if activeSession === session { + activeSession = nil + } + lock.unlock() + return finalStatus + } + + private func scheduleMaximumDurationStopIfNeeded(sessionID: String) { + guard maximumDuration > 0 else { + return + } + + DispatchQueue.global(qos: .utility).asyncAfter(deadline: .now() + maximumDuration) { [weak self] in + _ = try? self?.finish( + state: .stopped, + endReason: eventStreamMaximumDurationEndReason, + onlySessionID: sessionID + ) + } + } + + private func appendEvent(_ event: [String: Any], sessionOverride: EventStreamSession? = nil) throws { + lock.lock() + guard let session = sessionOverride ?? activeSession else { + lock.unlock() + return + } + session.eventCount += 1 + let eventURL = session.eventsURL + let status = session.status + lock.unlock() + + var payload = eventStreamCanonicalEventPayload(event) + if payload["timestamp"] == nil { + payload["timestamp"] = eventStreamTimestamp(Date()) + } + if payload["sessionId"] == nil { + payload["sessionId"] = session.id + } + + try appendJSONObjectLine(payload, to: eventURL) + try writeStatus(status) + } + + private func appendSuppressedEvent(_ event: [String: Any]) throws { + lock.lock() + guard let session = activeSession else { + lock.unlock() + return + } + session.suppressedEventCount += 1 + let suppressedURL = session.suppressedEventsURL + let status = session.status + lock.unlock() + + var payload = eventStreamCanonicalEventPayload(event) + if payload["timestamp"] == nil { + payload["timestamp"] = eventStreamTimestamp(Date()) + } + if payload["sessionId"] == nil { + payload["sessionId"] = session.id + } + + try appendJSONObjectLine(payload, to: suppressedURL) + try writeStatus(status) + } + + private func recordDebugError( + subsystem: String, + reason: String, + error: Error? = nil, + context: [String: Any] = [:] + ) throws { + let payload = eventStreamDebugErrorPayload( + subsystem: subsystem, + reason: reason, + error: error, + context: context + ) + try appendEvent(payload) + } + + private func writeStatus(_ status: EventStreamRecordingStatus) throws { + let metadataURL: URL? + if let metadataPath = status.metadataPath { + metadataURL = URL(fileURLWithPath: metadataPath) + } else { + metadataURL = nil + } + + let data = try JSONSerialization.data( + withJSONObject: status.asDictionary, + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + let sessionHandoffData = try JSONSerialization.data( + withJSONObject: status.sessionHandoffDictionary, + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + + if let metadataURL { + try data.write(to: metadataURL, options: [.atomic]) + try sessionHandoffData.write( + to: metadataURL.deletingLastPathComponent().appendingPathComponent("session.json"), + options: [.atomic] + ) + } + + try FileManager.default.createDirectory(at: rootDirectory, withIntermediateDirectories: true) + try data.write(to: rootDirectory.appendingPathComponent("latest-session.json"), options: [.atomic]) + if status.isActive { + try data.write(to: rootDirectory.appendingPathComponent("active-session.json"), options: [.atomic]) + } else { + try? FileManager.default.removeItem(at: rootDirectory.appendingPathComponent("active-session.json")) + } + } + + private func readLatestStatus() throws -> EventStreamRecordingStatus { + let data = try Data(contentsOf: rootDirectory.appendingPathComponent("latest-session.json")) + let decoded = try JSONDecoder().decode(EventStreamRecordingStatus.self, from: data) + if decoded.isActive { + let unavailableStatus = EventStreamRecordingStatus( + sessionID: decoded.sessionID, + state: .stopped, + startedAt: decoded.startedAt, + endedAt: eventStreamTimestamp(Date()), + endReason: "recording_process_unavailable", + eventsPath: decoded.eventsPath, + metadataPath: decoded.metadataPath, + suppressedEventsPath: decoded.suppressedEventsPath, + currentSegmentEventsPath: nil, + currentSegmentMetadataPath: nil, + eventCount: decoded.eventCount, + suppressedEventCount: decoded.suppressedEventCount + ) + try? writeStatus(unavailableStatus) + return unavailableStatus + } + return decoded + } + + private func readStoredStatus(sessionID: String) throws -> EventStreamRecordingStatus { + guard eventStreamSessionIdentifierIsSafe(sessionID) else { + throw ComputerUseError.message("Invalid event stream session id.") + } + let sessionDirectory = rootDirectory.appendingPathComponent(sessionID, isDirectory: true) + let metadataURL = sessionDirectory.appendingPathComponent("metadata.json") + let sessionURL = sessionDirectory.appendingPathComponent("session.json") + let statusURL = FileManager.default.fileExists(atPath: metadataURL.path) ? metadataURL : sessionURL + let data = try Data(contentsOf: statusURL) + return try JSONDecoder().decode(EventStreamRecordingStatus.self, from: data) + } + + private func installInputMonitorsIfNeeded() { + guard installsInputMonitors else { + return + } + + performEventStreamOnMainSync { [self] in + guard self.inputMonitorTokens.isEmpty, self.workspaceObserver == nil else { + return + } + + if self.installsCGEventTap, let token = self.installCGEventTapInputMonitor() { + self.inputMonitorTokens.append(token) + } else { + self.installNSEventInputMonitors() + } + + self.workspaceObserver = NSWorkspace.shared.notificationCenter.addObserver( + forName: NSWorkspace.didActivateApplicationNotification, + object: nil, + queue: .main + ) { [weak self] _ in + self?.recordFocusedWindowChanged(reason: "applicationActivated") + } + + if self.inputMonitorTokens.isEmpty { + try? self.recordDebugError( + subsystem: "inputMonitoring", + reason: "inputMonitorsUnavailable", + context: [ + "monitorsInstalled": 0, + "recordsRawEvents": self.recordsRawEvents, + ] + ) + } + } + } + + private func removeInputMonitors() { + guard installsInputMonitors else { + return + } + + performEventStreamOnMainSync { [self] in + for token in self.inputMonitorTokens { + token.remove() + } + self.inputMonitorTokens.removeAll() + self.lock.lock() + self.pendingPointerEvent = nil + self.lock.unlock() + + if let workspaceObserver = self.workspaceObserver { + NSWorkspace.shared.notificationCenter.removeObserver(workspaceObserver) + self.workspaceObserver = nil + } + } + } + + @MainActor + private func installNSEventInputMonitors() { + if let token = inputMonitorInstaller( + [.leftMouseDown, .rightMouseDown, .otherMouseDown], + { [weak self] event in + self?.recordMouseDown(event) + } + ) { + inputMonitorTokens.append(EventStreamInputMonitorToken { + NSEvent.removeMonitor(token) + }) + } + + if let token = inputMonitorInstaller( + [.leftMouseDragged, .rightMouseDragged, .otherMouseDragged], + { [weak self] event in + self?.recordMouseDragged(event) + } + ) { + inputMonitorTokens.append(EventStreamInputMonitorToken { + NSEvent.removeMonitor(token) + }) + } + + if let token = inputMonitorInstaller( + [.leftMouseUp, .rightMouseUp, .otherMouseUp], + { [weak self] event in + self?.recordMouseUp(event) + } + ) { + inputMonitorTokens.append(EventStreamInputMonitorToken { + NSEvent.removeMonitor(token) + }) + } + + if let token = inputMonitorInstaller( + [.keyDown], + { [weak self] event in + self?.recordKeyDown(event) + } + ) { + inputMonitorTokens.append(EventStreamInputMonitorToken { + NSEvent.removeMonitor(token) + }) + } + + if recordsRawEvents, + let token = inputMonitorInstaller( + [.scrollWheel], + { [weak self] event in + self?.recordScrollWheel(event) + } + ) + { + inputMonitorTokens.append(EventStreamInputMonitorToken { + NSEvent.removeMonitor(token) + }) + } + } + + @MainActor + private func installCGEventTapInputMonitor() -> EventStreamInputMonitorToken? { + var mask: CGEventMask = 0 + for type in [ + CGEventType.leftMouseDown, + .rightMouseDown, + .otherMouseDown, + .leftMouseDragged, + .rightMouseDragged, + .otherMouseDragged, + .leftMouseUp, + .rightMouseUp, + .otherMouseUp, + .keyDown, + ] { + mask |= CGEventMask(1) << CGEventMask(type.rawValue) + } + if recordsRawEvents { + mask |= CGEventMask(1) << CGEventMask(CGEventType.scrollWheel.rawValue) + } + + let context = EventStreamCGEventTapContext(service: self) + guard let tap = CGEvent.tapCreate( + tap: .cgSessionEventTap, + place: .headInsertEventTap, + options: .listenOnly, + eventsOfInterest: mask, + callback: eventStreamCGEventTapCallback, + userInfo: Unmanaged.passUnretained(context).toOpaque() + ) else { + return nil + } + + guard let source = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0) else { + CFMachPortInvalidate(tap) + return nil + } + + CFRunLoopAddSource(CFRunLoopGetMain(), source, .commonModes) + CGEvent.tapEnable(tap: tap, enable: true) + + return EventStreamInputMonitorToken(context: context) { + CGEvent.tapEnable(tap: tap, enable: false) + CFRunLoopRemoveSource(CFRunLoopGetMain(), source, .commonModes) + CFMachPortInvalidate(tap) + } + } + + fileprivate func recordInputEventFromCGEventTap(_ event: NSEvent, type: CGEventType) { + switch type { + case .leftMouseDown, .rightMouseDown, .otherMouseDown: + recordMouseDown(event) + case .leftMouseDragged, .rightMouseDragged, .otherMouseDragged: + recordMouseDragged(event) + case .leftMouseUp, .rightMouseUp, .otherMouseUp: + recordMouseUp(event) + case .keyDown: + recordKeyDown(event) + case .scrollWheel: + recordScrollWheel(event) + default: + break + } + } + + private func showRecordingControlsIfNeeded(startedAt: Date) { + guard showsRecordingControls, installsInputMonitors else { + return + } + + performEventStreamOnMainSync { [self] in + guard NSApp.isRunning else { + return + } + + self.recordingControls?.close() + let controls = EventStreamRecordingControls( + startedAt: startedAt, + onStop: { [weak self] in + _ = try? self?.stop() + }, + onCancel: { [weak self] in + _ = try? self?.cancel() + } + ) + self.recordingControls = controls + controls.show() + } + } + + private func dismissRecordingControls() { + guard showsRecordingControls, installsInputMonitors else { + return + } + + performEventStreamOnMainSync { [self] in + self.recordingControls?.close() + self.recordingControls = nil + } + } + + private func recordMouseDown(_ event: NSEvent) { + guard let button = eventStreamMouseButton(forEventType: event.type) else { + return + } + + let appKitPoint = event.locationInWindow + let screenPoint = eventStreamScreenStatePoint(fromAppKitGlobalPoint: appKitPoint) + let pendingEvent = EventStreamPendingPointerEvent( + button: button, + clickCount: max(event.clickCount, 1), + startedAt: Date(), + appKitStartLocation: appKitPoint, + screenStartLocation: screenPoint, + rawEvents: recordsRawEvents ? [ + eventStreamRawEventSummary(event, screenPoint: screenPoint, button: button) + ] : [] + ) + + lock.lock() + pendingPointerEvent = pendingEvent + lock.unlock() + } + + private func recordMouseDragged(_ event: NSEvent) { + guard let button = eventStreamMouseButton(forEventType: event.type) else { + return + } + + let appKitPoint = event.locationInWindow + let screenPoint = eventStreamScreenStatePoint(fromAppKitGlobalPoint: appKitPoint) + + lock.lock() + if var pendingPointerEvent, pendingPointerEvent.button == button { + pendingPointerEvent.appKitLastLocation = appKitPoint + pendingPointerEvent.screenLastLocation = screenPoint + if recordsRawEvents { + pendingPointerEvent.appendRawEvent( + eventStreamRawEventSummary(event, screenPoint: screenPoint, button: button) + ) + } + pendingPointerEvent.didDrag = eventStreamPointerDidDrag( + start: pendingPointerEvent.screenStartLocation, + end: screenPoint, + observedDrag: pendingPointerEvent.didDrag + ) + self.pendingPointerEvent = pendingPointerEvent + } + lock.unlock() + } + + private func recordMouseUp(_ event: NSEvent) { + guard let button = eventStreamMouseButton(forEventType: event.type) else { + return + } + + let appKitPoint = event.locationInWindow + let screenPoint = eventStreamScreenStatePoint(fromAppKitGlobalPoint: appKitPoint) + let fallbackPendingEvent = EventStreamPendingPointerEvent( + button: button, + clickCount: max(event.clickCount, 1), + startedAt: Date(), + appKitStartLocation: appKitPoint, + screenStartLocation: screenPoint, + rawEvents: recordsRawEvents ? [ + eventStreamRawEventSummary(event, screenPoint: screenPoint, button: button) + ] : [] + ) + + lock.lock() + let pendingEvent = pendingPointerEvent?.button == button ? pendingPointerEvent : fallbackPendingEvent + pendingPointerEvent = nil + lock.unlock() + + guard var pointerEvent = pendingEvent else { + return + } + pointerEvent.appKitLastLocation = appKitPoint + pointerEvent.screenLastLocation = screenPoint + if recordsRawEvents { + pointerEvent.appendRawEvent(eventStreamRawEventSummary(event, screenPoint: screenPoint, button: button)) + } + + if eventStreamPointerDidDrag( + start: pointerEvent.screenStartLocation, + end: pointerEvent.screenLastLocation, + observedDrag: pointerEvent.didDrag + ) { + recordFocusedWindowChangedIfNeeded(reason: "inputContextChanged") + recordMouseDrag(pointerEvent) + recordSelectionChangedIfNeeded(reason: "mouse.drag") + } else { + recordFocusedWindowChangedIfNeeded(reason: "inputContextChanged") + recordMouseClick(pointerEvent) + recordSelectionChangedIfNeeded(reason: eventStreamMouseEventType(forButton: pointerEvent.button)) + } + } + + private func recordMouseClick(_ pointerEvent: EventStreamPendingPointerEvent) { + var payload: [String: Any] = [ + "type": eventStreamMouseEventType(forButton: pointerEvent.button), + "timestamp": eventStreamTimestamp(Date()), + "button": pointerEvent.button, + "clickCount": pointerEvent.clickCount, + "location": [ + "x": pointerEvent.screenLastLocation.x, + "y": pointerEvent.screenLastLocation.y, + ], + "appKitLocation": [ + "x": pointerEvent.appKitLastLocation.x, + "y": pointerEvent.appKitLastLocation.y, + ], + ] + payload.merge(currentEventStreamAppContext()) { current, _ in current } + if let target = currentEventStreamPointerTarget(atScreenPoint: pointerEvent.screenLastLocation) { + payload["targetAccessibilityElement"] = target + } + if recordsRawEvents, !pointerEvent.rawEvents.isEmpty { + payload["experimentalRawEvents"] = pointerEvent.rawEvents + } + try? appendEvent(payload) + } + + private func recordMouseDrag(_ pointerEvent: EventStreamPendingPointerEvent) { + let distance = eventStreamPointerDistance(pointerEvent.screenStartLocation, pointerEvent.screenLastLocation) + var payload: [String: Any] = [ + "type": "mouse.drag", + "timestamp": eventStreamTimestamp(Date()), + "button": pointerEvent.button, + "startLocation": [ + "x": pointerEvent.screenStartLocation.x, + "y": pointerEvent.screenStartLocation.y, + ], + "endLocation": [ + "x": pointerEvent.screenLastLocation.x, + "y": pointerEvent.screenLastLocation.y, + ], + "startAppKitLocation": [ + "x": pointerEvent.appKitStartLocation.x, + "y": pointerEvent.appKitStartLocation.y, + ], + "endAppKitLocation": [ + "x": pointerEvent.appKitLastLocation.x, + "y": pointerEvent.appKitLastLocation.y, + ], + "distance": distance, + "durationMilliseconds": max(0, Int(Date().timeIntervalSince(pointerEvent.startedAt) * 1000)), + ] + payload.merge(currentEventStreamAppContext()) { current, _ in current } + if let target = currentEventStreamPointerTarget(atScreenPoint: pointerEvent.screenLastLocation) { + payload["targetAccessibilityElement"] = target + } + if recordsRawEvents, !pointerEvent.rawEvents.isEmpty { + payload["experimentalRawEvents"] = pointerEvent.rawEvents + } + try? appendEvent(payload) + } + + private func recordKeyDown(_ event: NSEvent) { + guard let payloadKind = eventStreamKeyboardPayloadKind( + characters: event.characters, + keyCode: event.keyCode, + modifierFlags: event.modifierFlags + ) else { + return + } + + recordFocusedWindowChangedIfNeeded(reason: "inputContextChanged") + let appContext = currentEventStreamAppContext() + let focusedElement = currentEventStreamFocusedElement() + let secureInput = focusedElement.map(eventStreamFocusedElementIsSecure) ?? false + let terminalInput = focusedElement.map { eventStreamFocusedElementIsTerminal($0, appContext: appContext) } ?? false + var payload: [String: Any] = [ + "type": payloadKind.type, + "timestamp": eventStreamTimestamp(Date()), + "keyCode": Int(event.keyCode), + ] + if let text = payloadKind.text { + if secureInput { + payload["secureInput"] = true + payload["textLength"] = text.count + } else { + payload["text"] = text + } + } + if let key = payloadKind.key { + payload["key"] = key + } + let modifiers = eventStreamModifierNames(event.modifierFlags) + if !modifiers.isEmpty { + payload["modifiers"] = modifiers + } + payload.merge(appContext) { current, _ in current } + if let focusedElement { + payload["focusedAccessibilityElement"] = terminalInput + ? eventStreamTerminalRedactedFocusedElement(focusedElement) + : focusedElement + } + if recordsRawEvents { + payload["experimentalRawEvents"] = [ + eventStreamRawEventSummary(event, redactCharacters: secureInput || terminalInput) + ] + } + try? appendEvent(payload) + if let focusedElement { + recordTerminalValueChangedIfNeeded(reason: payloadKind.type, focusedElement: focusedElement, appContext: appContext) + } + recordSelectionChangedIfNeeded(reason: payloadKind.type) + } + + private func recordScrollWheel(_ event: NSEvent) { + guard recordsRawEvents else { + return + } + + let appKitPoint = event.locationInWindow + let screenPoint = eventStreamScreenStatePoint(fromAppKitGlobalPoint: appKitPoint) + recordFocusedWindowChangedIfNeeded(reason: "inputContextChanged") + var payload: [String: Any] = [ + "type": "experimentalRawEvents", + "timestamp": eventStreamTimestamp(Date()), + "reason": "scrollWheel", + "experimentalRawEvents": [ + eventStreamRawEventSummary(event, screenPoint: screenPoint) + ], + ] + payload.merge(currentEventStreamAppContext()) { current, _ in current } + if let target = currentEventStreamPointerTarget(atScreenPoint: screenPoint) { + payload["targetAccessibilityElement"] = target + } + try? appendEvent(payload) + recordSelectionChangedIfNeeded(reason: "scrollWheel") + } + + private func recordFocusedWindowChangedIfNeeded(reason: String) { + guard let currentContextKey = currentEventStreamWindowContextKey() else { + return + } + + lock.lock() + let shouldRecord = lastObservedWindowContextKey != currentContextKey + if shouldRecord { + lastObservedWindowContextKey = currentContextKey + } + lock.unlock() + + if shouldRecord { + recordWindowChanged(reason: reason, observedContextKey: currentContextKey) + recordFocusedWindowChanged(reason: reason, observedContextKey: currentContextKey) + } + } + + private func recordSelectionChangedIfNeeded(reason: String) { + guard let focusedElement = currentEventStreamFocusedElement() else { + return + } + + let selectedText = focusedElement["selectedText"] as? String ?? "" + let signature = selectedText.isEmpty + ? eventStreamSelectionClearedSignature(focusedElement: focusedElement) + : eventStreamSelectionSignature(focusedElement: focusedElement) + let selectionCleared = selectedText.isEmpty + + guard let signature else { + return + } + + lock.lock() + let shouldRecord: Bool + if selectionCleared { + shouldRecord = lastSelectionSignature != nil + if shouldRecord { + lastSelectionSignature = nil + } + } else { + shouldRecord = lastSelectionSignature != signature + if shouldRecord { + lastSelectionSignature = signature + } + } + lock.unlock() + + guard shouldRecord else { + return + } + + var payload = eventStreamSelectionChangedPayload( + reason: reason, + selectedText: selectedText, + selectionSignature: signature, + focusedElement: focusedElement, + selectionCleared: selectionCleared + ) + payload.merge(currentEventStreamAppContext()) { current, _ in current } + try? appendEvent(payload) + } + + private func recordTerminalValueChangedIfNeeded( + reason: String, + focusedElement: [String: Any], + appContext: [String: Any] + ) { + guard eventStreamFocusedElementIsTerminal(focusedElement, appContext: appContext), + let value = eventStreamTerminalValue(focusedElement), + !eventStreamFocusedElementIsSecure(focusedElement) + else { + return + } + + let valueHash = eventStreamStableTextHash(value) + let signature = eventStreamTerminalValueSignature(focusedElement: focusedElement, valueHash: valueHash) + + lock.lock() + let shouldRecord = lastTerminalValueSignature != signature + if shouldRecord { + lastTerminalValueSignature = signature + } + lock.unlock() + + guard shouldRecord else { + return + } + + var payload: [String: Any] = [ + "type": "terminal.value_changed", + "timestamp": eventStreamTimestamp(Date()), + "reason": reason, + "valueHash": valueHash, + "valueLength": value.count, + "terminalValueSignature": signature, + "focusedAccessibilityElement": eventStreamTerminalRedactedFocusedElement(focusedElement), + ] + payload.merge(appContext) { current, _ in current } + try? appendEvent(payload) + } + + private func recordFocusedWindowChanged(reason: String) { + recordFocusedWindowChanged(reason: reason, observedContextKey: nil) + } + + private func recordWindowChanged(reason: String, observedContextKey: String? = nil) { + guard let contextKey = observedContextKey ?? currentEventStreamWindowContextKey() else { + return + } + + var payload = eventStreamWindowChangedPayload( + reason: reason, + contextKey: contextKey, + context: currentEventStreamAppContext() + ) + if payload["window"] == nil { + payload["windowUnavailable"] = true + } + try? appendEvent(payload) + } + + private func recordFocusedWindowChanged(reason: String, observedContextKey: String?) { + if let contextKey = observedContextKey ?? currentEventStreamWindowContextKey() { + lock.lock() + lastObservedWindowContextKey = contextKey + lock.unlock() + } + + var payload: [String: Any] = [ + "type": "AX.focusedWindowChanged", + "timestamp": eventStreamTimestamp(Date()), + "reason": reason, + "screenshotNeededForContext": false, + ] + payload.merge(currentEventStreamAppContext()) { current, _ in current } + if let accessibilityPayload = recordAccessibilityPayload() { + payload["accessibilityInspectorPayload"] = accessibilityPayload + payload["diffFromPrevious"] = accessibilityPayload["diffFromPrevious"] as? Bool ?? false + payload["screenshotNeededForContext"] = accessibilityPayload["screenshotNeededForContext"] as? Bool ?? false + } + try? appendEvent(payload) + } + + private func recordAccessibilityPayload() -> [String: Any]? { + guard let app = NSWorkspace.shared.frontmostApplication else { + try? appendSuppressedEvent([ + "type": "AX.snapshot.suppressed", + "reason": "frontmostApplicationUnavailable", + ]) + try? recordDebugError( + subsystem: "accessibility", + reason: "frontmostApplicationUnavailable" + ) + return nil + } + + let descriptor = RunningAppDescriptor( + name: AppDiscovery.appName(app), + bundleIdentifier: app.bundleIdentifier, + pid: app.processIdentifier, + runningApplication: app + ) + + do { + let snapshot = try SnapshotBuilder.build( + for: descriptor, + includeScreenshot: screenshotPolicy != .never + ) + return recordAccessibilityPayload( + snapshot: snapshot, + appName: descriptor.name, + bundleIdentifier: descriptor.bundleIdentifier + ) + } catch { + try? appendSuppressedEvent([ + "type": "AX.snapshot.suppressed", + "reason": "snapshotUnavailable", + "message": (error as? LocalizedError)?.errorDescription ?? String(describing: error), + "app": [ + "name": descriptor.name, + "pid": Int(descriptor.pid), + "bundleIdentifier": descriptor.bundleIdentifier ?? "", + ], + ]) + try? recordDebugError( + subsystem: "accessibility", + reason: "snapshotUnavailable", + error: error, + context: [ + "app": [ + "name": descriptor.name, + "pid": Int(descriptor.pid), + "bundleIdentifier": descriptor.bundleIdentifier ?? "", + ], + ] + ) + return nil + } + } + + private func recordAccessibilityPayload( + snapshot: AppSnapshot, + appName: String, + bundleIdentifier: String? + ) -> [String: Any]? { + let contextKey = eventStreamAXContextKey( + appName: appName, + bundleIdentifier: bundleIdentifier, + windowTitle: snapshot.windowTitle + ) + let currentTreeLines = snapshot.treeLines + + lock.lock() + let previousTreeLines = lastAXContextKey == contextKey ? lastAXTreeLines : nil + let baselineTreeLines = initialAXContextKey == contextKey ? initialAXTreeLines : nil + if initialAXContextKey != contextKey { + initialAXContextKey = contextKey + initialAXTreeLines = currentTreeLines + } + lastAXContextKey = contextKey + lastAXTreeLines = currentTreeLines + lock.unlock() + + if let previousTreeLines, + let difference = eventStreamCompactAXDiff( + previous: previousTreeLines, + current: currentTreeLines + ) + { + let cumulativeDifference = baselineTreeLines.flatMap { baselineTreeLines in + eventStreamCompactAXDiff( + previous: baselineTreeLines, + current: currentTreeLines + ) + } + if baselineTreeLines != nil, cumulativeDifference == nil { + try? appendSuppressedEvent([ + "type": "AX.diff.suppressed", + "reason": "cumulativeDiffExceededBudget", + "contextKey": contextKey, + ]) + try? recordDebugError( + subsystem: "accessibility", + reason: "cumulativeDiffExceededBudget", + context: [ + "contextKeyHash": eventStreamStableTextHash(contextKey), + ] + ) + } + let screenshotContext = persistScreenshotIfNeeded( + snapshot: snapshot, + payloadKind: "diff", + treeLines: difference.lines + ) + return eventStreamAccessibilityPayload( + kind: "diff", + snapshot: snapshot, + renderedText: difference.renderedText, + treeLines: difference.lines, + diffFromPrevious: true, + cumulativeDifference: cumulativeDifference, + screenshotContext: screenshotContext + ) + } + + let screenshotContext = persistScreenshotIfNeeded( + snapshot: snapshot, + payloadKind: "full", + treeLines: currentTreeLines + ) + return eventStreamAccessibilityPayload( + kind: "full", + snapshot: snapshot, + renderedText: snapshot.renderedText, + treeLines: currentTreeLines, + diffFromPrevious: false, + cumulativeDifference: nil, + screenshotContext: screenshotContext + ) + } + + private func persistScreenshotIfNeeded( + snapshot: AppSnapshot, + payloadKind: String, + treeLines: [String] + ) -> EventStreamScreenshotContext { + let isNeeded = eventStreamScreenshotNeededForContext( + policy: screenshotPolicy, + payloadKind: payloadKind, + treeLineCount: treeLines.count, + hasFocusedSummary: snapshot.focusedSummary != nil + ) + guard isNeeded else { + return EventStreamScreenshotContext(needed: false, path: nil) + } + + guard let screenshotPNGData = snapshot.screenshotPNGData else { + return EventStreamScreenshotContext(needed: true, path: nil) + } + + lock.lock() + let screenshotsDirectory = activeSession?.screenshotsDirectory + lock.unlock() + + guard let screenshotsDirectory else { + return EventStreamScreenshotContext(needed: true, path: nil) + } + + do { + try FileManager.default.createDirectory(at: screenshotsDirectory, withIntermediateDirectories: true) + let filename = "\(eventStreamCompactTimestamp(Date()))-\(payloadKind)-context.png" + let url = screenshotsDirectory.appendingPathComponent(filename) + try screenshotPNGData.write(to: url, options: [.atomic]) + return EventStreamScreenshotContext(needed: true, path: url.path) + } catch { + try? appendSuppressedEvent([ + "type": "screenshot.suppressed", + "reason": "writeFailed", + "message": (error as? LocalizedError)?.errorDescription ?? String(describing: error), + ]) + try? recordDebugError( + subsystem: "screenshot", + reason: "writeFailed", + error: error + ) + return EventStreamScreenshotContext(needed: true, path: nil) + } + } +} + +private final class EventStreamSession { + let id: String + let directory: URL + let startedAt: String + var endedAt: String? + var endReason: String? + var state: EventStreamRecordingState = .recording + var isFinishing = false + var eventCount = 0 + var suppressedEventCount = 0 + + init(id: String, directory: URL, startedAtDate: Date) { + self.id = id + self.directory = directory + self.startedAt = eventStreamTimestamp(startedAtDate) + } + + var eventsURL: URL { + directory.appendingPathComponent("events.jsonl") + } + + var metadataURL: URL { + directory.appendingPathComponent("metadata.json") + } + + var sessionURL: URL { + directory.appendingPathComponent("session.json") + } + + var suppressedEventsURL: URL { + directory.appendingPathComponent("suppressed.jsonl") + } + + var screenshotsDirectory: URL { + directory.appendingPathComponent("screenshots", isDirectory: true) + } + + var status: EventStreamRecordingStatus { + EventStreamRecordingStatus( + sessionID: id, + state: state, + startedAt: startedAt, + endedAt: endedAt, + endReason: endReason, + eventsPath: eventsURL.path, + metadataPath: metadataURL.path, + suppressedEventsPath: suppressedEventsURL.path, + currentSegmentEventsPath: state == .recording ? eventsURL.path : nil, + currentSegmentMetadataPath: state == .recording ? metadataURL.path : nil, + eventCount: eventCount, + suppressedEventCount: suppressedEventCount + ) + } +} + +private struct EventStreamPendingPointerEvent { + let button: String + let clickCount: Int + let startedAt: Date + let appKitStartLocation: CGPoint + let screenStartLocation: CGPoint + var appKitLastLocation: CGPoint + var screenLastLocation: CGPoint + var didDrag = false + var rawEvents: [[String: Any]] + + init( + button: String, + clickCount: Int, + startedAt: Date, + appKitStartLocation: CGPoint, + screenStartLocation: CGPoint, + rawEvents: [[String: Any]] + ) { + self.button = button + self.clickCount = clickCount + self.startedAt = startedAt + self.appKitStartLocation = appKitStartLocation + self.screenStartLocation = screenStartLocation + self.appKitLastLocation = appKitStartLocation + self.screenLastLocation = screenStartLocation + self.rawEvents = rawEvents + } + + mutating func appendRawEvent(_ rawEvent: [String: Any]) { + if rawEvents.count < eventStreamMaximumRawEventsPerInputEvent { + rawEvents.append(rawEvent) + } else if !rawEvents.isEmpty { + rawEvents[rawEvents.count - 1] = rawEvent + } + } +} + +private func eventStreamApplicationContext( + appName: String, + bundleIdentifier: String?, + pid: pid_t +) -> [String: Any] { + var application: [String: Any] = [ + "name": appName, + "pid": Int(pid), + ] + if let bundleIdentifier { + application["bundleIdentifier"] = bundleIdentifier + } + return application +} + +private func eventStreamWindowContext(snapshot: AppSnapshot) -> [String: Any]? { + var window: [String: Any] = [:] + if let windowTitle = snapshot.windowTitle, !windowTitle.isEmpty { + window["title"] = windowTitle + } + if let targetWindowID = snapshot.targetWindowID { + window["id"] = targetWindowID + } + if let targetWindowLayer = snapshot.targetWindowLayer { + window["layer"] = targetWindowLayer + } + if let windowBounds = snapshot.windowBounds { + window["bounds"] = [ + "x": windowBounds.origin.x, + "y": windowBounds.origin.y, + "width": windowBounds.size.width, + "height": windowBounds.size.height, + ] + } + return window.isEmpty ? nil : window +} + +private func currentEventStreamAppContext() -> [String: Any] { + guard let app = NSWorkspace.shared.frontmostApplication else { + return [:] + } + + var context: [String: Any] = [ + "app": eventStreamApplicationContext( + appName: AppDiscovery.appName(app), + bundleIdentifier: app.bundleIdentifier, + pid: app.processIdentifier + ), + ] + if let window = frontmostWindowContext(pid: app.processIdentifier) { + context["window"] = window + } + return context +} + +private func currentEventStreamWindowContextKey() -> String? { + guard let app = NSWorkspace.shared.frontmostApplication else { + return nil + } + + let windowTitle = frontmostWindowContext(pid: app.processIdentifier)?["title"] as? String + return eventStreamAXContextKey( + appName: AppDiscovery.appName(app), + bundleIdentifier: app.bundleIdentifier, + windowTitle: windowTitle + ) +} + +func eventStreamAXContextKey( + appName: String, + bundleIdentifier: String?, + windowTitle: String? +) -> String { + [ + bundleIdentifier ?? appName, + windowTitle ?? "", + ].joined(separator: "\u{1f}") +} + +func eventStreamWindowChangedPayload( + reason: String, + contextKey: String, + context: [String: Any], + timestamp: String = eventStreamTimestamp(Date()) +) -> [String: Any] { + var payload: [String: Any] = [ + "type": "window.changed", + "timestamp": timestamp, + "reason": reason, + "windowContextKey": contextKey, + ] + payload.merge(context) { current, _ in current } + return payload +} + +func eventStreamDebugErrorPayload( + subsystem: String, + reason: String, + error: Error? = nil, + context: [String: Any] = [:], + timestamp: String = eventStreamTimestamp(Date()) +) -> [String: Any] { + var payload: [String: Any] = [ + "type": "debug.error", + "timestamp": timestamp, + "subsystem": subsystem, + "reason": reason, + ] + if let error { + payload["errorType"] = String(describing: Swift.type(of: error)) + } + payload.merge(eventStreamDebugSafeContext(context)) { current, _ in current } + return payload +} + +private func eventStreamDebugSafeContext(_ context: [String: Any]) -> [String: Any] { + var safeContext: [String: Any] = [:] + for (key, value) in context { + if let nested = value as? [String: Any] { + let sanitizedNested = eventStreamDebugSafeContext(nested) + if !sanitizedNested.isEmpty { + safeContext[key] = sanitizedNested + } + } else if let string = value as? String { + if key.lowercased().contains("path") { + continue + } + safeContext[key] = eventStreamTruncatedText(string, limit: 160) + } else if let number = value as? NSNumber { + safeContext[key] = number + } else if let boolean = value as? Bool { + safeContext[key] = boolean + } else if let integer = value as? Int { + safeContext[key] = integer + } else if let double = value as? Double { + safeContext[key] = double + } + } + return safeContext +} + +func eventStreamSelectionSignature(focusedElement: [String: Any]) -> String? { + guard let selectedText = focusedElement["selectedText"] as? String, + !selectedText.isEmpty + else { + return nil + } + + return [ + eventStreamSelectionSignaturePart(focusedElement["role"]), + eventStreamSelectionSignaturePart(focusedElement["subrole"]), + eventStreamSelectionSignaturePart(focusedElement["title"]), + eventStreamSelectionSignaturePart(focusedElement["label"]), + eventStreamSelectionSignaturePart(focusedElement["value"]), + selectedText, + eventStreamSelectionFrameSignature(focusedElement["frame"] as? [String: Any]), + ].joined(separator: "\u{1f}") +} + +func eventStreamSelectionClearedSignature(focusedElement: [String: Any]) -> String? { + let parts = [ + eventStreamSelectionSignaturePart(focusedElement["role"]), + eventStreamSelectionSignaturePart(focusedElement["subrole"]), + eventStreamSelectionSignaturePart(focusedElement["title"]), + eventStreamSelectionSignaturePart(focusedElement["label"]), + eventStreamSelectionSignaturePart(focusedElement["value"]), + eventStreamSelectionFrameSignature(focusedElement["frame"] as? [String: Any]), + ] + guard parts.contains(where: { !$0.isEmpty }) else { + return nil + } + + return (parts + [""]).joined(separator: "\u{1f}") +} + +func eventStreamSelectionChangedPayload( + reason: String, + selectedText: String, + selectionSignature: String, + focusedElement: [String: Any], + selectionCleared: Bool, + timestamp: String = eventStreamTimestamp(Date()) +) -> [String: Any] { + var payload: [String: Any] = [ + "type": "selection.changed", + "timestamp": timestamp, + "reason": reason, + "selectedText": selectedText, + "selectionSignature": selectionSignature, + "focusedAccessibilityElement": focusedElement, + ] + if selectionCleared { + payload["selectionCleared"] = true + } + return payload +} + +func eventStreamFocusedElementIsSecure(_ focusedElement: [String: Any]) -> Bool { + if focusedElement["secureInput"] as? Bool == true || focusedElement["protectedContent"] as? Bool == true { + return true + } + + let role = (focusedElement["role"] as? String ?? "").lowercased() + let subrole = (focusedElement["subrole"] as? String ?? "").lowercased() + let roleDescription = (focusedElement["roleDescription"] as? String ?? "").lowercased() + + if subrole.contains("secure") || subrole.contains("password") { + return true + } + if roleDescription.contains("secure") || roleDescription.contains("password") { + return true + } + return role.contains("secure") || role.contains("password") +} + +func eventStreamFocusedElementIsTerminal(_ focusedElement: [String: Any], appContext: [String: Any]) -> Bool { + if eventStreamFocusedElementIsSecure(focusedElement) { + return false + } + + let bundleIdentifier = ((appContext["app"] as? [String: Any])?["bundleIdentifier"] as? String ?? "").lowercased() + let appName = ((appContext["app"] as? [String: Any])?["name"] as? String ?? "").lowercased() + let terminalAppIdentifiers: Set = [ + "com.apple.terminal", + "com.googlecode.iterm2", + "dev.warp.warp-stable", + "dev.warp.warp", + ] + let terminalApp = terminalAppIdentifiers.contains(bundleIdentifier) + || appName == "terminal" + || appName == "iterm" + || appName == "iterm2" + || appName == "warp" + + let role = (focusedElement["role"] as? String ?? "").lowercased() + let subrole = (focusedElement["subrole"] as? String ?? "").lowercased() + let roleDescription = (focusedElement["roleDescription"] as? String ?? "").lowercased() + let title = (focusedElement["title"] as? String ?? "").lowercased() + let label = (focusedElement["label"] as? String ?? "").lowercased() + let textRole = role.contains("text") || role.contains("webarea") || role.contains("statictext") + let terminalHint = [role, subrole, roleDescription, title, label].contains { value in + value.contains("terminal") || value.contains("shell") || value.contains("console") + } + + return terminalApp && (textRole || terminalHint) +} + +func eventStreamTerminalValue(_ focusedElement: [String: Any]) -> String? { + guard let value = focusedElement["value"] as? String, + !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + else { + return nil + } + return value +} + +func eventStreamTerminalValueSignature(focusedElement: [String: Any], valueHash: String) -> String { + [ + eventStreamSelectionSignaturePart(focusedElement["role"]), + eventStreamSelectionSignaturePart(focusedElement["subrole"]), + eventStreamSelectionSignaturePart(focusedElement["title"]), + eventStreamSelectionSignaturePart(focusedElement["label"]), + eventStreamSelectionFrameSignature(focusedElement["frame"] as? [String: Any]), + valueHash, + ].joined(separator: "\u{1f}") +} + +func eventStreamTerminalRedactedFocusedElement(_ focusedElement: [String: Any]) -> [String: Any] { + var redacted = focusedElement + if let value = focusedElement["value"] as? String { + redacted["valueHash"] = eventStreamStableTextHash(value) + redacted["valueLength"] = value.count + redacted.removeValue(forKey: "value") + } + if let selectedText = focusedElement["selectedText"] as? String { + redacted["selectedTextLength"] = selectedText.count + redacted.removeValue(forKey: "selectedText") + } + redacted["terminalValueRedacted"] = true + return redacted +} + +func eventStreamStableTextHash(_ text: String) -> String { + var hash: UInt64 = 14_695_981_039_346_656_037 + for byte in text.utf8 { + hash ^= UInt64(byte) + hash = hash &* 1_099_511_628_211 + } + return String(format: "%016llx", hash) +} + +private func eventStreamSelectionSignaturePart(_ value: Any?) -> String { + value as? String ?? "" +} + +private func eventStreamSelectionFrameSignature(_ frame: [String: Any]?) -> String { + guard let frame else { + return "" + } + + return ["x", "y", "width", "height"] + .map { key in eventStreamSelectionNumberSignature(frame[key]) } + .joined(separator: ",") +} + +private func eventStreamSelectionNumberSignature(_ value: Any?) -> String { + if let number = value as? NSNumber { + return String(format: "%.3f", number.doubleValue) + } + if let value = value as? CGFloat { + return String(format: "%.3f", Double(value)) + } + if let value = value as? Double { + return String(format: "%.3f", value) + } + if let value = value as? Int { + return String(format: "%.3f", Double(value)) + } + return "" +} + +func eventStreamScreenStatePoint( + fromAppKitGlobalPoint point: CGPoint, + screenMappings: [VisualCursorScreenMapping] = currentVisualCursorScreenMappings() +) -> CGPoint { + guard let mapping = screenMappings.first(where: { $0.appKitFrame.contains(point) }) else { + return point + } + + let localX = point.x - mapping.appKitFrame.minX + let localY = mapping.appKitFrame.maxY - point.y + + return CGPoint( + x: mapping.screenStateFrame.minX + localX, + y: mapping.screenStateFrame.minY + localY + ) +} + +private func currentEventStreamPointerTarget(atScreenPoint point: CGPoint) -> [String: Any]? { + guard let app = NSWorkspace.shared.frontmostApplication else { + return nil + } + + let appElement = AXUIElementCreateApplication(app.processIdentifier) + var hitElement: AXUIElement? + let result = AXUIElementCopyElementAtPosition(appElement, Float(point.x), Float(point.y), &hitElement) + guard result == .success, let hitElement else { + return nil + } + + return eventStreamAccessibilityElementSummary( + hitElement, + windowBounds: currentEventStreamWindowBounds(forPID: app.processIdentifier) + ) +} + +private func currentEventStreamFocusedElement() -> [String: Any]? { + let systemWide = AXUIElementCreateSystemWide() + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(systemWide, kAXFocusedUIElementAttribute as CFString, &value) + guard result == .success, let value else { + return nil + } + + let element = (value as! AXUIElement) + return eventStreamAccessibilityElementSummary(element, windowBounds: nil) +} + +private func eventStreamAccessibilityElementSummary(_ element: AXUIElement, windowBounds: CGRect?) -> [String: Any] { + var summary: [String: Any] = [:] + + if let role = eventStreamStringValue(of: element, attribute: kAXRoleAttribute) { + summary["role"] = role + } + if let subrole = eventStreamStringValue(of: element, attribute: kAXSubroleAttribute) { + summary["subrole"] = subrole + } + if let roleDescription = eventStreamStringValue(of: element, attribute: kAXRoleDescriptionAttribute) { + summary["roleDescription"] = eventStreamTruncatedText(roleDescription) + } + if let title = eventStreamStringValue(of: element, attribute: kAXTitleAttribute) { + summary["title"] = eventStreamTruncatedText(title) + } + if let label = eventStreamStringValue(of: element, attribute: kAXDescriptionAttribute) { + summary["label"] = eventStreamTruncatedText(label) + } + if let value = eventStreamStringValue(of: element, attribute: kAXValueAttribute) { + summary["value"] = eventStreamTruncatedText(value) + } + if let selectedText = eventStreamStringValue(of: element, attribute: kAXSelectedTextAttribute) { + summary["selectedText"] = eventStreamTruncatedText(selectedText) + } + if eventStreamBooleanValue(of: element, attribute: "AXProtectedContent") == true { + summary["protectedContent"] = true + } + if let frame = eventStreamFrame(of: element) { + summary["frame"] = eventStreamRectDictionary(frame) + if let windowBounds { + summary["windowFrame"] = eventStreamRectDictionary(windowRelativeFrame(elementFrame: frame, windowBounds: windowBounds)) + } + } + if let actions = eventStreamActions(of: element), !actions.isEmpty { + summary["actions"] = actions + } + if eventStreamFocusedElementIsSecure(summary) { + summary["secureInput"] = true + summary.removeValue(forKey: "value") + summary.removeValue(forKey: "selectedText") + } + + return summary +} + +private func eventStreamStringValue(of element: AXUIElement, attribute: String) -> String? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + guard result == .success, let value else { + return nil + } + + if CFGetTypeID(value) == CFStringGetTypeID(), let string = value as? String { + let trimmed = string.trimmingCharacters(in: .whitespacesAndNewlines) + return trimmed.isEmpty ? nil : trimmed + } + + if CFGetTypeID(value) == CFBooleanGetTypeID() { + return CFBooleanGetValue((value as! CFBoolean)) ? "true" : "false" + } + + if let number = value as? NSNumber { + return number.stringValue + } + + return nil +} + +private func eventStreamBooleanValue(of element: AXUIElement, attribute: String) -> Bool? { + var value: CFTypeRef? + let result = AXUIElementCopyAttributeValue(element, attribute as CFString, &value) + guard result == .success, let value else { + return nil + } + + if CFGetTypeID(value) == CFBooleanGetTypeID() { + return CFBooleanGetValue((value as! CFBoolean)) + } + if let number = value as? NSNumber { + return number.boolValue + } + return nil +} + +private func eventStreamActions(of element: AXUIElement) -> [String]? { + var actions: CFArray? + let result = AXUIElementCopyActionNames(element, &actions) + guard result == .success else { + return nil + } + + return actions as? [String] +} + +private func eventStreamFrame(of element: AXUIElement) -> CGRect? { + var positionValue: CFTypeRef? + var sizeValue: CFTypeRef? + let positionResult = AXUIElementCopyAttributeValue(element, kAXPositionAttribute as CFString, &positionValue) + let sizeResult = AXUIElementCopyAttributeValue(element, kAXSizeAttribute as CFString, &sizeValue) + guard + positionResult == .success, + sizeResult == .success, + let positionValue, + let sizeValue + else { + return nil + } + + let positionAXValue = positionValue as! AXValue + let sizeAXValue = sizeValue as! AXValue + var position = CGPoint.zero + var size = CGSize.zero + guard AXValueGetValue(positionAXValue, .cgPoint, &position), + AXValueGetValue(sizeAXValue, .cgSize, &size) + else { + return nil + } + + return CGRect(origin: position, size: size) +} + +private func eventStreamRectDictionary(_ rect: CGRect) -> [String: Any] { + [ + "x": rect.origin.x, + "y": rect.origin.y, + "width": rect.size.width, + "height": rect.size.height, + ] +} + +private func currentEventStreamWindowBounds(forPID pid: pid_t) -> CGRect? { + guard let rawWindows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + return nil + } + + guard let window = rawWindows.first(where: { ($0[kCGWindowOwnerPID as String] as? pid_t) == pid }), + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let x = bounds["X"] as? NSNumber, + let y = bounds["Y"] as? NSNumber, + let width = bounds["Width"] as? NSNumber, + let height = bounds["Height"] as? NSNumber + else { + return nil + } + + return CGRect( + x: x.doubleValue, + y: y.doubleValue, + width: width.doubleValue, + height: height.doubleValue + ) +} + +private func eventStreamTruncatedText(_ text: String, limit: Int = 500) -> String { + guard text.count > limit else { + return text + } + + return String(text.prefix(limit)) + "..." +} + +struct EventStreamAXDifference: Equatable { + let lines: [String] + + var renderedText: String { + var output = ["The following is a diff from the previous accessibility tree"] + output.append(contentsOf: lines) + return output.joined(separator: "\n") + } + + var cumulativeRenderedText: String { + var output = ["The following is a cumulative diff from the initial accessibility tree"] + output.append(contentsOf: lines) + return output.joined(separator: "\n") + } +} + +struct EventStreamScreenshotContext: Equatable { + let needed: Bool + let path: String? +} + +func eventStreamCompactAXDiff( + previous: [String], + current: [String], + maxDiffLines: Int = 240 +) -> EventStreamAXDifference? { + guard previous != current else { + return EventStreamAXDifference(lines: []) + } + + guard let operations = eventStreamAXDiffOperations(previous: previous, current: current) else { + return nil + } + + var lines: [String] = [] + var pendingRemoved: [String] = [] + var pendingAdded: [String] = [] + + func flushPending() -> Bool { + guard !pendingRemoved.isEmpty || !pendingAdded.isEmpty else { + return true + } + + var consumedAddedIndexes = Set() + for removedLine in pendingRemoved { + if let removedIdentity = eventStreamAXLineIdentity(removedLine), + let addedIndex = pendingAdded.indices.first(where: { addedIndex in + !consumedAddedIndexes.contains(addedIndex) + && eventStreamAXLineIdentity(pendingAdded[addedIndex]) == removedIdentity + }) + { + lines.append("~ \(removedLine) -> \(pendingAdded[addedIndex])") + consumedAddedIndexes.insert(addedIndex) + } else { + lines.append("- \(removedLine)") + } + + if lines.count > maxDiffLines { + return false + } + } + + for addedIndex in pendingAdded.indices where !consumedAddedIndexes.contains(addedIndex) { + lines.append("+ \(pendingAdded[addedIndex])") + if lines.count > maxDiffLines { + return false + } + } + + pendingRemoved.removeAll() + pendingAdded.removeAll() + return true + } + + for operation in operations { + switch operation { + case .same: + guard flushPending() else { + return nil + } + case .removed(let line): + pendingRemoved.append(line) + case .added(let line): + pendingAdded.append(line) + } + } + + guard flushPending() else { + return nil + } + + return EventStreamAXDifference(lines: lines) +} + +private enum EventStreamAXDiffOperation: Equatable { + case same + case removed(String) + case added(String) +} + +private func eventStreamAXDiffOperations(previous: [String], current: [String]) -> [EventStreamAXDiffOperation]? { + let maxMatrixCells = 1_000_000 + let columns = current.count + 1 + guard (previous.count + 1) * columns <= maxMatrixCells else { + return nil + } + + let previousComparable = previous.map(eventStreamAXLineComparable) + let currentComparable = current.map(eventStreamAXLineComparable) + var table = Array(repeating: 0, count: (previous.count + 1) * columns) + + func offset(_ previousIndex: Int, _ currentIndex: Int) -> Int { + previousIndex * columns + currentIndex + } + + if !previous.isEmpty, !current.isEmpty { + for previousIndex in stride(from: previous.count - 1, through: 0, by: -1) { + for currentIndex in stride(from: current.count - 1, through: 0, by: -1) { + if previousComparable[previousIndex] == currentComparable[currentIndex] { + table[offset(previousIndex, currentIndex)] = table[offset(previousIndex + 1, currentIndex + 1)] + 1 + } else { + table[offset(previousIndex, currentIndex)] = max( + table[offset(previousIndex + 1, currentIndex)], + table[offset(previousIndex, currentIndex + 1)] + ) + } + } + } + } + + var operations: [EventStreamAXDiffOperation] = [] + var previousIndex = 0 + var currentIndex = 0 + while previousIndex < previous.count || currentIndex < current.count { + if previousIndex < previous.count, + currentIndex < current.count, + previousComparable[previousIndex] == currentComparable[currentIndex] + { + operations.append(.same) + previousIndex += 1 + currentIndex += 1 + } else if currentIndex < current.count, + (previousIndex == previous.count + || table[offset(previousIndex, currentIndex + 1)] >= table[offset(previousIndex + 1, currentIndex)]) + { + operations.append(.added(current[currentIndex])) + currentIndex += 1 + } else if previousIndex < previous.count { + operations.append(.removed(previous[previousIndex])) + previousIndex += 1 + } + } + + return operations +} + +private func eventStreamAXLineComparable(_ line: String) -> String { + line.replacingOccurrences( + of: #"^\s*\d+\s+"#, + with: "", + options: .regularExpression + ) +} + +private func eventStreamAXLineIdentity(_ line: String) -> String? { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard let firstSpace = trimmed.firstIndex(where: { $0 == " " || $0 == "\t" }) else { + return nil + } + let elementIndex = String(trimmed[.. [String: Any] { + var payload: [String: Any] = [ + "kind": kind, + "diffFromPrevious": diffFromPrevious, + "renderedText": renderedText, + "treeLines": treeLines, + "screenshotNeededForContext": screenshotContext.needed, + "screenshotAvailable": snapshot.screenshotPNGData != nil, + ] + if !diffFromPrevious { + payload["fullTree"] = treeLines + } + if let cumulativeDifference { + payload["cumulativeDiffFromInitial"] = true + payload["cumulativeRenderedText"] = cumulativeDifference.cumulativeRenderedText + payload["cumulativeTreeLines"] = cumulativeDifference.lines + } else { + payload["cumulativeDiffFromInitial"] = false + } + if let screenshotPath = screenshotContext.path { + payload["screenshotPath"] = screenshotPath + } + + if let windowTitle = snapshot.windowTitle { + payload["windowTitle"] = windowTitle + } + if let focusedSummary = snapshot.focusedSummary { + payload["focusedSummary"] = focusedSummary + } + if let selectedText = snapshot.selectedText { + payload["selectedText"] = selectedText + } + if let windowBounds = snapshot.windowBounds { + payload["windowBounds"] = [ + "x": windowBounds.origin.x, + "y": windowBounds.origin.y, + "width": windowBounds.size.width, + "height": windowBounds.size.height, + ] + } + + return payload +} + +private func frontmostWindowContext(pid: pid_t) -> [String: Any]? { + guard let rawWindows = CGWindowListCopyWindowInfo([.optionOnScreenOnly, .excludeDesktopElements], kCGNullWindowID) as? [[String: Any]] else { + return nil + } + + guard let window = rawWindows.first(where: { ($0[kCGWindowOwnerPID as String] as? pid_t) == pid }) else { + return nil + } + + var context: [String: Any] = [:] + if let windowID = window[kCGWindowNumber as String] as? NSNumber { + context["id"] = windowID.intValue + } + if let title = window[kCGWindowName as String] as? String, !title.isEmpty { + context["title"] = title + } + if let layer = window[kCGWindowLayer as String] as? NSNumber { + context["layer"] = layer.intValue + } + if let bounds = window[kCGWindowBounds as String] as? [String: Any] { + context["bounds"] = bounds + } + + return context.isEmpty ? nil : context +} + +private func defaultEventStreamRootDirectory() -> URL { + if let override = ProcessInfo.processInfo.environment["OPEN_COMPUTER_USE_EVENT_STREAM_DIR"], + !override.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + { + return URL(fileURLWithPath: override, isDirectory: true) + } + + let applicationSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first + ?? FileManager.default.temporaryDirectory + return applicationSupport + .appendingPathComponent("OpenComputerUse", isDirectory: true) + .appendingPathComponent("recordings", isDirectory: true) +} + +public func eventStreamRecordingControlsEnabled(environment: [String: String] = ProcessInfo.processInfo.environment) -> Bool { + guard let rawValue = environment["OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + else { + return true + } + + return !["0", "false", "no", "off"].contains(rawValue) +} + +public func eventStreamScreenshotPolicy(environment: [String: String] = ProcessInfo.processInfo.environment) -> EventStreamScreenshotPolicy { + guard let rawValue = environment["OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + else { + return .auto + } + + return EventStreamScreenshotPolicy(rawValue: rawValue) ?? .auto +} + +public func eventStreamRecordingMaximumDuration(environment: [String: String] = ProcessInfo.processInfo.environment) -> TimeInterval { + guard let rawValue = environment["OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS"]? + .trimmingCharacters(in: .whitespacesAndNewlines), + let value = TimeInterval(rawValue), + value >= 0 + else { + return eventStreamMaximumRecordingDuration + } + + return value +} + +public func eventStreamRawEventsEnabled(environment: [String: String] = ProcessInfo.processInfo.environment) -> Bool { + guard let rawValue = environment["OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + else { + return true + } + + return !["0", "false", "no", "off"].contains(rawValue) +} + +public func eventStreamStartApprovalPolicy(environment: [String: String] = ProcessInfo.processInfo.environment) -> EventStreamStartApprovalPolicy { + guard let rawValue = environment["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL"]? + .trimmingCharacters(in: .whitespacesAndNewlines) + .lowercased() + else { + return .automatic + } + + switch rawValue { + case "auto", "automatic": + return .automatic + case "1", "true", "yes", "on", "interactive", "prompt", "ui": + return .interactive + case "mcp", "elicitation", "mcp-elicitation", "mcp_elicitation": + return .mcpElicitation + case "0", "false", "no", "off", "approve", "approved", "auto-approve", "autoapprove": + return .approve + case "deny", "denied": + return .deny + case "cancel", "cancelled", "canceled": + return .cancel + default: + return .automatic + } +} + +func eventStreamMouseEventType(forButton button: String) -> String { + button == "right" ? "mouse.context_menu" : "mouse.click" +} + +func eventStreamMouseButton(forEventType eventType: NSEvent.EventType) -> String? { + switch eventType { + case .leftMouseDown, .leftMouseDragged, .leftMouseUp: + return "left" + case .rightMouseDown, .rightMouseDragged, .rightMouseUp: + return "right" + case .otherMouseDown, .otherMouseDragged, .otherMouseUp: + return "middle" + default: + return nil + } +} + +func eventStreamPointerDistance(_ start: CGPoint, _ end: CGPoint) -> CGFloat { + hypot(end.x - start.x, end.y - start.y) +} + +func eventStreamPointerDidDrag( + start: CGPoint, + end: CGPoint, + observedDrag: Bool, + threshold: CGFloat = eventStreamMouseDragDistanceThreshold +) -> Bool { + observedDrag || eventStreamPointerDistance(start, end) >= threshold +} + +func eventStreamRawEventSummary( + eventType: NSEvent.EventType, + appKitLocation: CGPoint? = nil, + screenLocation: CGPoint? = nil, + button: String? = nil, + clickCount: Int? = nil, + keyCode: UInt16? = nil, + characters: String? = nil, + modifierFlags: NSEvent.ModifierFlags = [], + scrollingDeltaX: CGFloat? = nil, + scrollingDeltaY: CGFloat? = nil, + hasPreciseScrollingDeltas: Bool? = nil, + phase: NSEvent.Phase = [], + momentumPhase: NSEvent.Phase = [], + redactCharacters: Bool = false +) -> [String: Any] { + var summary: [String: Any] = [ + "eventType": eventStreamRawEventTypeName(eventType), + ] + if let appKitLocation { + summary["appKitLocation"] = [ + "x": appKitLocation.x, + "y": appKitLocation.y, + ] + } + if let screenLocation { + summary["location"] = [ + "x": screenLocation.x, + "y": screenLocation.y, + ] + } + if let button { + summary["button"] = button + } + if let clickCount { + summary["clickCount"] = clickCount + } + if let keyCode { + summary["keyCode"] = Int(keyCode) + } + if let characters, !characters.isEmpty { + if redactCharacters { + summary["charactersRedacted"] = true + summary["characterCount"] = characters.count + } else { + summary["characters"] = characters + } + } + let modifiers = eventStreamModifierNames(modifierFlags) + if !modifiers.isEmpty { + summary["modifiers"] = modifiers + } + if let scrollingDeltaX { + summary["scrollingDeltaX"] = scrollingDeltaX + } + if let scrollingDeltaY { + summary["scrollingDeltaY"] = scrollingDeltaY + } + if let hasPreciseScrollingDeltas { + summary["hasPreciseScrollingDeltas"] = hasPreciseScrollingDeltas + } + if let phaseName = eventStreamRawPhaseName(phase) { + summary["phase"] = phaseName + } + if let momentumPhaseName = eventStreamRawPhaseName(momentumPhase) { + summary["momentumPhase"] = momentumPhaseName + } + return summary +} + +private func eventStreamRawEventSummary( + _ event: NSEvent, + screenPoint: CGPoint? = nil, + button: String? = nil, + redactCharacters: Bool = false +) -> [String: Any] { + let isPointerEvent = eventStreamMouseButton(forEventType: event.type) != nil + let appKitLocation = event.type == .keyDown ? nil : event.locationInWindow + let scrollPhase = event.type == .scrollWheel ? event.phase : [] + let scrollMomentumPhase = event.type == .scrollWheel ? event.momentumPhase : [] + return eventStreamRawEventSummary( + eventType: event.type, + appKitLocation: appKitLocation, + screenLocation: screenPoint, + button: button, + clickCount: isPointerEvent && event.clickCount > 0 ? event.clickCount : nil, + keyCode: event.type == .keyDown ? event.keyCode : nil, + characters: event.type == .keyDown ? event.characters : nil, + modifierFlags: event.modifierFlags, + scrollingDeltaX: event.type == .scrollWheel ? event.scrollingDeltaX : nil, + scrollingDeltaY: event.type == .scrollWheel ? event.scrollingDeltaY : nil, + hasPreciseScrollingDeltas: event.type == .scrollWheel ? event.hasPreciseScrollingDeltas : nil, + phase: scrollPhase, + momentumPhase: scrollMomentumPhase, + redactCharacters: redactCharacters + ) +} + +func eventStreamRawEventTypeName(_ eventType: NSEvent.EventType) -> String { + switch eventType { + case .leftMouseDown: + return "leftMouseDown" + case .leftMouseUp: + return "leftMouseUp" + case .rightMouseDown: + return "rightMouseDown" + case .rightMouseUp: + return "rightMouseUp" + case .otherMouseDown: + return "otherMouseDown" + case .otherMouseUp: + return "otherMouseUp" + case .leftMouseDragged: + return "leftMouseDragged" + case .rightMouseDragged: + return "rightMouseDragged" + case .otherMouseDragged: + return "otherMouseDragged" + case .keyDown: + return "keyDown" + case .scrollWheel: + return "scrollWheel" + default: + return "eventType\(eventType.rawValue)" + } +} + +func eventStreamRawPhaseName(_ phase: NSEvent.Phase) -> String? { + if phase.isEmpty { + return nil + } + + var names: [String] = [] + if phase.contains(.mayBegin) { + names.append("mayBegin") + } + if phase.contains(.began) { + names.append("began") + } + if phase.contains(.stationary) { + names.append("stationary") + } + if phase.contains(.changed) { + names.append("changed") + } + if phase.contains(.ended) { + names.append("ended") + } + if phase.contains(.cancelled) { + names.append("cancelled") + } + return names.isEmpty ? nil : names.joined(separator: ",") +} + +struct EventStreamKeyboardPayloadKind: Equatable { + let type: String + let text: String? + let key: String? +} + +func eventStreamKeyboardPayloadKind( + characters: String?, + keyCode: UInt16, + modifierFlags: NSEvent.ModifierFlags +) -> EventStreamKeyboardPayloadKind? { + let key = eventStreamKeyName(keyCode: keyCode, characters: characters) + let normalizedModifiers = eventStreamModifierNames(modifierFlags) + if key == "Return" || key == "Enter" { + return EventStreamKeyboardPayloadKind(type: "keyboard.submit", text: nil, key: key) + } + if !normalizedModifiers.isEmpty { + return EventStreamKeyboardPayloadKind(type: "keyboard.shortcut", text: nil, key: key) + } + guard let characters, !characters.isEmpty else { + return nil + } + return EventStreamKeyboardPayloadKind(type: "keyboard.text_input", text: characters, key: nil) +} + +func eventStreamModifierNames(_ modifierFlags: NSEvent.ModifierFlags) -> [String] { + var names: [String] = [] + if modifierFlags.contains(.command) { + names.append("command") + } + if modifierFlags.contains(.option) { + names.append("option") + } + if modifierFlags.contains(.control) { + names.append("control") + } + if modifierFlags.contains(.shift) { + names.append("shift") + } + return names +} + +func eventStreamKeyName(keyCode: UInt16, characters: String?) -> String { + switch keyCode { + case 36: + return "Return" + case 76: + return "Enter" + case 48: + return "Tab" + case 51: + return "Backspace" + case 53: + return "Escape" + case 123: + return "Left" + case 124: + return "Right" + case 125: + return "Down" + case 126: + return "Up" + default: + if let characters, !characters.isEmpty { + return characters.uppercased() + } + return "KeyCode\(keyCode)" + } +} + +func eventStreamScreenshotNeededForContext( + policy: EventStreamScreenshotPolicy, + payloadKind _: String, + treeLineCount: Int, + hasFocusedSummary: Bool +) -> Bool { + switch policy { + case .never: + return false + case .always: + return true + case .auto: + return treeLineCount <= 8 || !hasFocusedSummary + } +} + +private func makeEventStreamSessionID(date: Date) -> String { + "session-\(eventStreamCompactTimestamp(date))-\(UUID().uuidString.lowercased())" +} + +private func eventStreamSessionIdentifierIsSafe(_ sessionID: String) -> Bool { + guard sessionID.hasPrefix("session-"), !sessionID.isEmpty else { + return false + } + return sessionID.allSatisfy { character in + let scalars = character.unicodeScalars + guard scalars.count == 1, let scalar = scalars.first else { + return false + } + return (scalar.value >= 97 && scalar.value <= 122) + || (scalar.value >= 65 && scalar.value <= 90) + || (scalar.value >= 48 && scalar.value <= 57) + || scalar.value == 45 + } +} + +private func eventStreamTimestamp(_ date: Date) -> String { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime, .withFractionalSeconds] + return formatter.string(from: date) +} + +private func eventStreamCompactTimestamp(_ date: Date) -> String { + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone(secondsFromGMT: 0) + formatter.dateFormat = "yyyyMMdd'T'HHmmssSSS'Z'" + return formatter.string(from: date) +} + +private func appendJSONObjectLine(_ object: [String: Any], to url: URL) throws { + let data = try JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes]) + guard let line = String(data: data, encoding: .utf8) else { + throw ComputerUseError.message("Failed to encode event stream event.") + } + try appendTextLine(line, to: url) +} + +private func eventStreamCanonicalEventPayload(_ event: [String: Any]) -> [String: Any] { + var payload = event + if payload["kind"] == nil, let type = payload["type"] as? String { + payload["kind"] = type + } + if payload["type"] == nil, let kind = payload["kind"] as? String { + payload["type"] = kind + } + return payload +} + +private func appendTextLine(_ line: String, to url: URL) throws { + let data = Data((line + "\n").utf8) + if !FileManager.default.fileExists(atPath: url.path) { + try data.write(to: url, options: [.atomic]) + return + } + + let handle = try FileHandle(forWritingTo: url) + defer { + try? handle.close() + } + try handle.seekToEnd() + try handle.write(contentsOf: data) +} + +private func writeEmptyFileIfNeeded(_ url: URL) throws { + if !FileManager.default.fileExists(atPath: url.path) { + try Data().write(to: url, options: [.atomic]) + } +} + +@discardableResult +private func performEventStreamOnMainSync(_ body: @MainActor @escaping () -> Result) -> Result { + if Thread.isMainThread { + return MainActor.assumeIsolated { + body() + } + } + + return DispatchQueue.main.sync { + MainActor.assumeIsolated { + body() + } + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamSkillScaffold.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamSkillScaffold.swift new file mode 100644 index 00000000..686c75bf --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamSkillScaffold.swift @@ -0,0 +1,501 @@ +import Foundation + +public struct EventStreamSkillScaffoldOptions: Equatable, Sendable { + public let inputPath: String + public let skillName: String + public let description: String? + public let outputDirectory: String + public let overwrite: Bool + public let includeText: Bool + + public init( + inputPath: String, + skillName: String, + description: String? = nil, + outputDirectory: String, + overwrite: Bool = false, + includeText: Bool = false + ) { + self.inputPath = inputPath + self.skillName = skillName + self.description = description + self.outputDirectory = outputDirectory + self.overwrite = overwrite + self.includeText = includeText + } +} + +private let eventStreamSkillScaffoldPathKeys: Set = [ + "sessionDir", + "metadataPath", + "eventsPath", +] + +public func scaffoldEventStreamSkill(options: EventStreamSkillScaffoldOptions) throws -> [String: Any] { + let skillName = options.skillName.trimmingCharacters(in: .whitespacesAndNewlines) + guard eventStreamSkillNameIsValid(skillName) else { + return [ + "ok": false, + "error": "invalidSkillName", + "message": "skill name must use lowercase letters, numbers, and hyphens", + ] + } + + let validation = validateEventStreamRecording(options: EventStreamRecordingValidationOptions( + inputPath: options.inputPath, + strictOCU: false, + requiredEventTypes: [], + requireSkillDraft: true + )) + guard validation["ok"] as? Bool == true else { + var result: [String: Any] = [ + "ok": false, + "error": "recordingValidationFailed", + ] + if let errors = validation["errors"] { + result["errors"] = errors + } + if let warnings = validation["warnings"] { + result["warnings"] = warnings + } + if let skillDraftReady = validation["skillDraftReady"] { + result["skillDraftReady"] = skillDraftReady + } + if let skillDraftReasons = validation["skillDraftReasons"] { + result["skillDraftReasons"] = skillDraftReasons + } + return result + } + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: options.inputPath, + includeText: options.includeText, + requireAction: true + )) + guard summary["ok"] as? Bool == true else { + var result: [String: Any] = [ + "ok": false, + "error": "recordingSummaryFailed", + ] + if let errors = summary["errors"] { + result["errors"] = errors + } + if let warnings = summary["warnings"] { + result["warnings"] = warnings + } + return result + } + + let outputURL = URL(fileURLWithPath: options.outputDirectory) + let referencesURL = outputURL.appendingPathComponent("references") + let skillURL = outputURL.appendingPathComponent("SKILL.md") + let summaryURL = referencesURL.appendingPathComponent("recording-summary.json") + + var isDirectory: ObjCBool = false + if FileManager.default.fileExists(atPath: outputURL.path, isDirectory: &isDirectory) { + guard options.overwrite else { + return [ + "ok": false, + "error": "outputDirectoryExists", + "message": "output directory already exists", + "outputDir": outputURL.path, + ] + } + try FileManager.default.removeItem(at: outputURL) + } + + let sanitizedSummary = eventStreamSkillSanitizedSummary(summary) + let readiness = sanitizedSummary as? [String: Any] + let readinessObject = readiness?["skillReadiness"] as? [String: Any] + if readinessObject?["canCreateSkillDraft"] as? Bool != true { + return [ + "ok": false, + "error": "insufficientSkillReadiness", + "message": readinessObject?["recommendedNextStep"] as? String + ?? "Recording does not contain enough usable evidence to create a skill draft.", + "skillReadiness": readinessObject ?? [:], + ] + } + + let description = options.description?.trimmingCharacters(in: .whitespacesAndNewlines).nilIfEmpty + ?? "Replay a workflow captured by Open Computer Use Record & Replay for \(skillName)." + + try FileManager.default.createDirectory(at: referencesURL, withIntermediateDirectories: true) + try eventStreamSkillRenderedMarkdown( + skillName: skillName, + description: description, + summary: sanitizedSummary + ).write(to: skillURL, atomically: true, encoding: .utf8) + + let summaryData = try JSONSerialization.data( + withJSONObject: sanitizedSummary, + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + var summaryText = String(data: summaryData, encoding: .utf8) ?? "{}" + summaryText.append("\n") + try summaryText.write(to: summaryURL, atomically: true, encoding: .utf8) + + return [ + "ok": true, + "skillName": skillName, + "outputDir": outputURL.path, + "skillPath": skillURL.path, + "summaryPath": summaryURL.path, + ] +} + +private func eventStreamSkillNameIsValid(_ name: String) -> Bool { + guard (3 ... 64).contains(name.count) else { + return false + } + guard let first = name.first, let last = name.last, first.isLetterOrNumberASCII, last.isLetterOrNumberASCII else { + return false + } + return name.allSatisfy { character in + character.isLowercaseLetterASCII || character.isNumberASCII || character == "-" + } +} + +private func eventStreamSkillSanitizedSummary(_ value: Any) -> Any { + if let dictionary = value as? [String: Any] { + var sanitized: [String: Any] = [:] + for (key, child) in dictionary { + if eventStreamSkillScaffoldPathKeys.contains(key) { + sanitized[key] = "" + } else if key == "screenshotPaths", let paths = child as? [Any] { + sanitized[key] = paths.map { item in + guard let path = item as? String else { + return item + } + return URL(fileURLWithPath: path).lastPathComponent + } + } else { + sanitized[key] = eventStreamSkillSanitizedSummary(child) + } + } + return sanitized + } + if let array = value as? [Any] { + return array.map { eventStreamSkillSanitizedSummary($0) } + } + return value +} + +private func eventStreamSkillRenderedMarkdown( + skillName: String, + description: String, + summary: Any +) -> String { + let summary = summary as? [String: Any] ?? [:] + let windows = summary["windows"] as? [[String: Any]] ?? [] + let windowItems = windows.prefix(10).map { window in + let app = (window["appName"] as? String) ?? (window["bundleIdentifier"] as? String) ?? "Unknown app" + if let title = window["windowTitle"] as? String, !title.isEmpty { + return "\(app) / \(title)" + } + return app + } + + let actions = summary["actionSequence"] as? [[String: Any]] ?? [] + let actionItems = actions.prefix(30).map(eventStreamSkillActionDescription) + let runtimeInputs = summary["runtimeInputs"] as? [[String: Any]] ?? [] + var inputItems = runtimeInputs.prefix(30).map(eventStreamSkillRuntimeInputDescription) + if inputItems.isEmpty { + inputItems = actions.prefix(30).compactMap(eventStreamSkillInputDescription) + } + let warnings = (summary["warnings"] as? [Any] ?? []).map { "\($0)" } + let evidence = summary["skillEvidence"] as? [String: Any] ?? [:] + let readiness = summary["skillReadiness"] as? [String: Any] ?? [:] + let readinessReasons = (readiness["reasons"] as? [Any] ?? []).map { "\($0)" } + let safetySignals = summary["safetySignals"] as? [[String: Any]] ?? [] + let safetyItems = safetySignals.prefix(20).map(eventStreamSkillSafetySignalDescription) + let summaryLimits = summary["summaryLimits"] as? [String: Any] ?? [:] + let summaryLimitItems = eventStreamSkillSummaryLimitDescriptions(summaryLimits) + + return """ + --- + name: \(eventStreamSkillFrontmatterValue(skillName)) + description: \(eventStreamSkillFrontmatterValue(description)) + --- + + # \(eventStreamSkillTitle(from: skillName)) + + ## Purpose + + Replay or adapt the workflow captured with Open Computer Use Record & Replay. This is a generated draft: preserve the observed sequence, but replace user-specific values with explicit inputs before using it on real data. + + ## Inputs To Confirm + + - The user's concrete goal for this replay. + - Required app/account/login state. + - Any names, paths, message text, dates, or account-specific values that were redacted or should be parameterized. + - Whether any step sends, deletes, purchases, uploads, approves, or otherwise changes external state. + + ## Runtime Inputs + + \(eventStreamSkillMarkdownList(inputItems, fallback: "No text or selection inputs were inferred. Still ask the user for any values that should vary between replays.")) + + ## Workflow Readiness + + - Status: \(readiness["status"] ?? "unknown") + - Can create skill draft: \(eventStreamSkillBoolString(readiness["canCreateSkillDraft"])) + - Requires human review: \(eventStreamSkillBoolString(readiness["requiresHumanReview"])) + - Recommended next step: \(readiness["recommendedNextStep"] ?? "Inspect `events.jsonl` before finalizing this skill.") + + \(eventStreamSkillMarkdownList(readinessReasons, fallback: "No readiness issues were detected by the summary helper.")) + + ## Summary Limits + + \(eventStreamSkillMarkdownList(summaryLimitItems, fallback: "No high-volume summary fields were truncated.")) + + ## Recorded Context + + \(eventStreamSkillMarkdownList(windowItems, fallback: "No stable app/window context was captured; inspect `references/recording-summary.json`.")) + + ## Agent Replay Procedure + + - Before replaying UI actions, check whether a connector, API, or dedicated tool can perform the stable semantic operation more reliably than desktop automation; use Computer Use for visually dependent verification, unsupported UI interactions, or when manipulating the interface is itself the workflow. + - Start by calling `get_app_state` for the recorded app or the app the user names; verify the expected window or equivalent screen is present. + - For each observed step, resolve the current target by visible label, role, title, or neighboring text from `references/recording-summary.json`; use `element_index` actions when a semantic element is available. + - Use coordinate clicks or drags only as a last resort after refreshing the screenshot and confirming the coordinate system still matches the current window. + - For text entry, collect the runtime value from the user or task context, focus the observed input target, then use `type_text` or `set_value` according to the current app state. + - Pause and ask for explicit confirmation before sends, deletes, purchases, approvals, uploads, or other externally visible changes. + + ## Replay Steps + + \(eventStreamSkillMarkdownList(actionItems, fallback: "No high-level action sequence was captured; ask the user to re-record after fixing permissions.")) + + ## Evidence + + - Event count: \(summary["eventCount"] ?? "unknown") + - Action event count: \(summary["actionEventCount"] ?? "unknown") + - End reason: \(summary["endReason"] ?? "unknown") + - Has AX context: \(eventStreamSkillBoolString(evidence["hasAXContext"])) + - Has target elements: \(eventStreamSkillBoolString(evidence["hasTargetElements"])) + - Has redaction signals: \(eventStreamSkillBoolString(evidence["hasRedactionSignals"])) + - Has safety signals: \(eventStreamSkillBoolString(evidence["hasSafetySignals"])) + + ## Warnings + + \(eventStreamSkillMarkdownList(warnings, fallback: "No summary warnings were reported.")) + + ## Safety + + - Do not reconstruct passwords, OTPs, API keys, terminal buffers, or other redacted values. + - Ask for confirmation before externally visible or destructive actions. + - Prefer semantic targets, labels, app names, and keyboard shortcuts over raw coordinates. + - Re-check the current app state before executing each step; recorded UI positions may drift. + + ### Confirmation Signals + + \(eventStreamSkillMarkdownList(safetyItems, fallback: "No specific confirmation-sensitive actions were detected by the summary helper. Still ask before externally visible changes.")) + + ## Verification + + - Before packaging this skill, rerun `open-computer-use event-stream validate --json --strict-ocu --require-skill-draft ` on the source recording. + - Compare the final app state with the user's requested outcome, not just the recorded event sequence. + - If replay behavior depends on missing AX targets, screenshots, or redacted text, refine this draft against `events.jsonl` before installing it. + - If `summaryLimits.hasTruncatedSummary` is true, inspect the original `events.jsonl` for omitted steps before finalizing this skill. + + ## Finalizing The Skill + + - Treat this directory as a skill draft, not a standalone runbook or replay plan. + - Read and follow the `skill-creator` skill before packaging or installing the skill. + - Complete the `skill-creator` workflow, including validation, before reporting that this skill was created. + - Keep reusable workflow intent, prerequisites, runtime inputs, safety confirmations, and verification steps in the skill; omit raw event logs and user-specific values. + - The final deliverable should be an actual discoverable skill directory, not only this generated Markdown draft. + + ## Source + + The generated summary is stored in `references/recording-summary.json`. Treat `events.jsonl` from the original recording as the primary evidence when refining this draft. + """ +} + +private func eventStreamSkillRuntimeInputDescription(_ input: [String: Any]) -> String { + let kind = input["kind"].map { "\($0)" } ?? "value" + let target = eventStreamSkillTargetDescription(input["target"] as? [String: Any] ?? [:]) + let suffix: String + if let textLength = input["textLength"] as? Int { + suffix = " (\(textLength) recorded characters; use a fresh runtime value)." + } else if let selectedTextLength = input["selectedTextLength"] as? Int { + suffix = " (\(selectedTextLength) recorded selected characters; confirm current semantics)." + } else { + suffix = "." + } + let sensitivity = input["sensitive"] as? Bool == true ? " Treat this as sensitive input." : "" + + switch kind { + case "text": + return "Runtime text for \(target)\(suffix)\(sensitivity)" + case "selection": + return "Runtime selection or selected-content meaning for \(target)\(suffix)" + default: + return "Runtime \(kind) for \(target)\(suffix)" + } +} + +private func eventStreamSkillSafetySignalDescription(_ signal: [String: Any]) -> String { + let sourceEventType = signal["sourceEventType"].map { "\($0)" } ?? "recorded action" + let reason = signal["reason"].map { "\($0)" } ?? "confirmationRequired" + let target = eventStreamSkillTargetDescription(signal["target"] as? [String: Any] ?? [:]) + let linePrefix: String + if let line = signal["line"] as? Int { + linePrefix = "Line \(line): " + } else { + linePrefix = "" + } + return "\(linePrefix)`\(sourceEventType)` matched \(reason) on \(target); ask for explicit confirmation before replaying this step." +} + +private func eventStreamSkillSummaryLimitDescriptions(_ limits: [String: Any]) -> [String] { + guard limits["hasTruncatedSummary"] as? Bool == true else { + return [] + } + let omittedCounts = limits["omittedCounts"] as? [String: Any] ?? [:] + return omittedCounts + .compactMap { key, value -> (String, Int)? in + if let intValue = value as? Int, intValue > 0 { + return (key, intValue) + } + return nil + } + .sorted { $0.0 < $1.0 } + .map { key, count in + "\(count) \(key) item(s) were omitted from `references/recording-summary.json`; inspect the original `events.jsonl` before finalizing replay steps." + } +} + +private func eventStreamSkillInputDescription(_ action: [String: Any]) -> String? { + guard let actionType = action["type"] as? String else { + return nil + } + let target = eventStreamSkillTargetDescription(action["element"] as? [String: Any] ?? [:]) + + switch actionType { + case "keyboard.text_input": + if let textLength = action["textLength"] as? Int { + return "Runtime text for \(target) (\(textLength) recorded characters; recorded literal text is omitted by default)." + } + return "Runtime text for \(target); recorded literal text is omitted by default." + case "selection.changed": + return "Current selection or selected content semantics if the workflow depends on the recorded selection." + default: + return nil + } +} + +private func eventStreamSkillActionDescription(_ action: [String: Any]) -> String { + let actionType = action["type"] as? String + let prefix: String + if let line = action["line"] as? Int { + prefix = "Line \(line): " + } else { + prefix = "" + } + + let window = action["window"] as? [String: Any] ?? [:] + let target = eventStreamSkillTargetDescription(action["element"] as? [String: Any] ?? [:]) + let app = (window["appName"] as? String) ?? (window["bundleIdentifier"] as? String) ?? "the recorded app" + + switch actionType { + case "mouse.click": + return "\(prefix)Click \(target) in \(app)." + case "mouse.context_menu": + return "\(prefix)Open the context menu on \(target) in \(app)." + case "mouse.drag": + return "\(prefix)Drag from the recorded start location to end location in \(app)." + case "keyboard.text_input": + if let textLength = action["textLength"] as? Int { + return "\(prefix)Enter user-provided text into \(target) (\(textLength) recorded characters)." + } + return "\(prefix)Enter user-provided text into \(target)." + case "keyboard.submit": + return "\(prefix)Submit the current focused control in \(app)." + case "keyboard.shortcut": + let key = action["key"].map { "\($0)" } ?? "" + let modifiers = action["modifiers"] as? [String] ?? [] + let chord = (modifiers + [key]).filter { !$0.isEmpty }.joined(separator: "+") + return "\(prefix)Press shortcut `\(chord)` in \(app)." + case "terminal.value_changed": + return "\(prefix)Observe terminal output/state change; keep command/output values sanitized." + case "selection.changed": + return "\(prefix)Use the recorded selection change as context, but do not rely on stale selected text." + case "experimentalRawEvents": + if action["reason"] as? String == "scrollWheel" { + return "\(prefix)Scroll in \(app) using the recorded wheel direction; re-check the visible state after scrolling." + } + return "\(prefix)Replay raw input evidence in \(app) only after inspecting the original `events.jsonl`." + default: + return "\(prefix)Replay `\(actionType ?? "unknown")` using the recorded context." + } +} + +private func eventStreamSkillTargetDescription(_ element: [String: Any]) -> String { + let targetBits = ["role", "title", "label", "description"].compactMap { key -> String? in + guard let value = element[key] as? String, !value.isEmpty else { + return nil + } + return "\(key)=\(String(reflecting: value))" + } + return targetBits.isEmpty ? "the observed target" : targetBits.joined(separator: ", ") +} + +private func eventStreamSkillMarkdownList(_ items: [String], fallback: String) -> String { + let effectiveItems = items.isEmpty ? [fallback] : items + return effectiveItems.map { "- \($0)" }.joined(separator: "\n") +} + +private func eventStreamSkillFrontmatterValue(_ value: String) -> String { + value.replacingOccurrences(of: "\n", with: " ").trimmingCharacters(in: .whitespacesAndNewlines) +} + +private func eventStreamSkillTitle(from skillName: String) -> String { + skillName + .split(separator: "-") + .map { part in + guard let first = part.first else { + return "" + } + return first.uppercased() + part.dropFirst() + } + .joined(separator: " ") +} + +private func eventStreamSkillBoolString(_ value: Any?) -> String { + if let bool = value as? Bool { + return bool ? "true" : "false" + } + return "false" +} + +private extension Character { + var isLowercaseLetterASCII: Bool { + guard let scalar = unicodeScalars.only else { + return false + } + return scalar.value >= 97 && scalar.value <= 122 + } + + var isNumberASCII: Bool { + guard let scalar = unicodeScalars.only else { + return false + } + return scalar.value >= 48 && scalar.value <= 57 + } + + var isLetterOrNumberASCII: Bool { + isLowercaseLetterASCII || isNumberASCII + } +} + +private extension String { + var nilIfEmpty: String? { + isEmpty ? nil : self + } +} + +private extension Character.UnicodeScalarView { + var only: UnicodeScalar? { + count == 1 ? first : nil + } +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamToolDefinitions.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamToolDefinitions.swift new file mode 100644 index 00000000..e1a1a421 --- /dev/null +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/EventStreamToolDefinitions.swift @@ -0,0 +1,41 @@ +import Foundation + +public enum EventStreamToolDefinitions { + public static let all: [ToolDefinition] = [ + ToolDefinition( + name: "event_stream_start", + description: "Start recording the user's actions for up to 30 minutes. If a recording is already active, return that active session instead of starting another one.", + annotations: eventStreamAnnotations(readOnly: false, idempotent: false), + inputSchema: eventStreamEmptyObjectSchema() + ), + ToolDefinition( + name: "event_stream_status", + description: "Get the current or most recent Record & Replay recording status including paths to metadata and events during the recording.", + annotations: eventStreamAnnotations(readOnly: true, idempotent: true), + inputSchema: eventStreamEmptyObjectSchema() + ), + ToolDefinition( + name: "event_stream_stop", + description: "Stop the active event stream recording if one is running and return status including paths to metadata and events during the recording.", + annotations: eventStreamAnnotations(readOnly: false, idempotent: true), + inputSchema: eventStreamEmptyObjectSchema() + ), + ] +} + +private func eventStreamEmptyObjectSchema() -> [String: Any] { + [ + "type": "object", + "properties": [:], + "additionalProperties": false, + ] +} + +private func eventStreamAnnotations(readOnly: Bool, idempotent: Bool) -> [String: Any] { + [ + "destructiveHint": false, + "idempotentHint": idempotent, + "openWorldHint": false, + "readOnlyHint": readOnly, + ] +} diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift index 575bc71f..f28178b2 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/MCPServer.swift @@ -1,5 +1,10 @@ import Foundation +public protocol LineBasedMCPServer: AnyObject { + func run() throws + func handle(line: String) -> String? +} + let computerUseServerInstructions = """ Computer Use tools let you interact with macOS apps by performing UI actions. @@ -17,7 +22,7 @@ Avoid falling back to AppleScript during a computer use session. Prefer Computer Ask the user before taking destructive or externally visible actions such as sending, deleting, or purchasing. If helpful, you can ask follow-up questions before taking action to make sure you’re understanding the user’s request correctly. """ -public final class StdioMCPServer { +public final class StdioMCPServer: LineBasedMCPServer { private let dispatcher: ComputerUseToolDispatcher public init(service: ComputerUseService = ComputerUseService()) { diff --git a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift index 24d90da3..f21c4dc6 100644 --- a/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift +++ b/packages/OpenComputerUseKit/Sources/OpenComputerUseKit/OpenComputerUseCLI.swift @@ -3,6 +3,7 @@ import Foundation public enum OpenComputerUseCLICommand: Equatable { case launchOnboarding case mcp + case eventStream(EventStreamCLIInvocation) case doctor case listApps case snapshot(app: String, showFullText: Bool = false) @@ -17,6 +18,18 @@ public enum OpenComputerUseCallInvocation: Equatable { case sequence(callsJSON: String?, callsFile: String?, interCallDelay: TimeInterval) } +public enum EventStreamCLIInvocation: Equatable { + case mcp + case start(json: Bool) + case status(json: Bool) + case stop(json: Bool) + case cancel(json: Bool) + case wait(json: Bool, sessionID: String?, timeout: TimeInterval?, notifyCommand: [String]?) + case summarize(inputPath: String, includeText: Bool, requireAction: Bool) + case validate(inputPath: String, strictOCU: Bool, requiredEventTypes: [String], requireSkillDraft: Bool) + case scaffoldSkill(inputPath: String, skillName: String, description: String?, outputDirectory: String, overwrite: Bool, includeText: Bool) +} + public let openComputerUseDefaultInterCallDelay: TimeInterval = 1 public func shouldUseMacOSAppAgentProxy( @@ -32,7 +45,9 @@ public func shouldUseMacOSAppAgentProxy( switch command { case .launchOnboarding: return !runningFromLaunchServicesAppInstance - case .mcp, .doctor, .listApps, .snapshot, .call: + case .eventStream(.summarize), .eventStream(.validate), .eventStream(.scaffoldSkill): + return false + case .mcp, .eventStream, .doctor, .listApps, .snapshot, .call: return true case .turnEnded, .help, .version: return false @@ -76,6 +91,8 @@ public func parseOpenComputerUseCLI(arguments: [String]) throws -> OpenComputerU return .version case "mcp": return try parseSimpleCommand(name: "mcp", arguments: Array(arguments.dropFirst()), result: .mcp) + case "event-stream": + return try parseEventStream(arguments: Array(arguments.dropFirst())) case "doctor": return try parseSimpleCommand(name: "doctor", arguments: Array(arguments.dropFirst()), result: .doctor) case "list-apps": @@ -107,6 +124,7 @@ public func openComputerUseHelpText(command: String? = nil) -> String { Commands: mcp Start the stdio MCP server. + event-stream Record macOS actions into a Record & Replay event stream. doctor Print permission status and launch onboarding if needed. list-apps Print running or recently used apps. snapshot Print the current accessibility snapshot for an app. @@ -130,6 +148,22 @@ public func openComputerUseHelpText(command: String? = nil) -> String { Start the stdio MCP server. """ + case "event-stream": + return """ + Usage: + open-computer-use event-stream mcp + open-computer-use event-stream start [--json] + open-computer-use event-stream status [--json] + open-computer-use event-stream stop [--json] + open-computer-use event-stream cancel [--json] + open-computer-use event-stream wait [--json] [--session-id ] [--timeout ] [--notify-command ''] + open-computer-use event-stream summarize [--json] [--include-text] [--require-action] + open-computer-use event-stream validate [--json] [--strict-ocu] [--require-event-type ] [--require-skill-draft] + open-computer-use event-stream scaffold-skill [--json] --skill-name --output-dir [--description ] [--overwrite] [--include-text] + + Start the Record & Replay-compatible event stream MCP server, or control + a recording through the long-lived Open Computer Use app agent. + """ case "doctor": return """ Usage: @@ -210,6 +244,225 @@ public func openComputerUseHelpText(command: String? = nil) -> String { } } +public func runOpenComputerUseEventStream( + _ invocation: EventStreamCLIInvocation, + service: EventStreamService = .shared +) throws -> OpenComputerUseCallOutput { + let status: EventStreamRecordingStatus + var extraFields: [String: Any] = [:] + switch invocation { + case .mcp: + throw OpenComputerUseCLIError(message: "event-stream mcp is a server command", helpCommand: "event-stream") + case .start: + status = try service.start() + case .status: + status = service.status() + case .stop: + status = try service.stop() + case .cancel: + status = try service.cancel() + case let .wait(_, sessionID, timeout, notifyCommand): + let result = service.waitResult(sessionID: sessionID, timeout: timeout) + status = result.status + extraFields["waitTimedOut"] = result.timedOut + extraFields["waitSessionMatched"] = result.sessionMatched + if let notifyCommand { + let statusDictionary = status.asDictionary.merging(extraFields) { _, new in new } + let notification = runEventStreamWaitNotification( + command: notifyCommand, + statusDictionary: statusDictionary, + skipped: result.timedOut + ) + extraFields["notification"] = notification + return OpenComputerUseCallOutput( + jsonObject: status.asDictionary.merging(extraFields) { _, new in new }, + hasToolError: (notification["ok"] as? Bool) == false + ) + } + case let .summarize(inputPath, includeText, requireAction): + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: inputPath, + includeText: includeText, + requireAction: requireAction + )) + return OpenComputerUseCallOutput( + jsonObject: summary, + hasToolError: (summary["ok"] as? Bool) == false + ) + case let .validate(inputPath, strictOCU, requiredEventTypes, requireSkillDraft): + let validation = validateEventStreamRecording(options: EventStreamRecordingValidationOptions( + inputPath: inputPath, + strictOCU: strictOCU, + requiredEventTypes: requiredEventTypes, + requireSkillDraft: requireSkillDraft + )) + return OpenComputerUseCallOutput( + jsonObject: validation, + hasToolError: (validation["ok"] as? Bool) == false + ) + case let .scaffoldSkill(inputPath, skillName, description, outputDirectory, overwrite, includeText): + let result = try scaffoldEventStreamSkill(options: EventStreamSkillScaffoldOptions( + inputPath: inputPath, + skillName: skillName, + description: description, + outputDirectory: outputDirectory, + overwrite: overwrite, + includeText: includeText + )) + return OpenComputerUseCallOutput( + jsonObject: result, + hasToolError: (result["ok"] as? Bool) == false + ) + } + + var dictionary = status.asDictionary + for (key, value) in extraFields { + dictionary[key] = value + } + return OpenComputerUseCallOutput(jsonObject: dictionary, hasToolError: false) +} + +private func runEventStreamWaitNotification( + command: [String], + statusDictionary: [String: Any], + skipped: Bool +) -> [String: Any] { + var notification: [String: Any] = [ + "attempted": false, + "skipped": skipped, + "ok": true, + "command": command, + ] + + if skipped { + notification["reason"] = "waitTimedOut" + return notification + } + + guard let executable = command.first, !executable.isEmpty else { + notification["ok"] = false + notification["reason"] = "emptyCommand" + return notification + } + + let statusData: Data + do { + statusData = try JSONSerialization.data( + withJSONObject: statusDictionary, + options: [.sortedKeys, .withoutEscapingSlashes] + ) + } catch { + notification["ok"] = false + notification["reason"] = "statusSerializationFailed" + notification["error"] = String(describing: error) + return notification + } + let statusText = String(data: statusData, encoding: .utf8) ?? "{}" + let timeoutSeconds = eventStreamWaitNotificationTimeoutSeconds() + + let process = Process() + process.executableURL = URL(fileURLWithPath: executable) + process.arguments = Array(command.dropFirst()) + var environment = ProcessInfo.processInfo.environment + environment["OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON"] = statusText + if let sessionID = statusDictionary["sessionId"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID"] = sessionID + } + if let state = statusDictionary["state"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_STATE"] = state + } + if let endReason = statusDictionary["endReason"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON"] = endReason + } + if let metadataPath = statusDictionary["metadataPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH"] = metadataPath + } else if let metadataPath = statusDictionary["currentSegmentMetadataPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH"] = metadataPath + } + if let sessionPath = statusDictionary["sessionPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH"] = sessionPath + } + if let eventsPath = statusDictionary["eventsPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH"] = eventsPath + } else if let eventsPath = statusDictionary["currentSegmentEventsPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH"] = eventsPath + } + if let suppressedEventsPath = statusDictionary["suppressedEventsPath"] as? String { + environment["OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH"] = suppressedEventsPath + } + process.environment = environment + + let inputPipe = Pipe() + process.standardInput = inputPipe + let stdoutNull = FileHandle(forWritingAtPath: "/dev/null") + let stderrNull = FileHandle(forWritingAtPath: "/dev/null") + process.standardOutput = stdoutNull ?? Pipe() + process.standardError = stderrNull ?? Pipe() + + notification["attempted"] = true + notification["skipped"] = false + notification["timeoutSeconds"] = timeoutSeconds + + var didLaunch = false + do { + try process.run() + didLaunch = true + var inputData = statusData + inputData.append(Data("\n".utf8)) + try inputPipe.fileHandleForWriting.write(contentsOf: inputData) + try inputPipe.fileHandleForWriting.close() + } catch { + if process.isRunning { + process.terminate() + } + try? stdoutNull?.close() + try? stderrNull?.close() + notification["ok"] = false + notification["reason"] = didLaunch ? "stdinWriteFailed" : "launchFailed" + notification["error"] = String(describing: error) + return notification + } + + let semaphore = DispatchSemaphore(value: 0) + DispatchQueue.global(qos: .utility).async { + process.waitUntilExit() + semaphore.signal() + } + + if semaphore.wait(timeout: .now() + timeoutSeconds) == .timedOut { + if process.isRunning { + process.terminate() + } + notification["ok"] = false + notification["timedOut"] = true + notification["reason"] = "timeout" + } else { + let exitCode = Int(process.terminationStatus) + notification["exitCode"] = exitCode + notification["timedOut"] = false + notification["ok"] = exitCode == 0 + if exitCode != 0 { + notification["reason"] = "nonZeroExit" + } + } + + try? stdoutNull?.close() + try? stderrNull?.close() + return notification +} + +private func eventStreamWaitNotificationTimeoutSeconds() -> TimeInterval { + let fallback: TimeInterval = 10 + guard + let value = ProcessInfo.processInfo.environment["OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS"], + let seconds = TimeInterval(value), + seconds > 0 + else { + return fallback + } + return seconds +} + private func parseSimpleCommand( name: String, arguments: [String], @@ -226,6 +479,306 @@ private func parseSimpleCommand( throw OpenComputerUseCLIError(message: "\(name) does not accept any arguments", helpCommand: name) } +private func parseEventStream(arguments: [String]) throws -> OpenComputerUseCLICommand { + guard let subcommand = arguments.first else { + throw OpenComputerUseCLIError(message: "event-stream requires a subcommand", helpCommand: "event-stream") + } + + if subcommand == "-h" || subcommand == "--help" { + return .help(command: "event-stream") + } + + let rest = Array(arguments.dropFirst()) + switch subcommand { + case "mcp": + return try parseSimpleCommand(name: "event-stream mcp", arguments: rest, result: .eventStream(.mcp)) + case "start": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(.start(json: try parseEventStreamJSONOnlyOptions(subcommand: "start", arguments: rest))) + case "status": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(.status(json: try parseEventStreamJSONOnlyOptions(subcommand: "status", arguments: rest))) + case "stop": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(.stop(json: try parseEventStreamJSONOnlyOptions(subcommand: "stop", arguments: rest))) + case "cancel": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(.cancel(json: try parseEventStreamJSONOnlyOptions(subcommand: "cancel", arguments: rest))) + case "wait": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(try parseEventStreamWait(arguments: rest)) + case "summarize": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(try parseEventStreamSummarize(arguments: rest)) + case "validate": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(try parseEventStreamValidate(arguments: rest)) + case "scaffold-skill": + if isHelpOnly(rest) { + return .help(command: "event-stream") + } + return .eventStream(try parseEventStreamScaffoldSkill(arguments: rest)) + default: + if subcommand.hasPrefix("-") { + throw OpenComputerUseCLIError(message: "Unknown event-stream option: \(subcommand)", helpCommand: "event-stream") + } + + throw OpenComputerUseCLIError(message: "Unknown event-stream subcommand: \(subcommand)", helpCommand: "event-stream") + } +} + +private func parseEventStreamValidate(arguments: [String]) throws -> EventStreamCLIInvocation { + var inputPath: String? + var strictOCU = false + var requiredEventTypes: [String] = [] + var requireSkillDraft = false + + var index = 0 + while index < arguments.count { + let argument = arguments[index] + + switch argument { + case "--json": + break + case "--strict-ocu": + strictOCU = true + case "--require-skill-draft": + requireSkillDraft = true + case "--require-event-type": + requiredEventTypes.append(try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream")) + case "--metadata-path", "--session-path", "--events-path": + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream validate accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "-h", "--help": + throw OpenComputerUseCLIError(message: "event-stream validate help must be requested as `open-computer-use event-stream --help`", helpCommand: "event-stream") + default: + if argument.hasPrefix("-") { + throw OpenComputerUseCLIError(message: "Unknown event-stream validate option: \(argument)", helpCommand: "event-stream") + } + + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream validate accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = argument + } + + index += 1 + } + + guard let inputPath else { + throw OpenComputerUseCLIError(message: "event-stream validate requires a session directory, metadata.json, session.json, or events.jsonl", helpCommand: "event-stream") + } + + return .validate( + inputPath: inputPath, + strictOCU: strictOCU, + requiredEventTypes: requiredEventTypes, + requireSkillDraft: requireSkillDraft + ) +} + +private func parseEventStreamSummarize(arguments: [String]) throws -> EventStreamCLIInvocation { + var inputPath: String? + var includeText = false + var requireAction = false + + var index = 0 + while index < arguments.count { + let argument = arguments[index] + + switch argument { + case "--json": + break + case "--include-text": + includeText = true + case "--require-action": + requireAction = true + case "--metadata-path", "--session-path", "--events-path": + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream summarize accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "-h", "--help": + throw OpenComputerUseCLIError(message: "event-stream summarize help must be requested as `open-computer-use event-stream --help`", helpCommand: "event-stream") + default: + if argument.hasPrefix("-") { + throw OpenComputerUseCLIError(message: "Unknown event-stream summarize option: \(argument)", helpCommand: "event-stream") + } + + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream summarize accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = argument + } + + index += 1 + } + + guard let inputPath else { + throw OpenComputerUseCLIError(message: "event-stream summarize requires a session directory, metadata.json, session.json, or events.jsonl", helpCommand: "event-stream") + } + + return .summarize(inputPath: inputPath, includeText: includeText, requireAction: requireAction) +} + +private func parseEventStreamScaffoldSkill(arguments: [String]) throws -> EventStreamCLIInvocation { + var inputPath: String? + var skillName: String? + var description: String? + var outputDirectory: String? + var overwrite = false + var includeText = false + + var index = 0 + while index < arguments.count { + let argument = arguments[index] + + switch argument { + case "--json": + break + case "--skill-name": + skillName = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "--description": + description = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "--output-dir": + outputDirectory = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "--overwrite": + overwrite = true + case "--include-text": + includeText = true + case "--metadata-path", "--session-path", "--events-path": + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = try parseOptionValue(argument, arguments: arguments, index: &index, helpCommand: "event-stream") + case "-h", "--help": + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill help must be requested as `open-computer-use event-stream --help`", helpCommand: "event-stream") + default: + if argument.hasPrefix("-") { + throw OpenComputerUseCLIError(message: "Unknown event-stream scaffold-skill option: \(argument)", helpCommand: "event-stream") + } + + guard inputPath == nil else { + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill accepts exactly one input path", helpCommand: "event-stream") + } + inputPath = argument + } + + index += 1 + } + + guard let inputPath else { + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill requires a session directory, metadata.json, session.json, or events.jsonl", helpCommand: "event-stream") + } + guard let skillName, !skillName.isEmpty else { + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill requires --skill-name", helpCommand: "event-stream") + } + guard let outputDirectory, !outputDirectory.isEmpty else { + throw OpenComputerUseCLIError(message: "event-stream scaffold-skill requires --output-dir", helpCommand: "event-stream") + } + + return .scaffoldSkill( + inputPath: inputPath, + skillName: skillName, + description: description, + outputDirectory: outputDirectory, + overwrite: overwrite, + includeText: includeText + ) +} + +private func parseEventStreamJSONOnlyOptions(subcommand: String, arguments: [String]) throws -> Bool { + if arguments.isEmpty { + return false + } + + if arguments.count == 1, arguments[0] == "--json" { + return true + } + + let invalid = arguments.first ?? "" + throw OpenComputerUseCLIError( + message: "Unknown event-stream \(subcommand) option: \(invalid)", + helpCommand: "event-stream" + ) +} + +private func isHelpOnly(_ arguments: [String]) -> Bool { + arguments.count == 1 && (arguments[0] == "-h" || arguments[0] == "--help") +} + +private func parseEventStreamWait(arguments: [String]) throws -> EventStreamCLIInvocation { + var json = false + var sessionID: String? + var timeout: TimeInterval? + var notifyCommand: [String]? + + var index = 0 + while index < arguments.count { + let argument = arguments[index] + + switch argument { + case "--json": + json = true + case "--session-id": + sessionID = try parseOptionValue("--session-id", arguments: arguments, index: &index, helpCommand: "event-stream") + case "--timeout": + timeout = try parseTimeIntervalOptionValue("--timeout", arguments: arguments, index: &index, helpCommand: "event-stream") + case "--notify-command": + let value = try parseOptionValue("--notify-command", arguments: arguments, index: &index, helpCommand: "event-stream") + notifyCommand = try parseEventStreamNotifyCommand(value) + case "-h", "--help": + throw OpenComputerUseCLIError(message: "event-stream wait help must be requested as `open-computer-use event-stream --help`", helpCommand: "event-stream") + default: + if argument.hasPrefix("-") { + throw OpenComputerUseCLIError(message: "Unknown event-stream wait option: \(argument)", helpCommand: "event-stream") + } + + throw OpenComputerUseCLIError(message: "event-stream wait does not accept positional arguments", helpCommand: "event-stream") + } + + index += 1 + } + + return .wait(json: json, sessionID: sessionID, timeout: timeout, notifyCommand: notifyCommand) +} + +private func parseEventStreamNotifyCommand(_ value: String) throws -> [String] { + guard let data = value.data(using: .utf8) else { + throw OpenComputerUseCLIError(message: "--notify-command must be a JSON array of strings", helpCommand: "event-stream") + } + let object: Any + do { + object = try JSONSerialization.jsonObject(with: data) + } catch { + throw OpenComputerUseCLIError(message: "--notify-command must be a JSON array of strings", helpCommand: "event-stream") + } + guard let array = object as? [Any], !array.isEmpty else { + throw OpenComputerUseCLIError(message: "--notify-command must be a non-empty JSON array of strings", helpCommand: "event-stream") + } + let strings = array.compactMap { $0 as? String } + guard strings.count == array.count, strings.allSatisfy({ !$0.isEmpty }) else { + throw OpenComputerUseCLIError(message: "--notify-command must be a non-empty JSON array of non-empty strings", helpCommand: "event-stream") + } + return strings +} + private func parseTurnEnded(arguments: [String]) throws -> OpenComputerUseCLICommand { if arguments.count == 1, let option = arguments.first, option == "-h" || option == "--help" { return .help(command: "turn-ended") @@ -386,11 +939,12 @@ private func parseCall(arguments: [String]) throws -> OpenComputerUseCLICommand private func parseOptionValue( _ option: String, arguments: [String], - index: inout Int + index: inout Int, + helpCommand: String = "call" ) throws -> String { let valueIndex = index + 1 guard valueIndex < arguments.count else { - throw OpenComputerUseCLIError(message: "\(option) requires a value", helpCommand: "call") + throw OpenComputerUseCLIError(message: "\(option) requires a value", helpCommand: helpCommand) } index = valueIndex @@ -400,13 +954,14 @@ private func parseOptionValue( private func parseTimeIntervalOptionValue( _ option: String, arguments: [String], - index: inout Int + index: inout Int, + helpCommand: String = "call" ) throws -> TimeInterval { - let rawValue = try parseOptionValue(option, arguments: arguments, index: &index) + let rawValue = try parseOptionValue(option, arguments: arguments, index: &index, helpCommand: helpCommand) guard let value = Double(rawValue), value.isFinite, value >= 0 else { throw OpenComputerUseCLIError( message: "\(option) requires a non-negative number of seconds", - helpCommand: "call" + helpCommand: helpCommand ) } diff --git a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift index 4691fc9b..1193fef9 100644 --- a/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift +++ b/packages/OpenComputerUseKit/Tests/OpenComputerUseKitTests/OpenComputerUseKitTests.swift @@ -3,6 +3,32 @@ import ImageIO import XCTest @testable import OpenComputerUseKit +private final class EventStreamStatusBox: @unchecked Sendable { + var value: EventStreamRecordingStatus? +} + +private final class JSONDictionaryBox: @unchecked Sendable { + var value: [String: Any]? +} + +private final class EventStreamInputHandlerBox: @unchecked Sendable { + private let lock = NSLock() + private var values: [(NSEvent.EventTypeMask, (NSEvent) -> Void)] = [] + + func append(_ value: (NSEvent.EventTypeMask, (NSEvent) -> Void)) { + lock.lock() + values.append(value) + lock.unlock() + } + + func snapshot() -> [(NSEvent.EventTypeMask, (NSEvent) -> Void)] { + lock.lock() + let snapshot = values + lock.unlock() + return snapshot + } +} + final class OpenComputerUseKitTests: XCTestCase { func testCLIRecognizesGlobalHelpAndVersionFlags() throws { XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["-h"]), .help(command: nil)) @@ -16,6 +42,7 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["snapshot", "--help"]), .help(command: "snapshot")) XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["doctor", "-h"]), .help(command: "doctor")) XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["call", "--help"]), .help(command: "call")) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "--help"]), .help(command: "event-stream")) } func testCLIRecognizesSingleToolCallCommand() throws { @@ -63,6 +90,73 @@ final class OpenComputerUseKitTests: XCTestCase { ) } + func testCLIRecognizesEventStreamCommands() throws { + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "mcp"]), .eventStream(.mcp)) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "start"]), .eventStream(.start(json: false))) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "start", "--json"]), .eventStream(.start(json: true))) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "status", "--json"]), .eventStream(.status(json: true))) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "stop", "--json"]), .eventStream(.stop(json: true))) + XCTAssertEqual(try parseOpenComputerUseCLI(arguments: ["event-stream", "cancel", "--json"]), .eventStream(.cancel(json: true))) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "wait", "--json", "--session-id", "session-123", "--timeout", "0.5"]), + .eventStream(.wait(json: true, sessionID: "session-123", timeout: 0.5, notifyCommand: nil)) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "wait", "--notify-command", #"["/bin/echo","done"]"#]), + .eventStream(.wait(json: false, sessionID: nil, timeout: nil, notifyCommand: ["/bin/echo", "done"])) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "summarize", "--json", "--metadata-path", "/tmp/session/metadata.json"]), + .eventStream(.summarize(inputPath: "/tmp/session/metadata.json", includeText: false, requireAction: false)) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "summarize", "--include-text", "--require-action", "/tmp/session"]), + .eventStream(.summarize(inputPath: "/tmp/session", includeText: true, requireAction: true)) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "validate", "--json", "--strict-ocu", "--require-event-type", "mouse.click", "--require-skill-draft", "--metadata-path", "/tmp/session/metadata.json"]), + .eventStream(.validate( + inputPath: "/tmp/session/metadata.json", + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: ["event-stream", "validate", "--json", "--require-skill-draft", "--events-path", "/tmp/session/events.jsonl"]), + .eventStream(.validate( + inputPath: "/tmp/session/events.jsonl", + strictOCU: false, + requiredEventTypes: [], + requireSkillDraft: true + )) + ) + XCTAssertEqual( + try parseOpenComputerUseCLI(arguments: [ + "event-stream", + "scaffold-skill", + "--json", + "--skill-name", + "recorded-example-workflow", + "--description", + "Replay an example workflow.", + "--output-dir", + "/tmp/generated-skill", + "--overwrite", + "--metadata-path", + "/tmp/session/metadata.json", + ]), + .eventStream(.scaffoldSkill( + inputPath: "/tmp/session/metadata.json", + skillName: "recorded-example-workflow", + description: "Replay an example workflow.", + outputDirectory: "/tmp/generated-skill", + overwrite: true, + includeText: false + )) + ) + } + func testCLIRequiresSnapshotArgument() { XCTAssertThrowsError(try parseOpenComputerUseCLI(arguments: ["snapshot"])) { error in XCTAssertEqual( @@ -179,6 +273,2920 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertEqual(ToolDefinitions.all.count, 9) } + func testEventStreamToolDefinitionsMatchOfficialSurface() { + XCTAssertEqual(EventStreamToolDefinitions.all.map(\.name), [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ]) + + let status = EventStreamToolDefinitions.all[1].asDictionary + let annotations = status["annotations"] as? [String: Any] + XCTAssertEqual(annotations?["readOnlyHint"] as? Bool, true) + XCTAssertEqual(annotations?["idempotentHint"] as? Bool, true) + } + + func testEventStreamServiceWritesSessionFilesAndStops() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + + let started = try service.start() + XCTAssertTrue(started.isActive) + XCTAssertEqual(started.state, .recording) + XCTAssertNotNil(started.eventsPath) + XCTAssertNotNil(started.metadataPath) + XCTAssertNotNil(started.suppressedEventsPath) + XCTAssertEqual(started.eventCount, 3) + + let repeatedStart = try service.start() + XCTAssertEqual(repeatedStart.sessionID, started.sessionID) + + try service.appendSyntheticEventForTesting([ + "type": "mouse.click", + "timestamp": "2026-06-26T00:00:00.000Z", + "button": "left", + ]) + + let stopped = try service.stop() + XCTAssertFalse(stopped.isActive) + XCTAssertEqual(stopped.state, .stopped) + XCTAssertEqual(stopped.endReason, "recording_controls_stopped") + XCTAssertEqual(stopped.eventCount, 5) + + let eventsPath = try XCTUnwrap(stopped.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""type":"session.started""#)) + XCTAssertTrue(events.contains(#""type":"window.changed""#)) + XCTAssertTrue(events.contains(#""type":"AX.focusedWindowChanged""#)) + XCTAssertTrue(events.contains(#""type":"mouse.click""#)) + XCTAssertTrue(events.contains(#""type":"session.ended""#)) + + let metadataPath = try XCTUnwrap(stopped.metadataPath) + let metadata = try String(contentsOfFile: metadataPath, encoding: .utf8) + XCTAssertTrue(metadata.contains(#""state" : "stopped""#)) + let sessionAliasPath = URL(fileURLWithPath: metadataPath) + .deletingLastPathComponent() + .appendingPathComponent("session.json") + .path + let sessionAlias = try String(contentsOfFile: sessionAliasPath, encoding: .utf8) + XCTAssertNotEqual(sessionAlias, metadata) + let sessionAliasJSON = try jsonObject(contentsOf: URL(fileURLWithPath: sessionAliasPath)) + XCTAssertEqual(sessionAliasJSON["id"] as? String, stopped.sessionID) + XCTAssertEqual(sessionAliasJSON["eventsPath"] as? String, stopped.eventsPath) + XCTAssertEqual(sessionAliasJSON["endReason"] as? String, "recording_controls_stopped") + XCTAssertNil(sessionAliasJSON["state"]) + XCTAssertNil(sessionAliasJSON["eventCount"]) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + } + + func testEventStreamServiceWritesAXFullDiffAndCumulativePayloads() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + screenshotPolicy: .never + ) + + _ = try service.start() + try service.appendFocusedWindowChangedForTesting( + reason: "test-initial-ax-payload", + appName: "Replay Fixture", + bundleIdentifier: "dev.opencomputeruse.replay-fixture", + snapshot: makeSnapshot( + treeLines: [ + "\t0 standard window \"Replay Fixture\"", + "\t1 static text \"Draft\"", + ], + focusedSummary: "1 static text Draft" + ) + ) + try service.appendFocusedWindowChangedForTesting( + reason: "test-updated-ax-payload", + appName: "Replay Fixture", + bundleIdentifier: "dev.opencomputeruse.replay-fixture", + snapshot: makeSnapshot( + treeLines: [ + "\t0 standard window \"Replay Fixture\"", + "\t1 static text \"Published\"", + "\t2 button \"Done\"", + ], + focusedSummary: "2 button Done" + ) + ) + + let stopped = try service.stop() + let eventsPath = try XCTUnwrap(stopped.eventsPath) + let events = try jsonObjects(contentsOf: URL(fileURLWithPath: eventsPath)) + let initialEvent = try XCTUnwrap(events.first { + $0["type"] as? String == "AX.focusedWindowChanged" + && $0["reason"] as? String == "test-initial-ax-payload" + }) + let updatedEvent = try XCTUnwrap(events.first { + $0["type"] as? String == "AX.focusedWindowChanged" + && $0["reason"] as? String == "test-updated-ax-payload" + }) + + let initialPayload = try XCTUnwrap(initialEvent["accessibilityInspectorPayload"] as? [String: Any]) + XCTAssertEqual(initialPayload["kind"] as? String, "full") + XCTAssertEqual(initialPayload["diffFromPrevious"] as? Bool, false) + XCTAssertEqual(initialPayload["cumulativeDiffFromInitial"] as? Bool, false) + XCTAssertEqual(initialPayload["fullTree"] as? [String], [ + "\t0 standard window \"Replay Fixture\"", + "\t1 static text \"Draft\"", + ]) + + let updatedPayload = try XCTUnwrap(updatedEvent["accessibilityInspectorPayload"] as? [String: Any]) + XCTAssertEqual(updatedPayload["kind"] as? String, "diff") + XCTAssertEqual(updatedPayload["diffFromPrevious"] as? Bool, true) + XCTAssertEqual(updatedPayload["cumulativeDiffFromInitial"] as? Bool, true) + XCTAssertNil(updatedPayload["fullTree"]) + let treeLines = try XCTUnwrap(updatedPayload["treeLines"] as? [String]) + XCTAssertTrue(treeLines.contains { $0.trimmingCharacters(in: .whitespaces).hasPrefix("~") }) + XCTAssertTrue(treeLines.contains { $0.trimmingCharacters(in: .whitespaces).hasPrefix("+") }) + let cumulativeTreeLines = try XCTUnwrap(updatedPayload["cumulativeTreeLines"] as? [String]) + XCTAssertEqual(cumulativeTreeLines, treeLines) + XCTAssertEqual(updatedEvent["diffFromPrevious"] as? Bool, true) + } + + func testEventStreamServicePersistsAXScreenshotContext() throws { + let root = try makeTemporaryDirectory() + let image = try makeSolidTestImage(width: 24, height: 16) + let screenshotData = try XCTUnwrap(boundedScreenshotPNGData( + for: image, + maxBytes: 1_000_000, + maxDimension: 320 + )) + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + screenshotPolicy: .always + ) + + _ = try service.start() + try service.appendFocusedWindowChangedForTesting( + reason: "test-screenshot-context", + appName: "Replay Fixture", + bundleIdentifier: "dev.opencomputeruse.replay-fixture", + snapshot: makeSnapshot( + treeLines: [ + "\t0 standard window \"Replay Fixture\"", + "\t1 button \"Done\"", + ], + focusedSummary: "1 button Done", + screenshotPNGData: screenshotData + ) + ) + + let stopped = try service.stop() + let eventsPath = try XCTUnwrap(stopped.eventsPath) + let metadataPath = try XCTUnwrap(stopped.metadataPath) + let sessionDirectory = URL(fileURLWithPath: metadataPath).deletingLastPathComponent() + let events = try jsonObjects(contentsOf: URL(fileURLWithPath: eventsPath)) + let screenshotEvent = try XCTUnwrap(events.first { + $0["type"] as? String == "AX.focusedWindowChanged" + && $0["reason"] as? String == "test-screenshot-context" + }) + let payload = try XCTUnwrap(screenshotEvent["accessibilityInspectorPayload"] as? [String: Any]) + + XCTAssertEqual(payload["screenshotNeededForContext"] as? Bool, true) + XCTAssertEqual(payload["screenshotAvailable"] as? Bool, true) + let screenshotPath = try XCTUnwrap(payload["screenshotPath"] as? String) + let screenshotURL = URL(fileURLWithPath: screenshotPath) + XCTAssertEqual(screenshotURL.deletingLastPathComponent(), sessionDirectory.appendingPathComponent("screenshots", isDirectory: true)) + XCTAssertTrue(FileManager.default.fileExists(atPath: screenshotPath)) + let persistedSize = try imageSize(in: Data(contentsOf: screenshotURL)) + XCTAssertEqual(persistedSize.width, 24) + XCTAssertEqual(persistedSize.height, 16) + } + + func testEventStreamServiceRecordsInputMonitorMouseAndKeyboardEvents() throws { + let root = try makeTemporaryDirectory() + let installedHandlers = EventStreamInputHandlerBox() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: true, + showsRecordingControls: false, + screenshotPolicy: .never, + installsCGEventTap: false, + inputMonitorInstaller: { mask, handler in + installedHandlers.append((mask, handler)) + return nil + } + ) + + _ = try service.start(approvalDecision: .approved) + let handlers = installedHandlers.snapshot() + XCTAssertTrue(handlers.contains { $0.0.contains(.leftMouseDown) }) + XCTAssertTrue(handlers.contains { $0.0.contains(.leftMouseUp) }) + XCTAssertTrue(handlers.contains { $0.0.contains(.keyDown) }) + + let mouseDown = try XCTUnwrap(NSEvent.mouseEvent( + with: .leftMouseDown, + location: NSPoint(x: 120, y: 180), + modifierFlags: [], + timestamp: 1, + windowNumber: 0, + context: nil, + eventNumber: 1, + clickCount: 1, + pressure: 1 + )) + let mouseUp = try XCTUnwrap(NSEvent.mouseEvent( + with: .leftMouseUp, + location: NSPoint(x: 120, y: 180), + modifierFlags: [], + timestamp: 1.1, + windowNumber: 0, + context: nil, + eventNumber: 2, + clickCount: 1, + pressure: 0 + )) + let keyDown = try XCTUnwrap(NSEvent.keyEvent( + with: .keyDown, + location: .zero, + modifierFlags: [], + timestamp: 1.2, + windowNumber: 0, + context: nil, + characters: "a", + charactersIgnoringModifiers: "a", + isARepeat: false, + keyCode: 0 + )) + + for (mask, handler) in handlers where mask.contains(.leftMouseDown) { + handler(mouseDown) + } + for (mask, handler) in handlers where mask.contains(.leftMouseUp) { + handler(mouseUp) + } + for (mask, handler) in handlers where mask.contains(.keyDown) { + handler(keyDown) + } + + let stopped = try service.stop() + let eventsPath = try XCTUnwrap(stopped.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""type":"mouse.click""#)) + XCTAssertTrue(events.contains(#""button":"left""#)) + XCTAssertTrue(events.contains(#""clickCount":1"#)) + XCTAssertTrue(events.contains(#""type":"keyboard.text_input""#)) + XCTAssertTrue(events.contains(#""keyCode":0"#)) + } + + func testEventStreamServiceCancelUsesOfficialControlsEndReason() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + + _ = try service.start() + let cancelled = try service.cancel() + + XCTAssertEqual(cancelled.state, .cancelled) + XCTAssertEqual(cancelled.endReason, "recording_controls_cancelled") + + let eventsPath = try XCTUnwrap(cancelled.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""endReason":"recording_controls_cancelled""#)) + } + + func testEventStreamStatusPersistsStaleActiveSessionAsUnavailable() throws { + let root = try makeTemporaryDirectory() + let sessionDirectory = root.appendingPathComponent("session-stale", isDirectory: true) + try FileManager.default.createDirectory(at: sessionDirectory, withIntermediateDirectories: true) + let metadataURL = sessionDirectory.appendingPathComponent("metadata.json") + let activeStatus = EventStreamRecordingStatus( + sessionID: "session-stale", + state: .recording, + startedAt: "2026-06-26T00:00:00.000Z", + endedAt: nil, + endReason: nil, + eventsPath: sessionDirectory.appendingPathComponent("events.jsonl").path, + metadataPath: metadataURL.path, + suppressedEventsPath: sessionDirectory.appendingPathComponent("suppressed.jsonl").path, + currentSegmentEventsPath: sessionDirectory.appendingPathComponent("events.jsonl").path, + currentSegmentMetadataPath: metadataURL.path, + eventCount: 3, + suppressedEventCount: 0 + ) + let activeData = try JSONSerialization.data( + withJSONObject: activeStatus.asDictionary, + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + try activeData.write(to: root.appendingPathComponent("latest-session.json"), options: [.atomic]) + try activeData.write(to: root.appendingPathComponent("active-session.json"), options: [.atomic]) + + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let status = service.status() + + XCTAssertEqual(status.sessionID, "session-stale") + XCTAssertEqual(status.state, .stopped) + XCTAssertEqual(status.endReason, "recording_process_unavailable") + XCTAssertNil(status.currentSegmentEventsPath) + XCTAssertNil(status.currentSegmentMetadataPath) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let persistedLatest = try String( + contentsOf: root.appendingPathComponent("latest-session.json"), + encoding: .utf8 + ) + XCTAssertTrue(persistedLatest.contains(#""state" : "stopped""#)) + XCTAssertTrue(persistedLatest.contains(#""endReason" : "recording_process_unavailable""#)) + + let persistedMetadata = try String(contentsOf: metadataURL, encoding: .utf8) + XCTAssertEqual(persistedLatest, persistedMetadata) + let persistedAlias = try String( + contentsOf: sessionDirectory.appendingPathComponent("session.json"), + encoding: .utf8 + ) + XCTAssertEqual(persistedAlias, persistedMetadata) + } + + func testEventStreamStartApprovalDeniedDoesNotCreateSession() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .deny + ) + + XCTAssertThrowsError(try service.start()) { error in + XCTAssertEqual((error as? EventStreamStartApprovalError)?.decision, .denied) + XCTAssertEqual((error as? LocalizedError)?.errorDescription, eventStreamStartApprovalDeniedMessage) + } + XCTAssertEqual(service.status(), .idle) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + } + + func testEventStreamStartApprovalCancelledDoesNotCreateSession() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .cancel + ) + + XCTAssertThrowsError(try service.start()) { error in + XCTAssertEqual((error as? EventStreamStartApprovalError)?.decision, .cancelled) + XCTAssertEqual((error as? LocalizedError)?.errorDescription, eventStreamStartApprovalCancelledMessage) + } + XCTAssertEqual(service.status(), .idle) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + } + + func testEventStreamServiceWaitReturnsAfterConcurrentStop() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let started = try service.start() + let sessionID = try XCTUnwrap(started.sessionID) + + let expectation = expectation(description: "wait returned") + let waitedStatus = EventStreamStatusBox() + DispatchQueue.global().async { + waitedStatus.value = service.wait(sessionID: sessionID, timeout: 2) + expectation.fulfill() + } + + Thread.sleep(forTimeInterval: 0.2) + let stopped = try service.stop() + wait(for: [expectation], timeout: 3) + + XCTAssertEqual(stopped.state, .stopped) + XCTAssertEqual(waitedStatus.value?.sessionID, sessionID) + XCTAssertEqual(waitedStatus.value?.state, .stopped) + XCTAssertEqual(waitedStatus.value?.endReason, "recording_controls_stopped") + } + + func testRunEventStreamCLIWaitReturnsAfterStop() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + let expectation = expectation(description: "cli wait returned") + let waitedStatus = JSONDictionaryBox() + DispatchQueue.global().async { + let output = try? runOpenComputerUseEventStream( + .wait(json: true, sessionID: sessionID, timeout: 2, notifyCommand: nil), + service: service + ) + waitedStatus.value = output?.jsonObject as? [String: Any] + expectation.fulfill() + } + + Thread.sleep(forTimeInterval: 0.2) + let stopped = try runOpenComputerUseEventStream(.stop(json: true), service: service) + wait(for: [expectation], timeout: 3) + + let stoppedStatus = try XCTUnwrap(stopped.jsonObject as? [String: Any]) + XCTAssertEqual(stoppedStatus["state"] as? String, "stopped") + XCTAssertEqual(waitedStatus.value?["sessionId"] as? String, sessionID) + XCTAssertEqual(waitedStatus.value?["state"] as? String, "stopped") + XCTAssertEqual(waitedStatus.value?["endReason"] as? String, "recording_controls_stopped") + XCTAssertEqual(waitedStatus.value?["waitTimedOut"] as? Bool, false) + } + + func testRunEventStreamCLIWaitReturnsImmediatelyAfterCompletedSession() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + _ = try runOpenComputerUseEventStream(.stop(json: true), service: service) + + let beforeWait = Date() + let waited = try runOpenComputerUseEventStream( + .wait(json: true, sessionID: sessionID, timeout: 5, notifyCommand: nil), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertLessThan(Date().timeIntervalSince(beforeWait), 0.5) + XCTAssertEqual(waitedStatus["sessionId"] as? String, sessionID) + XCTAssertEqual(waitedStatus["state"] as? String, "stopped") + XCTAssertEqual(waitedStatus["endReason"] as? String, "recording_controls_stopped") + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, true) + } + + func testRunEventStreamCLIWaitReturnsStoredCompletedSessionByID() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let firstStarted = try runOpenComputerUseEventStream(.start(json: true), service: service) + let firstStartedStatus = try XCTUnwrap(firstStarted.jsonObject as? [String: Any]) + let firstSessionID = try XCTUnwrap(firstStartedStatus["sessionId"] as? String) + _ = try runOpenComputerUseEventStream(.stop(json: true), service: service) + + let secondStarted = try runOpenComputerUseEventStream(.start(json: true), service: service) + let secondStartedStatus = try XCTUnwrap(secondStarted.jsonObject as? [String: Any]) + let secondSessionID = try XCTUnwrap(secondStartedStatus["sessionId"] as? String) + XCTAssertNotEqual(firstSessionID, secondSessionID) + + let beforeWait = Date() + let waited = try runOpenComputerUseEventStream( + .wait(json: true, sessionID: firstSessionID, timeout: 5, notifyCommand: nil), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertLessThan(Date().timeIntervalSince(beforeWait), 0.5) + XCTAssertEqual(waitedStatus["sessionId"] as? String, firstSessionID) + XCTAssertEqual(waitedStatus["state"] as? String, "stopped") + XCTAssertEqual(waitedStatus["endReason"] as? String, "recording_controls_stopped") + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, true) + + let activeStatus = service.status() + XCTAssertEqual(activeStatus.sessionID, secondSessionID) + XCTAssertEqual(activeStatus.state, .recording) + _ = try service.stop() + } + + func testRunEventStreamCLIWaitForUnknownSessionDoesNotBlockOrNotify() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify.sh") + let callbackMarkerPath = root.appendingPathComponent("called.txt") + try """ + #!/bin/sh + touch "$1" + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let beforeWait = Date() + let waited = try runOpenComputerUseEventStream( + .wait( + json: true, + sessionID: "session-20260626T000000000Z-missing", + timeout: nil, + notifyCommand: [callbackScript.path, callbackMarkerPath.path] + ), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertLessThan(Date().timeIntervalSince(beforeWait), 0.5) + XCTAssertFalse(waited.hasToolError) + XCTAssertEqual(waitedStatus["state"] as? String, "idle") + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, true) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, false) + let notification = try XCTUnwrap(waitedStatus["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, false) + XCTAssertEqual(notification["skipped"] as? Bool, true) + XCTAssertEqual(notification["reason"] as? String, "waitTimedOut") + XCTAssertFalse(FileManager.default.fileExists(atPath: callbackMarkerPath.path)) + } + + func testRunEventStreamCLIWaitMarksTimeout() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + let waited = try runOpenComputerUseEventStream( + .wait(json: true, sessionID: sessionID, timeout: 0.05, notifyCommand: nil), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertEqual(waitedStatus["sessionId"] as? String, sessionID) + XCTAssertEqual(waitedStatus["state"] as? String, "recording") + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, true) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, true) + + _ = try service.stop() + } + + func testRunEventStreamCLIWaitRunsNotifyCommandAfterStop() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify.sh") + let callbackStatusPath = root.appendingPathComponent("notify-status.json") + let callbackSessionPath = root.appendingPathComponent("notify-session.txt") + let callbackReasonPath = root.appendingPathComponent("notify-reason.txt") + try """ + #!/bin/sh + cat > "$1" + printf '%s' "$OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID" > "$2" + printf '%s' "$OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON" > "$3" + test "$OPEN_COMPUTER_USE_EVENT_STREAM_STATE" = "stopped" + test -n "$OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH" + test -f "$OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH" + test -n "$OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH" + test -f "$OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH" + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + let expectation = expectation(description: "cli wait returned") + let waitedStatus = JSONDictionaryBox() + DispatchQueue.global().async { + let output = try? runOpenComputerUseEventStream( + .wait( + json: true, + sessionID: sessionID, + timeout: 2, + notifyCommand: [ + callbackScript.path, + callbackStatusPath.path, + callbackSessionPath.path, + callbackReasonPath.path, + ] + ), + service: service + ) + waitedStatus.value = output?.jsonObject as? [String: Any] + expectation.fulfill() + } + + Thread.sleep(forTimeInterval: 0.2) + _ = try runOpenComputerUseEventStream(.stop(json: true), service: service) + wait(for: [expectation], timeout: 3) + + let waited = try XCTUnwrap(waitedStatus.value) + XCTAssertEqual(waited["sessionId"] as? String, sessionID) + XCTAssertEqual(waited["state"] as? String, "stopped") + XCTAssertEqual(waited["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waited["waitSessionMatched"] as? Bool, true) + let notification = try XCTUnwrap(waited["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, true) + XCTAssertEqual(notification["skipped"] as? Bool, false) + XCTAssertEqual(notification["ok"] as? Bool, true) + XCTAssertEqual(notification["exitCode"] as? Int, 0) + + let callbackStatus = try jsonObject(contentsOf: callbackStatusPath) + XCTAssertEqual(callbackStatus["sessionId"] as? String, sessionID) + XCTAssertEqual(callbackStatus["state"] as? String, "stopped") + XCTAssertEqual(callbackStatus["waitTimedOut"] as? Bool, false) + XCTAssertEqual(callbackStatus["waitSessionMatched"] as? Bool, true) + let callbackSession = try String(contentsOf: callbackSessionPath, encoding: .utf8) + XCTAssertEqual(callbackSession, sessionID) + let callbackReason = try String(contentsOf: callbackReasonPath, encoding: .utf8) + XCTAssertEqual(callbackReason, "recording_controls_stopped") + } + + func testRunEventStreamCLIWaitRunsNotifyCommandAfterCancel() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify.sh") + let callbackStatusPath = root.appendingPathComponent("notify-status.json") + let callbackReasonPath = root.appendingPathComponent("notify-reason.txt") + try """ + #!/bin/sh + cat > "$1" + printf '%s' "$OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON" > "$2" + test "$OPEN_COMPUTER_USE_EVENT_STREAM_STATE" = "cancelled" + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + let expectation = expectation(description: "cli wait returned after cancel") + let waitedStatus = JSONDictionaryBox() + DispatchQueue.global().async { + let output = try? runOpenComputerUseEventStream( + .wait( + json: true, + sessionID: sessionID, + timeout: 2, + notifyCommand: [callbackScript.path, callbackStatusPath.path, callbackReasonPath.path] + ), + service: service + ) + waitedStatus.value = output?.jsonObject as? [String: Any] + expectation.fulfill() + } + + Thread.sleep(forTimeInterval: 0.2) + _ = try runOpenComputerUseEventStream(.cancel(json: true), service: service) + wait(for: [expectation], timeout: 3) + + let waited = try XCTUnwrap(waitedStatus.value) + XCTAssertEqual(waited["sessionId"] as? String, sessionID) + XCTAssertEqual(waited["state"] as? String, "cancelled") + XCTAssertEqual(waited["endReason"] as? String, "recording_controls_cancelled") + XCTAssertEqual(waited["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waited["waitSessionMatched"] as? Bool, true) + let notification = try XCTUnwrap(waited["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, true) + XCTAssertEqual(notification["ok"] as? Bool, true) + + let callbackStatus = try jsonObject(contentsOf: callbackStatusPath) + XCTAssertEqual(callbackStatus["sessionId"] as? String, sessionID) + XCTAssertEqual(callbackStatus["state"] as? String, "cancelled") + XCTAssertEqual(callbackStatus["endReason"] as? String, "recording_controls_cancelled") + let callbackReason = try String(contentsOf: callbackReasonPath, encoding: .utf8) + XCTAssertEqual(callbackReason, "recording_controls_cancelled") + } + + func testRunEventStreamCLIWaitSkipsNotifyCommandOnTimeout() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify.sh") + let callbackMarkerPath = root.appendingPathComponent("called.txt") + try """ + #!/bin/sh + touch "$1" + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + + let waited = try runOpenComputerUseEventStream( + .wait( + json: true, + sessionID: sessionID, + timeout: 0.05, + notifyCommand: [callbackScript.path, callbackMarkerPath.path] + ), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertFalse(waited.hasToolError) + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, true) + let notification = try XCTUnwrap(waitedStatus["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, false) + XCTAssertEqual(notification["skipped"] as? Bool, true) + XCTAssertEqual(notification["ok"] as? Bool, true) + XCTAssertEqual(notification["reason"] as? String, "waitTimedOut") + XCTAssertFalse(FileManager.default.fileExists(atPath: callbackMarkerPath.path)) + + _ = try service.stop() + } + + func testRunEventStreamCLIWaitReportsNotifyCommandNonZeroExit() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify-fail.sh") + try """ + #!/bin/sh + exit 7 + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + _ = try runOpenComputerUseEventStream(.stop(json: true), service: service) + + let waited = try runOpenComputerUseEventStream( + .wait(json: true, sessionID: sessionID, timeout: 1, notifyCommand: [callbackScript.path]), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertTrue(waited.hasToolError) + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, true) + let notification = try XCTUnwrap(waitedStatus["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, true) + XCTAssertEqual(notification["skipped"] as? Bool, false) + XCTAssertEqual(notification["ok"] as? Bool, false) + XCTAssertEqual(notification["exitCode"] as? Int, 7) + XCTAssertEqual(notification["timedOut"] as? Bool, false) + XCTAssertEqual(notification["reason"] as? String, "nonZeroExit") + } + + func testRunEventStreamCLIWaitReportsNotifyCommandTimeout() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let callbackScript = root.appendingPathComponent("notify-sleep.sh") + try """ + #!/bin/sh + sleep 2 + """.write(to: callbackScript, atomically: true, encoding: .utf8) + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: callbackScript.path) + + let started = try runOpenComputerUseEventStream(.start(json: true), service: service) + let startedStatus = try XCTUnwrap(started.jsonObject as? [String: Any]) + let sessionID = try XCTUnwrap(startedStatus["sessionId"] as? String) + _ = try runOpenComputerUseEventStream(.stop(json: true), service: service) + + setenv("OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS", "0.1", 1) + defer { + unsetenv("OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS") + } + + let waited = try runOpenComputerUseEventStream( + .wait(json: true, sessionID: sessionID, timeout: 1, notifyCommand: [callbackScript.path]), + service: service + ) + let waitedStatus = try XCTUnwrap(waited.jsonObject as? [String: Any]) + + XCTAssertTrue(waited.hasToolError) + XCTAssertEqual(waitedStatus["waitTimedOut"] as? Bool, false) + XCTAssertEqual(waitedStatus["waitSessionMatched"] as? Bool, true) + let notification = try XCTUnwrap(waitedStatus["notification"] as? [String: Any]) + XCTAssertEqual(notification["attempted"] as? Bool, true) + XCTAssertEqual(notification["skipped"] as? Bool, false) + XCTAssertEqual(notification["ok"] as? Bool, false) + XCTAssertEqual(notification["timedOut"] as? Bool, true) + XCTAssertEqual(notification["reason"] as? String, "timeout") + XCTAssertEqual(notification["timeoutSeconds"] as? Double, 0.1) + } + + func testEventStreamRecordingSummaryRedactsTextByDefault() throws { + let fixture = try makeEventStreamSummaryFixture() + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + + XCTAssertEqual(summary["ok"] as? Bool, true) + XCTAssertEqual(summary["actionEventCount"] as? Int, 3) + XCTAssertEqual(summary["includesRawText"] as? Bool, false) + + let jsonData = try JSONSerialization.data(withJSONObject: summary) + let jsonText = String(data: jsonData, encoding: .utf8) ?? "" + XCTAssertFalse(jsonText.contains("secret-value")) + XCTAssertFalse(jsonText.contains("selected-secret")) + XCTAssertTrue(jsonText.contains("valueLength")) + XCTAssertTrue(jsonText.contains("selectedTextLength")) + + let runtimeInputs = try XCTUnwrap(summary["runtimeInputs"] as? [[String: Any]]) + XCTAssertEqual(runtimeInputs.count, 2) + XCTAssertEqual(runtimeInputs.first?["kind"] as? String, "text") + XCTAssertEqual(runtimeInputs.first?["sourceEventType"] as? String, "keyboard.text_input") + XCTAssertEqual(runtimeInputs.first?["textLength"] as? Int, 6) + XCTAssertEqual(runtimeInputs.first?["sensitive"] as? Bool, true) + XCTAssertEqual(runtimeInputs.last?["kind"] as? String, "selection") + XCTAssertEqual(runtimeInputs.last?["sourceEventType"] as? String, "selection.changed") + + let summaryLimits = try XCTUnwrap(summary["summaryLimits"] as? [String: Any]) + XCTAssertEqual(summaryLimits["hasTruncatedSummary"] as? Bool, false) + let omittedCounts = try XCTUnwrap(summaryLimits["omittedCounts"] as? [String: Int]) + XCTAssertEqual(omittedCounts["actionSequence"], 0) + + let safetySignals = try XCTUnwrap(summary["safetySignals"] as? [[String: Any]]) + XCTAssertEqual(safetySignals.count, 1) + XCTAssertEqual(safetySignals.first?["sourceEventType"] as? String, "mouse.click") + XCTAssertEqual(safetySignals.first?["reason"] as? String, "saveAction") + XCTAssertEqual(safetySignals.first?["confirmationRequired"] as? Bool, true) + let safetyTarget = try XCTUnwrap(safetySignals.first?["target"] as? [String: Any]) + XCTAssertEqual(safetyTarget["title"] as? String, "Save") + + let skillEvidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(skillEvidence["hasActionEvents"] as? Bool, true) + XCTAssertEqual(skillEvidence["hasPointerEvents"] as? Bool, true) + XCTAssertEqual(skillEvidence["hasInputEvents"] as? Bool, true) + XCTAssertEqual(skillEvidence["hasRedactionSignals"] as? Bool, true) + XCTAssertEqual(skillEvidence["hasSafetySignals"] as? Bool, true) + + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "needsReview") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, true) + XCTAssertEqual(skillReadiness["requiresHumanReview"] as? Bool, true) + let readinessReasons = try XCTUnwrap(skillReadiness["reasons"] as? [String]) + XCTAssertTrue(readinessReasons.contains("recording has no AX focused window context")) + XCTAssertTrue(readinessReasons.contains("recording includes redaction or secureInput signals")) + XCTAssertTrue(readinessReasons.contains("recording includes actions that may require explicit user confirmation")) + } + + func testEventStreamRecordingSummaryReportsTruncatedFields() throws { + let root = try makeTemporaryDirectory() + let eventsURL = root.appendingPathComponent("events.jsonl") + let metadataURL = root.appendingPathComponent("metadata.json") + + var events: [[String: Any]] = [ + [ + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.000Z", + "sessionId": "summary-long-fixture", + ], + [ + "type": "window.changed", + "timestamp": "2026-06-26T00:00:01.000Z", + "sessionId": "summary-long-fixture", + "appName": "Fixture", + "windowTitle": "Long Demo", + ], + ] + for index in 0 ..< 55 { + events.append([ + "type": "mouse.click", + "timestamp": "2026-06-26T00:00:\(String(format: "%02d", index + 2)).000Z", + "sessionId": "summary-long-fixture", + "appName": "Fixture", + "windowTitle": "Long Demo", + "targetAccessibilityElement": [ + "role": "AXButton", + "title": "Step \(index)", + ], + ]) + } + events.append([ + "type": "session.ended", + "timestamp": "2026-06-26T00:01:00.000Z", + "sessionId": "summary-long-fixture", + "endReason": "recording_controls_stopped", + ]) + + let lines = try events.map { event -> String in + let data = try JSONSerialization.data(withJSONObject: event, options: [.withoutEscapingSlashes]) + return String(data: data, encoding: .utf8) ?? "{}" + }.joined(separator: "\n") + try (lines + "\n").write(to: eventsURL, atomically: true, encoding: .utf8) + try [ + "sessionId": "summary-long-fixture", + "state": "stopped", + "active": false, + "endReason": "recording_controls_stopped", + "eventCount": events.count, + "suppressedEventCount": 0, + "eventsPath": eventsURL.path, + "metadataPath": metadataURL.path, + ].writeJSONObject(to: metadataURL) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: metadataURL.path, + includeText: false, + requireAction: true + )) + + XCTAssertEqual(summary["ok"] as? Bool, true) + XCTAssertEqual(summary["actionEventCount"] as? Int, 55) + XCTAssertEqual((summary["actionSequence"] as? [[String: Any]])?.count, 50) + XCTAssertEqual((summary["targetElements"] as? [[String: Any]])?.count, 25) + + let summaryLimits = try XCTUnwrap(summary["summaryLimits"] as? [String: Any]) + XCTAssertEqual(summaryLimits["hasTruncatedSummary"] as? Bool, true) + let omittedCounts = try XCTUnwrap(summaryLimits["omittedCounts"] as? [String: Int]) + XCTAssertEqual(omittedCounts["actionSequence"], 5) + XCTAssertEqual(omittedCounts["targetElements"], 30) + + let warnings = try XCTUnwrap(summary["warnings"] as? [String]) + XCTAssertTrue(warnings.contains("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill")) + + let skillEvidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(skillEvidence["hasTruncatedSummary"] as? Bool, true) + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + let readinessReasons = try XCTUnwrap(skillReadiness["reasons"] as? [String]) + XCTAssertTrue(readinessReasons.contains("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill")) + } + + func testEventStreamRecordingSummaryCanIncludeTextForSanitizedFixtures() throws { + let fixture = try makeEventStreamSummaryFixture() + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.sessionDirectory.path, + includeText: true, + requireAction: true + )) + let jsonData = try JSONSerialization.data(withJSONObject: summary) + let jsonText = String(data: jsonData, encoding: .utf8) ?? "" + + XCTAssertEqual(summary["includesRawText"] as? Bool, true) + XCTAssertTrue(jsonText.contains("secret-value")) + XCTAssertTrue(jsonText.contains("selected-secret")) + } + + func testEventStreamRecordingSummaryRequireActionMarksToolError() throws { + let root = try makeTemporaryDirectory() + let eventsURL = root.appendingPathComponent("events.jsonl") + let metadataURL = root.appendingPathComponent("metadata.json") + try #"{"type":"session.started","sessionId":"summary-no-action"}"# + .appending("\n") + .write(to: eventsURL, atomically: true, encoding: .utf8) + try [ + "sessionId": "summary-no-action", + "state": "recording", + "active": true, + "eventCount": 1, + "suppressedEventCount": 0, + "eventsPath": eventsURL.path, + "metadataPath": metadataURL.path, + ].writeJSONObject(to: metadataURL) + + let output = try runOpenComputerUseEventStream(.summarize( + inputPath: metadataURL.path, + includeText: false, + requireAction: true + )) + let summary = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["actionEventCount"] as? Int, 0) + let errors = try XCTUnwrap(summary["errors"] as? [String]) + XCTAssertTrue(errors.contains("required at least one high-level user action event")) + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "insufficient") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingSummaryCancelledIsInsufficientForSkillCreation() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + + let output = try runOpenComputerUseEventStream(.summarize( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + let summary = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["actionEventCount"] as? Int, 3) + XCTAssertEqual(summary["state"] as? String, "cancelled") + XCTAssertEqual(summary["endReason"] as? String, "recording_controls_cancelled") + let errors = try XCTUnwrap(summary["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + let warnings = try XCTUnwrap(summary["warnings"] as? [String]) + XCTAssertTrue(warnings.contains("recording was cancelled; do not create or update a skill from this event stream")) + + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "insufficient") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, false) + XCTAssertEqual(skillReadiness["requiresHumanReview"] as? Bool, true) + let readinessReasons = try XCTUnwrap(skillReadiness["reasons"] as? [String]) + XCTAssertTrue(readinessReasons.contains("recording was cancelled; do not create or update a skill from this event stream")) + } + + func testEventStreamRecordingSummaryDetectsCancelledEventsJSONLInput() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + + let output = try runOpenComputerUseEventStream(.summarize( + inputPath: fixture.eventsURL.path, + includeText: false, + requireAction: true + )) + let summary = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["endReason"] as? String, "recording_controls_cancelled") + let errors = try XCTUnwrap(summary["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "insufficient") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingSummaryIncompleteRecordingIsInsufficientForSkillCreation() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "recording", + endReason: "", + includeSessionEnded: false + ) + + let output = try runOpenComputerUseEventStream(.summarize( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + let summary = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["actionEventCount"] as? Int, 3) + XCTAssertEqual(summary["state"] as? String, "recording") + let errors = try XCTUnwrap(summary["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording is not complete; stop the recording before creating a skill")) + let warnings = try XCTUnwrap(summary["warnings"] as? [String]) + XCTAssertTrue(warnings.contains("recording is not complete; stop the recording before creating a skill")) + + let skillEvidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(skillEvidence["recordingIncomplete"] as? Bool, true) + + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "insufficient") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, false) + XCTAssertEqual(skillReadiness["requiresHumanReview"] as? Bool, true) + let readinessReasons = try XCTUnwrap(skillReadiness["reasons"] as? [String]) + XCTAssertTrue(readinessReasons.contains("recording is not complete; stop the recording before creating a skill")) + } + + func testEventStreamRecordingSummaryBlockingDiagnosticsAreInsufficientForSkillCreation() throws { + let fixture = try makeEventStreamSummaryFixture(includeBlockingDiagnostic: true) + + let output = try runOpenComputerUseEventStream(.summarize( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + let summary = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["actionEventCount"] as? Int, 3) + let errors = try XCTUnwrap(summary["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + let warnings = try XCTUnwrap(summary["warnings"] as? [String]) + XCTAssertTrue(warnings.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + + let blockingDiagnostics = try XCTUnwrap(summary["blockingDiagnostics"] as? [[String: Any]]) + XCTAssertEqual(blockingDiagnostics.count, 1) + XCTAssertEqual(blockingDiagnostics.first?["subsystem"] as? String, "inputMonitoring") + XCTAssertEqual(blockingDiagnostics.first?["reason"] as? String, "inputMonitorsUnavailable") + + let skillEvidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(skillEvidence["hasBlockingDiagnostics"] as? Bool, true) + + let skillReadiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(skillReadiness["status"] as? String, "insufficient") + XCTAssertEqual(skillReadiness["canCreateSkillDraft"] as? Bool, false) + XCTAssertEqual(skillReadiness["requiresHumanReview"] as? Bool, true) + let readinessReasons = try XCTUnwrap(skillReadiness["reasons"] as? [String]) + XCTAssertTrue(readinessReasons.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + } + + func testEventStreamSkillScaffoldWritesDraftAndSanitizedSummary() throws { + let fixture = try makeEventStreamSummaryFixture() + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("generated-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "recorded-example-workflow", + description: "Replay an example workflow.", + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertFalse(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, true) + XCTAssertEqual(result["skillName"] as? String, "recorded-example-workflow") + + let skillURL = outputDirectory.appendingPathComponent("SKILL.md") + let summaryURL = outputDirectory.appendingPathComponent("references/recording-summary.json") + let skillText = try String(contentsOf: skillURL, encoding: .utf8) + let summaryText = try String(contentsOf: summaryURL, encoding: .utf8) + + XCTAssertTrue(skillText.contains("name: recorded-example-workflow")) + XCTAssertTrue(skillText.contains("Click role=\"AXButton\", title=\"Save\" in the recorded app.")) + XCTAssertTrue(skillText.contains("Enter user-provided text into role=\"AXTextField\", label=\"Token\"")) + XCTAssertTrue(skillText.contains("## Runtime Inputs")) + XCTAssertTrue(skillText.contains("Runtime text for role=\"AXTextField\", label=\"Token\"")) + XCTAssertTrue(skillText.contains("use a fresh runtime value")) + XCTAssertTrue(skillText.contains("Treat this as sensitive input")) + XCTAssertTrue(skillText.contains("Runtime selection or selected-content meaning")) + XCTAssertTrue(skillText.contains("## Summary Limits")) + XCTAssertTrue(skillText.contains("No high-volume summary fields were truncated.")) + XCTAssertTrue(skillText.contains("### Confirmation Signals")) + XCTAssertTrue(skillText.contains("`mouse.click` matched saveAction")) + XCTAssertTrue(skillText.contains("ask for explicit confirmation before replaying this step")) + XCTAssertTrue(skillText.contains("## Workflow Readiness")) + XCTAssertTrue(skillText.contains("Status: needsReview")) + XCTAssertTrue(skillText.contains("recording has no AX focused window context")) + XCTAssertTrue(skillText.contains("## Agent Replay Procedure")) + XCTAssertTrue(skillText.contains("connector, API, or dedicated tool")) + XCTAssertTrue(skillText.contains("visually dependent verification")) + XCTAssertTrue(skillText.contains("get_app_state")) + XCTAssertTrue(skillText.contains("`element_index` actions")) + XCTAssertTrue(skillText.contains("## Verification")) + XCTAssertTrue(skillText.contains("open-computer-use event-stream validate --json --strict-ocu --require-skill-draft ")) + XCTAssertTrue(skillText.contains("## Finalizing The Skill")) + XCTAssertTrue(skillText.contains("skill-creator")) + XCTAssertTrue(skillText.contains("not a standalone runbook or replay plan")) + XCTAssertFalse(skillText.contains("secret-value")) + XCTAssertFalse(summaryText.contains(fixture.sessionDirectory.path)) + XCTAssertFalse(summaryText.contains("secret-value")) + XCTAssertTrue(summaryText.contains(#""sessionDir" : """#)) + XCTAssertTrue(summaryText.contains(#""metadataPath" : """#)) + XCTAssertTrue(summaryText.contains(#""eventsPath" : """#)) + XCTAssertTrue(summaryText.contains(#""runtimeInputs""#)) + XCTAssertTrue(summaryText.contains(#""safetySignals""#)) + XCTAssertTrue(summaryText.contains(#""summaryLimits""#)) + XCTAssertTrue(summaryText.contains(#""hasSafetySignals" : true"#)) + XCTAssertTrue(summaryText.contains(#""skillReadiness""#)) + + let duplicateOutput = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "recorded-example-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let duplicateResult = try XCTUnwrap(duplicateOutput.jsonObject as? [String: Any]) + XCTAssertTrue(duplicateOutput.hasToolError) + XCTAssertEqual(duplicateResult["error"] as? String, "outputDirectoryExists") + } + + func testEventStreamSummaryAndScaffoldSurfaceScrollRawEventsAsReplaySteps() throws { + let fixture = try makeEventStreamSummaryFixture() + var events = try jsonObjects(contentsOf: fixture.eventsURL) + let scrollEvent: [String: Any] = [ + "type": "experimentalRawEvents", + "timestamp": "2026-06-26T00:00:04.250Z", + "sessionId": "summary-fixture", + "reason": "scrollWheel", + "appName": "Fixture", + "windowTitle": "Demo", + "experimentalRawEvents": [ + [ + "eventType": "scrollWheel", + "scrollingDeltaX": 0, + "scrollingDeltaY": -4, + "hasPreciseScrollingDeltas": false, + ], + ], + ] + events.insert(scrollEvent, at: max(0, events.count - 1)) + try writeEventStreamFixtureEvents(events, fixture: fixture) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, true) + XCTAssertEqual(summary["actionEventCount"] as? Int, 4) + let actions = try XCTUnwrap(summary["actionSequence"] as? [[String: Any]]) + let scrollAction = try XCTUnwrap(actions.first { $0["reason"] as? String == "scrollWheel" }) + XCTAssertEqual(scrollAction["type"] as? String, "experimentalRawEvents") + XCTAssertEqual(scrollAction["rawEventTypes"] as? [String], ["scrollWheel"]) + XCTAssertEqual(scrollAction["scrollingDeltaY"] as? Int, -4) + + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("generated-scroll-skill") + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "recorded-scroll-workflow", + description: "Replay a scroll workflow.", + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + XCTAssertFalse(output.hasToolError) + let skillText = try String( + contentsOf: outputDirectory.appendingPathComponent("SKILL.md"), + encoding: .utf8 + ) + XCTAssertTrue(skillText.contains("Scroll in Fixture using the recorded wheel direction")) + } + + func testEventStreamSkillScaffoldAcceptsEventsJSONLInput() throws { + let fixture = try makeEventStreamSummaryFixture() + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("events-jsonl-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.eventsURL.path, + skillName: "events-jsonl-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertFalse(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, true) + XCTAssertTrue(FileManager.default.fileExists(atPath: outputDirectory.appendingPathComponent("SKILL.md").path)) + let summaryText = try String( + contentsOf: outputDirectory.appendingPathComponent("references/recording-summary.json"), + encoding: .utf8 + ) + XCTAssertTrue(summaryText.contains(#""eventsPath" : """#)) + XCTAssertFalse(summaryText.contains(fixture.sessionDirectory.path)) + } + + func testEventStreamSkillScaffoldRejectsInvalidSkillName() throws { + let fixture = try makeEventStreamSummaryFixture() + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "Bad_Name", + description: nil, + outputDirectory: fixture.sessionDirectory.appendingPathComponent("bad-skill").path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "invalidSkillName") + } + + func testEventStreamSkillScaffoldRejectsCancelledRecording() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("cancelled-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "cancelled-recording-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "recordingValidationFailed") + let errors = try XCTUnwrap(result["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputDirectory.path)) + } + + func testEventStreamSkillScaffoldRejectsCancelledEventsJSONLInput() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("cancelled-events-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.eventsURL.path, + skillName: "cancelled-events-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "recordingValidationFailed") + let errors = try XCTUnwrap(result["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputDirectory.path)) + } + + func testEventStreamSkillScaffoldRejectsIncompleteRecording() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "recording", + endReason: "", + includeSessionEnded: false + ) + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("incomplete-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "incomplete-recording-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "recordingValidationFailed") + let errors = try XCTUnwrap(result["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording is not complete; stop the recording before creating a skill")) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputDirectory.path)) + } + + func testEventStreamSkillScaffoldRejectsBlockingDiagnostics() throws { + let fixture = try makeEventStreamSummaryFixture(includeBlockingDiagnostic: true) + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("diagnostic-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.metadataURL.path, + skillName: "diagnostic-recording-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "recordingValidationFailed") + let errors = try XCTUnwrap(result["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputDirectory.path)) + } + + func testEventStreamSkillScaffoldRejectsInvalidRecordingStructure() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["eventCount"] = 999 + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + let outputDirectory = fixture.sessionDirectory.appendingPathComponent("invalid-structure-skill") + + let output = try runOpenComputerUseEventStream(.scaffoldSkill( + inputPath: fixture.sessionDirectory.path, + skillName: "invalid-structure-workflow", + description: nil, + outputDirectory: outputDirectory.path, + overwrite: false, + includeText: false + )) + let result = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(result["ok"] as? Bool, false) + XCTAssertEqual(result["error"] as? String, "recordingValidationFailed") + let errors = try XCTUnwrap(result["errors"] as? [String]) + XCTAssertTrue(errors.contains("eventCount=999 does not match events.jsonl lines=6")) + XCTAssertFalse(FileManager.default.fileExists(atPath: outputDirectory.path)) + } + + func testEventStreamRecordingValidationAcceptsStrictOCUFixture() throws { + let fixture = try makeEventStreamSummaryFixture() + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.metadataURL.path, + strictOCU: true, + requiredEventTypes: ["session.started", "mouse.click", "session.ended"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertFalse(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, true) + XCTAssertEqual(validation["eventCount"] as? Int, 6) + XCTAssertEqual(validation["metadataEventCount"] as? Int, 6) + XCTAssertEqual(validation["requireSkillDraft"] as? Bool, true) + XCTAssertEqual(validation["skillDraftReady"] as? Bool, true) + XCTAssertEqual(validation["skillDraftReasons"] as? [String], []) + let declaredPaths = try XCTUnwrap(validation["declaredPaths"] as? [String: [String: Any]]) + XCTAssertEqual(declaredPaths["metadataPath"]?["exists"] as? Bool, true) + XCTAssertEqual(declaredPaths["sessionPath"]?["exists"] as? Bool, true) + XCTAssertEqual(declaredPaths["eventsPath"]?["exists"] as? Bool, true) + XCTAssertEqual(declaredPaths["suppressedEventsPath"]?["exists"] as? Bool, true) + let eventTypes = try XCTUnwrap(validation["eventTypes"] as? [String: Int]) + XCTAssertEqual(eventTypes["mouse.click"], 1) + XCTAssertTrue(JSONSerialization.isValidJSONObject(validation)) + } + + func testEventStreamRecordingValidationStrictOCURequiresDeclaredHandoffPaths() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata.removeValue(forKey: "metadataPath") + metadata.removeValue(forKey: "sessionPath") + metadata.removeValue(forKey: "eventsPath") + metadata.removeValue(forKey: "suppressedEventsPath") + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("strict OCU validation requires metadataPath")) + XCTAssertTrue(errors.contains("strict OCU validation requires sessionPath")) + XCTAssertTrue(errors.contains("strict OCU validation requires eventsPath")) + XCTAssertTrue(errors.contains("strict OCU validation requires suppressedEventsPath")) + let declaredPaths = try XCTUnwrap(validation["declaredPaths"] as? [String: [String: Any]]) + XCTAssertNotNil(declaredPaths["eventsPath"]?["exists"] as? NSNull) + } + + func testEventStreamRecordingValidationReportsBadDeclaredHandoffPath() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["sessionPath"] = "missing-session.json" + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("sessionPath does not exist: missing-session.json")) + let declaredPaths = try XCTUnwrap(validation["declaredPaths"] as? [String: [String: Any]]) + XCTAssertEqual(declaredPaths["sessionPath"]?["exists"] as? Bool, false) + } + + func testEventStreamRecordingValidationRejectsExternalScreenshotPath() throws { + let fixture = try makeEventStreamSummaryFixture() + let externalScreenshotURL = fixture.sessionDirectory + .deletingLastPathComponent() + .appendingPathComponent("outside-session-screenshot.png") + try Data([0x89, 0x50, 0x4e, 0x47]).write(to: externalScreenshotURL) + + var events = try jsonObjects(contentsOf: fixture.eventsURL) + events[2]["accessibilityInspectorPayload"] = [ + "screenshotPath": externalScreenshotURL.path, + ] + try writeEventStreamFixtureEvents(events, fixture: fixture) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("screenshotPath from event line 3 must stay inside session directory: \(externalScreenshotURL.path)")) + XCTAssertFalse(errors.contains("screenshotPath from event line 3 does not exist: \(externalScreenshotURL.path)")) + } + + func testEventStreamRecordingValidationAcceptsEventsJSONLInputWithoutStrictOCU() throws { + let fixture = try makeEventStreamSummaryFixture() + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.eventsURL.path, + strictOCU: false, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertFalse(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, true) + XCTAssertEqual(validation["eventCount"] as? Int, 6) + XCTAssertEqual(validation["skillDraftReady"] as? Bool, true) + XCTAssertNil(validation["metadataPath"]) + let warnings = try XCTUnwrap(validation["warnings"] as? [String]) + XCTAssertTrue(warnings.contains("metadata/session files not available; validating events.jsonl only")) + } + + func testEventStreamRecordingValidationRejectsCancelledEventsJSONLInput() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.eventsURL.path, + strictOCU: false, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["endReason"] as? String, "recording_controls_cancelled") + XCTAssertEqual(validation["skillDraftReady"] as? Bool, false) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording was cancelled; do not create or update a skill from this event stream")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + } + + func testEventStreamRecordingValidationRejectsEventsJSONLInputWithStrictOCU() throws { + let fixture = try makeEventStreamSummaryFixture() + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.eventsURL.path, + strictOCU: true, + requiredEventTypes: [], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("strict OCU validation requires metadata.json or session.json")) + } + + func testEventStreamRecordingValidationRequireSkillDraftRejectsCancelledRecording() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "cancelled", + endReason: "recording_controls_cancelled" + ) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.metadataURL.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["requireSkillDraft"] as? Bool, true) + XCTAssertEqual(validation["skillDraftReady"] as? Bool, false) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording was cancelled; do not create or update a skill from this event stream")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording was cancelled; do not create or update a skill from this event stream")) + } + + func testEventStreamRecordingValidationRequireSkillDraftRejectsIncompleteRecording() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "recording", + endReason: "", + includeSessionEnded: false + ) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.metadataURL.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["requireSkillDraft"] as? Bool, true) + XCTAssertEqual(validation["skillDraftReady"] as? Bool, false) + XCTAssertEqual(validation["recordingIncomplete"] as? Bool, true) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording is not complete; stop the recording before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording is not complete; stop the recording before creating a skill")) + } + + func testEventStreamRecordingValidationRequireSkillDraftRejectsBlockingDiagnostics() throws { + let fixture = try makeEventStreamSummaryFixture(includeBlockingDiagnostic: true) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.metadataURL.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["requireSkillDraft"] as? Bool, true) + XCTAssertEqual(validation["skillDraftReady"] as? Bool, false) + let blockingDiagnostics = try XCTUnwrap(validation["blockingDiagnostics"] as? [[String: Any]]) + XCTAssertEqual(blockingDiagnostics.count, 1) + XCTAssertEqual(blockingDiagnostics.first?["reason"] as? String, "inputMonitorsUnavailable") + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill")) + } + + func testEventStreamRecordingValidationReportsMissingRequiredEventType() throws { + let fixture = try makeEventStreamSummaryFixture() + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: false, + requiredEventTypes: ["keyboard.shortcut"], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("missing required event type: keyboard.shortcut")) + } + + func testEventStreamRecordingValidationAcceptsOfficialStyleSessionHandoffAlias() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["startedAt"] = "2026-06-26T00:00:00.000Z" + metadata["endedAt"] = "2026-06-26T00:00:05.000Z" + try metadata.writeJSONObject(to: fixture.metadataURL) + let sessionAlias: [String: Any] = [ + "id": metadata["sessionId"] as? String ?? "summary-fixture", + "startedAt": "2026-06-26T00:00:00.000Z", + "endedAt": "2026-06-26T00:00:05.000Z", + "endReason": "recording_controls_stopped", + "eventsPath": metadata["eventsPath"] as? String ?? fixture.eventsURL.path, + ] + try sessionAlias.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertFalse(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, true) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertFalse(errors.contains("metadata.json and session.json differ")) + } + + func testEventStreamRecordingValidationReportsEventCountMismatch() throws { + let fixture = try makeEventStreamSummaryFixture() + let metadata = try jsonObject(contentsOf: fixture.metadataURL) + var mutated = metadata + mutated["eventCount"] = 999 + try mutated.writeJSONObject(to: fixture.metadataURL) + try mutated.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: [], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("eventCount=999 does not match events.jsonl lines=6")) + } + + func testEventStreamRecordingValidationRequiresCurrentSegmentPathsWhileRecording() throws { + let fixture = try makeEventStreamSummaryFixture( + state: "recording", + endReason: "", + includeSessionEnded: false + ) + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata.removeValue(forKey: "currentSegmentEventsPath") + metadata.removeValue(forKey: "currentSegmentMetadataPath") + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: [], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording state requires currentSegmentEventsPath")) + XCTAssertTrue(errors.contains("recording state requires currentSegmentMetadataPath")) + } + + func testEventStreamRecordingValidationRejectsCurrentSegmentPathsAfterFinalState() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["currentSegmentEventsPath"] = fixture.eventsURL.path + metadata["currentSegmentMetadataPath"] = fixture.metadataURL.path + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: [], + requireSkillDraft: false + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("final state must not include currentSegmentEventsPath")) + XCTAssertTrue(errors.contains("final state must not include currentSegmentMetadataPath")) + } + + func testEventStreamRecordingValidationRejectsSessionEndedBeforeFinalEvent() throws { + let fixture = try makeEventStreamSummaryFixture() + let trailingEvent: [String: Any] = [ + "type": "window.changed", + "timestamp": "2026-06-26T00:00:06.000Z", + "sessionId": "summary-fixture", + "appName": "Fixture", + "windowTitle": "Late Event", + ] + let trailingData = try JSONSerialization.data( + withJSONObject: trailingEvent, + options: [.withoutEscapingSlashes] + ) + let trailingLine = String(data: trailingData, encoding: .utf8) ?? "{}" + let currentEvents = try String(contentsOf: fixture.eventsURL, encoding: .utf8) + try (currentEvents + trailingLine + "\n").write(to: fixture.eventsURL, atomically: true, encoding: .utf8) + + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["eventCount"] = 7 + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["finalEventType"] as? String, "window.changed") + XCTAssertEqual(validation["sessionEndedIsFinal"] as? Bool, false) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording has events after session.ended; stop or cancel must be the final event before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("session.ended is not the final event")) + XCTAssertTrue(errors.contains("recording has events after session.ended; stop or cancel must be the final event before creating a skill")) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["finalEventType"] as? String, "window.changed") + XCTAssertEqual(summary["sessionEndedIsFinal"] as? Bool, false) + let evidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(evidence["sessionEndedNotFinal"] as? Bool, true) + let readiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(readiness["status"] as? String, "insufficient") + XCTAssertEqual(readiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingValidationRejectsMultipleSessionEndedEvents() throws { + let fixture = try makeEventStreamSummaryFixture() + let duplicateEndEvent: [String: Any] = [ + "type": "session.ended", + "timestamp": "2026-06-26T00:00:06.000Z", + "sessionId": "summary-fixture", + "endReason": "recording_controls_stopped", + ] + let duplicateEndData = try JSONSerialization.data( + withJSONObject: duplicateEndEvent, + options: [.withoutEscapingSlashes] + ) + let duplicateEndLine = String(data: duplicateEndData, encoding: .utf8) ?? "{}" + let currentEvents = try String(contentsOf: fixture.eventsURL, encoding: .utf8) + try (currentEvents + duplicateEndLine + "\n").write(to: fixture.eventsURL, atomically: true, encoding: .utf8) + + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["eventCount"] = 7 + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["sessionEndedCount"] as? Int, 2) + XCTAssertEqual(validation["sessionEndedCountInvalid"] as? Bool, true) + XCTAssertEqual(validation["sessionEndedIsFinal"] as? Bool, true) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording has multiple session.ended events; stop or cancel must close the event stream exactly once")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has multiple session.ended events")) + XCTAssertTrue(errors.contains("recording has multiple session.ended events; stop or cancel must close the event stream exactly once")) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["sessionEndedCount"] as? Int, 2) + let evidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(evidence["sessionEndedCountInvalid"] as? Bool, true) + let readiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(readiness["status"] as? String, "insufficient") + XCTAssertEqual(readiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingValidationRejectsMissingSessionStartedEvent() throws { + let fixture = try makeEventStreamSummaryFixture() + var events = try jsonObjects(contentsOf: fixture.eventsURL) + events.removeAll { $0["type"] as? String == "session.started" } + try writeEventStreamFixtureEvents(events, fixture: fixture) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, false) + XCTAssertEqual(validation["sessionStartedCount"] as? Int, 0) + XCTAssertEqual(validation["sessionStartedCountInvalid"] as? Bool, true) + XCTAssertEqual(validation["firstEventType"] as? String, "window.changed") + XCTAssertEqual(validation["sessionStartedIsFirst"] as? Bool, false) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording must contain exactly one session.started event before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has no session.started event")) + XCTAssertTrue(errors.contains("recording must contain exactly one session.started event before creating a skill")) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["sessionStartedCount"] as? Int, 0) + let evidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(evidence["sessionStartedCountInvalid"] as? Bool, true) + let readiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(readiness["status"] as? String, "insufficient") + XCTAssertEqual(readiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingValidationRejectsMultipleSessionStartedEvents() throws { + let fixture = try makeEventStreamSummaryFixture() + var events = try jsonObjects(contentsOf: fixture.eventsURL) + let duplicateStartEvent: [String: Any] = [ + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.500Z", + "sessionId": "summary-fixture", + ] + events.insert(duplicateStartEvent, at: 1) + try writeEventStreamFixtureEvents(events, fixture: fixture) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["sessionStartedCount"] as? Int, 2) + XCTAssertEqual(validation["sessionStartedCountInvalid"] as? Bool, true) + XCTAssertEqual(validation["sessionStartedIsFirst"] as? Bool, true) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording must contain exactly one session.started event before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("recording has multiple session.started events")) + } + + func testEventStreamRecordingValidationRejectsSessionStartedAfterFirstEvent() throws { + let fixture = try makeEventStreamSummaryFixture() + var events = try jsonObjects(contentsOf: fixture.eventsURL) + let first = events.removeFirst() + events.insert(first, at: 1) + try writeEventStreamFixtureEvents(events, fixture: fixture) + + let output = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(output.jsonObject as? [String: Any]) + + XCTAssertTrue(output.hasToolError) + XCTAssertEqual(validation["firstEventType"] as? String, "window.changed") + XCTAssertEqual(validation["sessionStartedCount"] as? Int, 1) + XCTAssertEqual(validation["sessionStartedCountInvalid"] as? Bool, false) + XCTAssertEqual(validation["sessionStartedIsFirst"] as? Bool, false) + let reasons = try XCTUnwrap(validation["skillDraftReasons"] as? [String]) + XCTAssertTrue(reasons.contains("recording has events before session.started; start must be the first event before creating a skill")) + let errors = try XCTUnwrap(validation["errors"] as? [String]) + XCTAssertTrue(errors.contains("session.started is not the first event")) + XCTAssertTrue(errors.contains("recording has events before session.started; start must be the first event before creating a skill")) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, false) + XCTAssertEqual(summary["firstEventType"] as? String, "window.changed") + XCTAssertEqual(summary["sessionStartedIsFirst"] as? Bool, false) + let evidence = try XCTUnwrap(summary["skillEvidence"] as? [String: Any]) + XCTAssertEqual(evidence["sessionStartedNotFirst"] as? Bool, true) + let readiness = try XCTUnwrap(summary["skillReadiness"] as? [String: Any]) + XCTAssertEqual(readiness["status"] as? String, "insufficient") + XCTAssertEqual(readiness["canCreateSkillDraft"] as? Bool, false) + } + + func testEventStreamRecordingToolsAcceptRelativeFixturePaths() throws { + let fixture = try makeEventStreamSummaryFixture() + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["metadataPath"] = "metadata.json" + metadata["sessionPath"] = "session.json" + metadata["eventsPath"] = "events.jsonl" + metadata["suppressedEventsPath"] = "suppressed.jsonl" + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) + + let validationOutput = try runOpenComputerUseEventStream(.validate( + inputPath: fixture.sessionDirectory.path, + strictOCU: true, + requiredEventTypes: ["mouse.click"], + requireSkillDraft: true + )) + let validation = try XCTUnwrap(validationOutput.jsonObject as? [String: Any]) + XCTAssertFalse(validationOutput.hasToolError) + XCTAssertEqual(validation["ok"] as? Bool, true) + + let summary = try summarizeEventStreamRecording(options: EventStreamRecordingSummaryOptions( + inputPath: fixture.metadataURL.path, + includeText: false, + requireAction: true + )) + XCTAssertEqual(summary["ok"] as? Bool, true) + XCTAssertEqual(summary["actionEventCount"] as? Int, 3) + } + + func testEventStreamServiceStopsAtMaximumDuration() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + maximumDuration: 0.2 + ) + let started = try service.start() + let sessionID = try XCTUnwrap(started.sessionID) + + let stopped = service.wait(sessionID: sessionID, timeout: 2) + + XCTAssertEqual(stopped.sessionID, sessionID) + XCTAssertEqual(stopped.state, .stopped) + XCTAssertEqual(stopped.endReason, eventStreamMaximumDurationEndReason) + + let eventsPath = try XCTUnwrap(stopped.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""type":"session.ended""#)) + XCTAssertTrue(events.contains(#""kind":"session.ended""#)) + XCTAssertTrue(events.contains(#""endReason":"\#(eventStreamMaximumDurationEndReason)""#)) + } + + func testEventStreamSuppressedEventsUpdateMetadata() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + + _ = try service.start() + try service.appendSuppressedEventForTesting([ + "type": "AX.snapshot.suppressed", + "reason": "test", + ]) + let status = service.status() + + XCTAssertEqual(status.suppressedEventCount, 1) + let suppressedPath = try XCTUnwrap(status.suppressedEventsPath) + let suppressed = try String(contentsOfFile: suppressedPath, encoding: .utf8) + XCTAssertTrue(suppressed.contains(#""type":"AX.snapshot.suppressed""#)) + XCTAssertTrue(suppressed.contains(#""kind":"AX.snapshot.suppressed""#)) + XCTAssertTrue(suppressed.contains(#""reason":"test""#)) + _ = try service.stop() + } + + func testEventStreamDebugErrorPayloadUsesOfficialEventNameAndSafeContext() throws { + let payload = eventStreamDebugErrorPayload( + subsystem: "accessibility", + reason: "snapshotUnavailable", + error: NSError(domain: "OpenComputerUseTests", code: 7), + context: [ + "path": "private-tmp-recording.png", + "app": [ + "name": "Terminal", + "pid": 123, + "bundleIdentifier": "com.apple.Terminal", + ], + ], + timestamp: "2026-06-26T00:00:00.000Z" + ) + + XCTAssertEqual(payload["type"] as? String, "debug.error") + XCTAssertEqual(payload["timestamp"] as? String, "2026-06-26T00:00:00.000Z") + XCTAssertEqual(payload["subsystem"] as? String, "accessibility") + XCTAssertEqual(payload["reason"] as? String, "snapshotUnavailable") + XCTAssertEqual(payload["errorType"] as? String, "NSError") + XCTAssertNil(payload["path"]) + let app = try XCTUnwrap(payload["app"] as? [String: Any]) + XCTAssertEqual(app["name"] as? String, "Terminal") + XCTAssertEqual(app["pid"] as? Int, 123) + XCTAssertEqual(app["bundleIdentifier"] as? String, "com.apple.Terminal") + } + + func testEventStreamDebugErrorsAppendToMainEvents() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + + _ = try service.start() + try service.appendDebugErrorForTesting( + subsystem: "accessibility", + reason: "snapshotUnavailable", + error: NSError(domain: "OpenComputerUseTests", code: 7) + ) + let status = service.status() + + let eventsPath = try XCTUnwrap(status.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""type":"debug.error""#)) + XCTAssertTrue(events.contains(#""reason":"snapshotUnavailable""#)) + XCTAssertTrue(events.contains(#""errorType":"NSError""#)) + XCTAssertEqual(status.eventCount, 4) + _ = try service.stop() + } + + func testEventStreamRecordsDebugErrorWhenInputMonitorsAreUnavailable() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: true, + showsRecordingControls: false, + recordsRawEvents: false, + installsCGEventTap: false, + inputMonitorInstaller: { _, _ in nil } + ) + + let status = try service.start() + + let eventsPath = try XCTUnwrap(status.eventsPath) + let events = try String(contentsOfFile: eventsPath, encoding: .utf8) + XCTAssertTrue(events.contains(#""type":"debug.error""#)) + XCTAssertTrue(events.contains(#""subsystem":"inputMonitoring""#)) + XCTAssertTrue(events.contains(#""reason":"inputMonitorsUnavailable""#)) + XCTAssertTrue(events.contains(#""monitorsInstalled":0"#)) + _ = try service.stop() + } + + func testEventStreamCompactAXDiffUsesOfficialStyleMarkers() throws { + let diff = try XCTUnwrap(eventStreamCompactAXDiff( + previous: [ + "0 window \"A\"", + "1 button \"Save\"", + "2 text \"Old\"", + ], + current: [ + "0 window \"A\"", + "1 button \"Save\"", + "2 text \"New\"", + "3 button \"Done\"", + ] + )) + + XCTAssertEqual(diff.lines, [ + "~ 2 text \"Old\" -> 2 text \"New\"", + "+ 3 button \"Done\"", + ]) + XCTAssertTrue(diff.renderedText.contains("The following is a diff from the previous accessibility tree")) + XCTAssertTrue(diff.cumulativeRenderedText.contains("The following is a cumulative diff from the initial accessibility tree")) + } + + func testEventStreamCompactAXDiffRecordsRemovedLines() throws { + let diff = try XCTUnwrap(eventStreamCompactAXDiff( + previous: [ + "0 window \"A\"", + "1 button \"Save\"", + "2 text \"Transient\"", + "3 button \"Done\"", + ], + current: [ + "0 window \"A\"", + "1 button \"Save\"", + "3 button \"Done\"", + ] + )) + + XCTAssertEqual(diff.lines, [ + "- 2 text \"Transient\"", + ]) + } + + func testEventStreamCompactAXDiffRecordsInsertedLinesWithoutIndexShiftNoise() throws { + let diff = try XCTUnwrap(eventStreamCompactAXDiff( + previous: [ + "0 window \"A\"", + "1 button \"Save\"", + "2 button \"Done\"", + ], + current: [ + "0 window \"A\"", + "1 button \"New\"", + "2 button \"Save\"", + "3 button \"Done\"", + ] + )) + + XCTAssertEqual(diff.lines, [ + "+ 1 button \"New\"", + ]) + } + + func testEventStreamCompactAXDiffPairsChangedAXValuesByElementIdentity() throws { + let diff = try XCTUnwrap(eventStreamCompactAXDiff( + previous: [ + "0 standard window Notes", + "1 search text field (settable, string) Alpha", + ], + current: [ + "0 standard window Notes", + "1 search text field (settable, string) Beta", + ] + )) + + XCTAssertEqual(diff.lines, [ + "~ 1 search text field (settable, string) Alpha -> 1 search text field (settable, string) Beta", + ]) + } + + func testEventStreamCompactAXDiffFallsBackWhenOverBudget() { + let diff = eventStreamCompactAXDiff( + previous: ["a", "b", "c"], + current: ["x", "y", "z"], + maxDiffLines: 1 + ) + + XCTAssertNil(diff) + } + + func testEventStreamAXContextKeyPrefersBundleIdentifierAndWindowTitle() { + XCTAssertEqual( + eventStreamAXContextKey( + appName: "TextEdit", + bundleIdentifier: "com.apple.TextEdit", + windowTitle: "Notes" + ), + "com.apple.TextEdit\u{1f}Notes" + ) + XCTAssertEqual( + eventStreamAXContextKey( + appName: "TextEdit", + bundleIdentifier: nil, + windowTitle: nil + ), + "TextEdit\u{1f}" + ) + } + + func testEventStreamWindowChangedPayloadUsesOfficialEventName() throws { + let payload = eventStreamWindowChangedPayload( + reason: "inputContextChanged", + contextKey: "com.apple.TextEdit\u{1f}Notes", + context: [ + "app": [ + "name": "TextEdit", + "bundleIdentifier": "com.apple.TextEdit", + "pid": 123, + ], + "window": [ + "title": "Notes", + "frame": [ + "x": 10, + "y": 20, + "width": 300, + "height": 200, + ], + ], + ], + timestamp: "2026-06-26T00:00:00.000Z" + ) + + XCTAssertEqual(payload["type"] as? String, "window.changed") + XCTAssertEqual(payload["timestamp"] as? String, "2026-06-26T00:00:00.000Z") + XCTAssertEqual(payload["reason"] as? String, "inputContextChanged") + XCTAssertEqual(payload["windowContextKey"] as? String, "com.apple.TextEdit\u{1f}Notes") + let app = try XCTUnwrap(payload["app"] as? [String: Any]) + XCTAssertEqual(app["bundleIdentifier"] as? String, "com.apple.TextEdit") + let window = try XCTUnwrap(payload["window"] as? [String: Any]) + XCTAssertEqual(window["title"] as? String, "Notes") + } + + func testEventStreamSelectionSignatureRequiresSelectedText() { + XCTAssertNil(eventStreamSelectionSignature(focusedElement: [ + "role": "AXTextArea", + "title": "Editor", + ])) + + let signature = eventStreamSelectionSignature(focusedElement: [ + "role": "AXTextArea", + "subrole": "AXStandardTextArea", + "title": "Editor", + "label": "Body", + "value": "hello world", + "selectedText": "hello", + "frame": [ + "x": 10, + "y": 20, + "width": 300, + "height": 40, + ], + ]) + + XCTAssertEqual( + signature, + [ + "AXTextArea", + "AXStandardTextArea", + "Editor", + "Body", + "hello world", + "hello", + "10.000,20.000,300.000,40.000", + ].joined(separator: "\u{1f}") + ) + XCTAssertNotEqual( + signature, + eventStreamSelectionSignature(focusedElement: [ + "role": "AXTextArea", + "title": "Editor", + "selectedText": "world", + ]) + ) + } + + func testEventStreamSelectionClearedPayloadKeepsOfficialEventName() throws { + let focusedElement: [String: Any] = [ + "role": "AXTextArea", + "subrole": "AXStandardTextArea", + "title": "Editor", + "label": "Body", + "value": "hello world", + "frame": [ + "x": 10, + "y": 20, + "width": 300, + "height": 40, + ], + ] + let signature = try XCTUnwrap(eventStreamSelectionClearedSignature(focusedElement: focusedElement)) + XCTAssertTrue(signature.hasSuffix("")) + + let payload = eventStreamSelectionChangedPayload( + reason: "keyboard.text_input", + selectedText: "", + selectionSignature: signature, + focusedElement: focusedElement, + selectionCleared: true, + timestamp: "2026-06-26T00:00:00.000Z" + ) + + XCTAssertEqual(payload["type"] as? String, "selection.changed") + XCTAssertEqual(payload["timestamp"] as? String, "2026-06-26T00:00:00.000Z") + XCTAssertEqual(payload["reason"] as? String, "keyboard.text_input") + XCTAssertEqual(payload["selectedText"] as? String, "") + XCTAssertEqual(payload["selectionCleared"] as? Bool, true) + XCTAssertEqual(payload["selectionSignature"] as? String, signature) + let payloadFocusedElement = try XCTUnwrap(payload["focusedAccessibilityElement"] as? [String: Any]) + XCTAssertEqual(payloadFocusedElement["title"] as? String, "Editor") + } + + func testEventStreamFocusedElementIsSecureUsesProtectedSignals() { + XCTAssertTrue(eventStreamFocusedElementIsSecure([ + "role": "AXTextField", + "protectedContent": true, + ])) + XCTAssertTrue(eventStreamFocusedElementIsSecure([ + "role": "AXTextField", + "subrole": "AXSecureTextField", + ])) + XCTAssertTrue(eventStreamFocusedElementIsSecure([ + "role": "AXTextField", + "roleDescription": "secure text field", + ])) + XCTAssertFalse(eventStreamFocusedElementIsSecure([ + "role": "AXTextField", + "label": "Password hint", + ])) + } + + func testEventStreamFocusedElementIsTerminalUsesAppContextAndTextRole() { + let terminalContext: [String: Any] = [ + "app": [ + "name": "Terminal", + "bundleIdentifier": "com.apple.Terminal", + ], + ] + XCTAssertTrue(eventStreamFocusedElementIsTerminal([ + "role": "AXTextArea", + "value": "prompt", + ], appContext: terminalContext)) + XCTAssertTrue(eventStreamFocusedElementIsTerminal([ + "role": "AXGroup", + "roleDescription": "terminal screen", + "value": "prompt", + ], appContext: terminalContext)) + XCTAssertFalse(eventStreamFocusedElementIsTerminal([ + "role": "AXTextArea", + "value": "body", + ], appContext: [ + "app": [ + "name": "TextEdit", + "bundleIdentifier": "com.apple.TextEdit", + ], + ])) + XCTAssertFalse(eventStreamFocusedElementIsTerminal([ + "role": "AXTextArea", + "secureInput": true, + "value": "secret", + ], appContext: terminalContext)) + } + + func testEventStreamTerminalValueHelpersRedactContent() { + let focusedElement: [String: Any] = [ + "role": "AXTextArea", + "title": "Shell", + "value": "terminal output", + "selectedText": "output", + "frame": [ + "x": 1, + "y": 2, + "width": 3, + "height": 4, + ], + ] + + XCTAssertEqual(eventStreamTerminalValue(focusedElement), "terminal output") + XCTAssertEqual(eventStreamStableTextHash("hello"), "a430d84680aabd0b") + XCTAssertEqual(eventStreamStableTextHash("terminal output"), "235656b19743263e") + + let signature = eventStreamTerminalValueSignature( + focusedElement: focusedElement, + valueHash: eventStreamStableTextHash("terminal output") + ) + XCTAssertTrue(signature.hasSuffix("235656b19743263e")) + + let redacted = eventStreamTerminalRedactedFocusedElement(focusedElement) + XCTAssertNil(redacted["value"]) + XCTAssertNil(redacted["selectedText"]) + XCTAssertEqual(redacted["valueHash"] as? String, "235656b19743263e") + XCTAssertEqual(redacted["valueLength"] as? Int, 15) + XCTAssertEqual(redacted["selectedTextLength"] as? Int, 6) + XCTAssertEqual(redacted["terminalValueRedacted"] as? Bool, true) + } + + func testEventStreamMouseEventTypeUsesOfficialContextMenuName() { + XCTAssertEqual(eventStreamMouseEventType(forButton: "left"), "mouse.click") + XCTAssertEqual(eventStreamMouseEventType(forButton: "middle"), "mouse.click") + XCTAssertEqual(eventStreamMouseEventType(forButton: "right"), "mouse.context_menu") + } + + func testEventStreamMouseButtonMapsDownDragAndUpEvents() { + XCTAssertEqual(eventStreamMouseButton(forEventType: .leftMouseDown), "left") + XCTAssertEqual(eventStreamMouseButton(forEventType: .leftMouseDragged), "left") + XCTAssertEqual(eventStreamMouseButton(forEventType: .leftMouseUp), "left") + XCTAssertEqual(eventStreamMouseButton(forEventType: .rightMouseDown), "right") + XCTAssertEqual(eventStreamMouseButton(forEventType: .rightMouseDragged), "right") + XCTAssertEqual(eventStreamMouseButton(forEventType: .rightMouseUp), "right") + XCTAssertEqual(eventStreamMouseButton(forEventType: .otherMouseDown), "middle") + XCTAssertEqual(eventStreamMouseButton(forEventType: .otherMouseDragged), "middle") + XCTAssertEqual(eventStreamMouseButton(forEventType: .otherMouseUp), "middle") + XCTAssertNil(eventStreamMouseButton(forEventType: .keyDown)) + } + + func testEventStreamPointerDidDragUsesDistanceThreshold() { + XCTAssertFalse(eventStreamPointerDidDrag( + start: CGPoint(x: 10, y: 10), + end: CGPoint(x: 12, y: 10), + observedDrag: false + )) + XCTAssertTrue(eventStreamPointerDidDrag( + start: CGPoint(x: 10, y: 10), + end: CGPoint(x: 13, y: 10), + observedDrag: false + )) + XCTAssertTrue(eventStreamPointerDidDrag( + start: CGPoint(x: 10, y: 10), + end: CGPoint(x: 11, y: 10), + observedDrag: true + )) + XCTAssertEqual(eventStreamPointerDistance(CGPoint(x: 0, y: 0), CGPoint(x: 3, y: 4)), 5) + } + + func testEventStreamKeyboardPayloadKindUsesOfficialEventNames() throws { + XCTAssertEqual( + eventStreamKeyboardPayloadKind(characters: "a", keyCode: 0, modifierFlags: []), + EventStreamKeyboardPayloadKind(type: "keyboard.text_input", text: "a", key: nil) + ) + XCTAssertEqual( + eventStreamKeyboardPayloadKind(characters: "\r", keyCode: 36, modifierFlags: []), + EventStreamKeyboardPayloadKind(type: "keyboard.submit", text: nil, key: "Return") + ) + XCTAssertEqual( + eventStreamKeyboardPayloadKind(characters: "s", keyCode: 1, modifierFlags: [.command]), + EventStreamKeyboardPayloadKind(type: "keyboard.shortcut", text: nil, key: "S") + ) + XCTAssertNil(eventStreamKeyboardPayloadKind(characters: nil, keyCode: 0, modifierFlags: [])) + } + + func testEventStreamModifierNamesUseStableOrder() { + XCTAssertEqual(eventStreamModifierNames([.shift, .command, .control, .option]), [ + "command", + "option", + "control", + "shift", + ]) + } + + func testEventStreamRecordingControlsEnvFlag() { + XCTAssertTrue(eventStreamRecordingControlsEnabled(environment: [:])) + XCTAssertFalse(eventStreamRecordingControlsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0"])) + XCTAssertFalse(eventStreamRecordingControlsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "false"])) + XCTAssertTrue(eventStreamRecordingControlsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "true"])) + } + + func testEventStreamRecordingControlsFrameClampsToVisibleFrame() { + let frame = eventStreamRecordingControlsFrame( + panelSize: NSSize(width: 520, height: 54), + visibleFrame: NSRect(x: 0, y: 40, width: 1440, height: 860) + ) + + XCTAssertEqual(frame.origin.x, 460) + XCTAssertEqual(frame.origin.y, 828) + XCTAssertEqual(frame.size.width, 520) + XCTAssertEqual(frame.size.height, 54) + } + + func testEventStreamRecordingControlsFrameSupportsNarrowAndNegativeScreens() { + let narrow = eventStreamRecordingControlsFrame( + panelSize: NSSize(width: 520, height: 54), + visibleFrame: NSRect(x: -1512, y: 458, width: 480, height: 982) + ) + XCTAssertEqual(narrow.origin.x, -1512) + XCTAssertEqual(narrow.origin.y, 1368) + + let short = eventStreamRecordingControlsFrame( + panelSize: NSSize(width: 520, height: 54), + visibleFrame: NSRect(x: 0, y: 10, width: 900, height: 40) + ) + XCTAssertEqual(short.origin.x, 190) + XCTAssertEqual(short.origin.y, 10) + } + + @MainActor + func testEventStreamRecordingControlsButtonsTriggerRuntimeCallbacks() { + var stopCount = 0 + var cancelCount = 0 + let controls = EventStreamRecordingControls( + startedAt: Date(), + onStop: { + stopCount += 1 + }, + onCancel: { + cancelCount += 1 + } + ) + defer { + controls.close() + } + + XCTAssertTrue(controls.performControlButtonForTesting(title: "Done")) + XCTAssertEqual(stopCount, 1) + XCTAssertEqual(cancelCount, 0) + + XCTAssertTrue(controls.performControlButtonForTesting(title: "Discard")) + XCTAssertEqual(stopCount, 1) + XCTAssertEqual(cancelCount, 1) + } + + func testEventStreamScreenshotPolicyEnvFlag() { + XCTAssertEqual(eventStreamScreenshotPolicy(environment: [:]), .auto) + XCTAssertEqual(eventStreamScreenshotPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "always"]), .always) + XCTAssertEqual(eventStreamScreenshotPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never"]), .never) + XCTAssertEqual(eventStreamScreenshotPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "unknown"]), .auto) + } + + func testEventStreamMaximumDurationEnvFlag() { + XCTAssertEqual(eventStreamRecordingMaximumDuration(environment: [:]), eventStreamMaximumRecordingDuration) + XCTAssertEqual(eventStreamRecordingMaximumDuration(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS": "2.5"]), 2.5) + XCTAssertEqual(eventStreamRecordingMaximumDuration(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS": "0"]), 0) + XCTAssertEqual(eventStreamRecordingMaximumDuration(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS": "-1"]), eventStreamMaximumRecordingDuration) + XCTAssertEqual(eventStreamRecordingMaximumDuration(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS": "bogus"]), eventStreamMaximumRecordingDuration) + } + + func testEventStreamRecordingDurationLimitDescription() { + XCTAssertEqual( + eventStreamRecordingDurationLimitDescription(eventStreamMaximumRecordingDuration), + "up to 30 minutes" + ) + XCTAssertEqual(eventStreamRecordingDurationLimitDescription(60), "up to 1 minute") + XCTAssertEqual(eventStreamRecordingDurationLimitDescription(2), "up to 2 seconds") + XCTAssertEqual(eventStreamRecordingDurationLimitDescription(0), "until stopped") + } + + func testEventStreamRawEventsEnvFlagDefaultsToEnabled() { + XCTAssertTrue(eventStreamRawEventsEnabled(environment: [:])) + XCTAssertFalse(eventStreamRawEventsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "0"])) + XCTAssertFalse(eventStreamRawEventsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "false"])) + XCTAssertTrue(eventStreamRawEventsEnabled(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "true"])) + } + + func testEventStreamStartApprovalPolicyEnvFlag() { + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: [:]), .automatic) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "auto"]), .automatic) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "true"]), .interactive) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve"]), .approve) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "0"]), .approve) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "deny"]), .deny) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "cancel"]), .cancel) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "mcp"]), .mcpElicitation) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "elicitation"]), .mcpElicitation) + XCTAssertEqual(eventStreamStartApprovalPolicy(environment: ["OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "unknown"]), .automatic) + } + + func testEventStreamRawEventSummaryCapturesScrollWithoutInventingScrollEventType() throws { + let summary = eventStreamRawEventSummary( + eventType: .scrollWheel, + appKitLocation: CGPoint(x: 10, y: 20), + screenLocation: CGPoint(x: 10, y: 980), + modifierFlags: [.shift], + scrollingDeltaX: 0, + scrollingDeltaY: -32, + hasPreciseScrollingDeltas: true, + phase: [.began, .changed] + ) + + XCTAssertEqual(summary["eventType"] as? String, "scrollWheel") + XCTAssertEqual(summary["scrollingDeltaY"] as? CGFloat, -32) + XCTAssertEqual(summary["hasPreciseScrollingDeltas"] as? Bool, true) + XCTAssertEqual(summary["modifiers"] as? [String], ["shift"]) + XCTAssertEqual(summary["phase"] as? String, "began,changed") + let location = try XCTUnwrap(summary["location"] as? [String: CGFloat]) + XCTAssertEqual(location["y"], 980) + } + + func testEventStreamRawEventSummaryRedactsSecureCharacters() { + let summary = eventStreamRawEventSummary( + eventType: .keyDown, + keyCode: 0, + characters: "secret", + redactCharacters: true + ) + + XCTAssertNil(summary["characters"]) + XCTAssertEqual(summary["charactersRedacted"] as? Bool, true) + XCTAssertEqual(summary["characterCount"] as? Int, 6) + } + + func testEventStreamScreenshotNeededForContextPolicy() { + XCTAssertTrue(eventStreamScreenshotNeededForContext( + policy: .always, + payloadKind: "full", + treeLineCount: 100, + hasFocusedSummary: true + )) + XCTAssertFalse(eventStreamScreenshotNeededForContext( + policy: .never, + payloadKind: "full", + treeLineCount: 1, + hasFocusedSummary: false + )) + XCTAssertTrue(eventStreamScreenshotNeededForContext( + policy: .auto, + payloadKind: "full", + treeLineCount: 4, + hasFocusedSummary: true + )) + XCTAssertTrue(eventStreamScreenshotNeededForContext( + policy: .auto, + payloadKind: "full", + treeLineCount: 40, + hasFocusedSummary: false + )) + XCTAssertFalse(eventStreamScreenshotNeededForContext( + policy: .auto, + payloadKind: "full", + treeLineCount: 40, + hasFocusedSummary: true + )) + } + + func testEventStreamMCPServerListsRecordAndReplayTools() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let server = EventStreamMCPServer(service: service) + + let initialize = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}"#)) + XCTAssertTrue(initialize.contains(#""name":"Record & Replay""#)) + let initializeObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(initialize.utf8)) as? [String: Any]) + let initializeResult = try XCTUnwrap(initializeObject["result"] as? [String: Any]) + XCTAssertEqual(initializeResult["protocolVersion"] as? String, "2025-11-25") + XCTAssertNil(initializeResult["instructions"]) + + let listTools = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}"#)) + XCTAssertTrue(listTools.contains(#""event_stream_start""#)) + XCTAssertTrue(listTools.contains(#""event_stream_status""#)) + XCTAssertTrue(listTools.contains(#""event_stream_stop""#)) + + let start = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}}"#)) + let statusText = try mcpPrimaryText(start) + XCTAssertTrue(statusText.contains(#""state" : "recording""#) || statusText.contains(#""state":"recording""#)) + let statusObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(statusText.utf8)) as? [String: Any]) + let eventsPath = try XCTUnwrap(statusObject["eventsPath"] as? String) + let metadataPath = try XCTUnwrap(statusObject["metadataPath"] as? String) + let sessionPath = try XCTUnwrap(statusObject["sessionPath"] as? String) + let sessionDirectoryPath = try XCTUnwrap(statusObject["sessionDirectoryPath"] as? String) + let sessionID = try XCTUnwrap(statusObject["sessionID"] as? String) + XCTAssertEqual(statusObject["isRecording"] as? Bool, true) + XCTAssertEqual(statusObject["maxDurationSeconds"] as? Int, 1800) + XCTAssertEqual(statusObject["sessionId"] as? String, sessionID) + XCTAssertEqual(metadataPath, sessionPath) + XCTAssertTrue(metadataPath.hasSuffix("/session.json")) + XCTAssertTrue(eventsPath.hasPrefix(sessionDirectoryPath)) + XCTAssertTrue(metadataPath.hasPrefix(sessionDirectoryPath)) + XCTAssertTrue(FileManager.default.fileExists(atPath: eventsPath)) + XCTAssertTrue(FileManager.default.fileExists(atPath: metadataPath)) + XCTAssertTrue(FileManager.default.fileExists(atPath: sessionPath)) + XCTAssertEqual( + try String(contentsOfFile: metadataPath, encoding: .utf8), + try String(contentsOfFile: sessionPath, encoding: .utf8) + ) + + let stop = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"event_stream_stop","arguments":{}}}"#)) + let stopText = try mcpPrimaryText(stop) + let stopObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(stopText.utf8)) as? [String: Any]) + XCTAssertEqual(stopObject["isRecording"] as? Bool, false) + XCTAssertEqual(stopObject["endReason"] as? String, "recording_controls_stopped") + XCTAssertEqual(stopObject["sessionID"] as? String, sessionID) + XCTAssertEqual(stopObject["sessionDirectoryPath"] as? String, sessionDirectoryPath) + XCTAssertEqual(stopObject["metadataPath"] as? String, sessionPath) + XCTAssertEqual(stopObject["sessionPath"] as? String, sessionPath) + XCTAssertNotNil(stopObject["endedAt"]) + } + + func testEventStreamMCPSurfaceMatchesOfficialFixture() throws { + let fixtureURL = repositoryRootURL() + .appendingPathComponent("docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json") + let fixture = try jsonObject(contentsOf: fixtureURL) + let fixtureInitialize = try XCTUnwrap(fixture["initialize"] as? [String: Any]) + let fixtureServerInfo = try XCTUnwrap(fixtureInitialize["serverInfo"] as? [String: Any]) + let fixtureTools = try XCTUnwrap(fixture["tools"] as? [[String: Any]]) + + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let server = EventStreamMCPServer(service: service) + + let initialize = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{}}"#)) + let initializeObject = try jsonObject(text: initialize) + let initializeResult = try XCTUnwrap(initializeObject["result"] as? [String: Any]) + let localServerInfo = try XCTUnwrap(initializeResult["serverInfo"] as? [String: Any]) + + XCTAssertEqual(initializeResult["protocolVersion"] as? String, fixtureInitialize["protocolVersion"] as? String) + XCTAssertEqual(localServerInfo["name"] as? String, fixtureServerInfo["name"] as? String) + XCTAssertNil(initializeResult["instructions"]) + XCTAssertEqual( + try canonicalJSON(initializeResult["capabilities"] as Any), + try canonicalJSON(fixtureInitialize["capabilities"] as Any) + ) + + let listTools = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}}"#)) + let listToolsObject = try jsonObject(text: listTools) + let listToolsResult = try XCTUnwrap(listToolsObject["result"] as? [String: Any]) + let localTools = try XCTUnwrap(listToolsResult["tools"] as? [[String: Any]]) + + XCTAssertEqual(try canonicalJSON(localTools), try canonicalJSON(fixtureTools)) + } + + func testEventStreamMCPStartApprovalDeniedReturnsToolError() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .deny + ) + let server = EventStreamMCPServer(service: service) + + let start = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}}"#)) + + XCTAssertTrue(start.contains(#""isError":true"#)) + XCTAssertTrue(start.contains(eventStreamStartApprovalDeniedMessage)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + } + + func testEventStreamMCPStartApprovalUsesElicitationAccept() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .mcpElicitation + ) + let server = EventStreamMCPServer(service: service) + + let initialize = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"capabilities":{"elicitation":{}}}}"#)) + let initializeObject = try jsonObject(text: initialize) + let initializeResult = try XCTUnwrap(initializeObject["result"] as? [String: Any]) + XCTAssertNil(initializeResult["instructions"]) + + var requestedApproval = false + let start = try XCTUnwrap(server.handle( + line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}}"#, + startApprovalRequester: { + requestedApproval = true + return .approved + } + )) + + XCTAssertTrue(requestedApproval) + let statusText = try mcpPrimaryText(start) + let statusObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(statusText.utf8)) as? [String: Any]) + XCTAssertEqual(statusObject["state"] as? String, "recording") + XCTAssertTrue(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + } + + func testEventStreamMCPStartApprovalElicitationRequestShape() throws { + let request = EventStreamMCPServer.startApprovalElicitationRequest(id: "approval-1") + XCTAssertEqual(request["jsonrpc"] as? String, "2.0") + XCTAssertEqual(request["id"] as? String, "approval-1") + XCTAssertEqual(request["method"] as? String, "elicitation/create") + + let params = try XCTUnwrap(request["params"] as? [String: Any]) + XCTAssertEqual(params["mode"] as? String, "form") + XCTAssertEqual( + params["message"] as? String, + "Open Computer Use wants to start Record & Replay and record your actions." + ) + + let requestedSchema = try XCTUnwrap(params["requestedSchema"] as? [String: Any]) + XCTAssertEqual(requestedSchema["type"] as? String, "object") + XCTAssertNotNil(requestedSchema["properties"] as? [String: Any]) + XCTAssertEqual((requestedSchema["required"] as? [Any])?.count, 0) + XCTAssertNil(requestedSchema["additionalProperties"]) + } + + func testEventStreamMCPStartApprovalUsesElicitationDecline() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .mcpElicitation + ) + let server = EventStreamMCPServer(service: service) + + _ = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"capabilities":{"elicitation":{}}}}"#)) + let start = try XCTUnwrap(server.handle( + line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}}"#, + startApprovalRequester: { .denied } + )) + + XCTAssertTrue(start.contains(#""isError":true"#)) + XCTAssertTrue(start.contains(eventStreamStartApprovalDeniedMessage)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + } + + func testEventStreamMCPStartApprovalCancelsWhenElicitationUnsupported() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService( + rootDirectory: root, + installsInputMonitors: false, + showsRecordingControls: false, + startApprovalPolicy: .mcpElicitation + ) + let server = EventStreamMCPServer(service: service) + + _ = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"capabilities":{}}}"#)) + var requestedApproval = false + let start = try XCTUnwrap(server.handle( + line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}}"#, + startApprovalRequester: { + requestedApproval = true + return .approved + } + )) + + XCTAssertFalse(requestedApproval) + XCTAssertTrue(start.contains(#""isError":true"#)) + XCTAssertTrue(start.contains(eventStreamStartApprovalCancelledMessage)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + } + + func testEventStreamMCPStatusAndStopRemainIdleBeforeStart() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let server = EventStreamMCPServer(service: service) + + let status = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"event_stream_status","arguments":{}}}"#)) + let statusText = try mcpPrimaryText(status) + let statusObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(statusText.utf8)) as? [String: Any]) + XCTAssertEqual(statusObject["isRecording"] as? Bool, false) + XCTAssertEqual(statusObject["maxDurationSeconds"] as? Int, 1800) + XCTAssertNil(statusObject["state"]) + XCTAssertNil(statusObject["active"]) + XCTAssertNil(statusObject["sessionId"]) + XCTAssertNil(statusObject["eventsPath"]) + XCTAssertNil(statusObject["metadataPath"]) + + let stop = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"event_stream_stop","arguments":{}}}"#)) + let stopText = try mcpPrimaryText(stop) + let stopObject = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(stopText.utf8)) as? [String: Any]) + XCTAssertEqual(stopObject["isRecording"] as? Bool, false) + XCTAssertEqual(stopObject["maxDurationSeconds"] as? Int, 1800) + XCTAssertNil(stopObject["state"]) + XCTAssertNil(stopObject["active"]) + XCTAssertNil(stopObject["sessionId"]) + XCTAssertNil(stopObject["eventsPath"]) + XCTAssertNil(stopObject["metadataPath"]) + + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + } + + func testEventStreamMCPToolsRejectArguments() throws { + let root = try makeTemporaryDirectory() + let service = EventStreamService(rootDirectory: root, installsInputMonitors: false, showsRecordingControls: false) + let server = EventStreamMCPServer(service: service) + + let status = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":1,"method":"tools/call","params":{"name":"event_stream_status","arguments":{"unexpected":true}}}"#)) + + XCTAssertTrue(status.contains(#""isError":true"#)) + XCTAssertTrue(status.contains("event_stream_status does not accept arguments")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let startWithNonObjectArguments = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"event_stream_start","arguments":[]}}"#)) + XCTAssertTrue(startWithNonObjectArguments.contains(#""isError":true"#)) + XCTAssertTrue(startWithNonObjectArguments.contains("tools/call arguments must be an object")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let startWithMissingName = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"arguments":{}}}"#)) + XCTAssertTrue(startWithMissingName.contains(#""isError":true"#)) + XCTAssertTrue(startWithMissingName.contains("tools/call params.name must be a non-empty string")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let startWithNonStringName = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":["event_stream_start"],"arguments":{}}}"#)) + XCTAssertTrue(startWithNonStringName.contains(#""isError":true"#)) + XCTAssertTrue(startWithNonStringName.contains("tools/call params.name must be a non-empty string")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let startWithMissingParams = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":5,"method":"tools/call"}"#)) + XCTAssertTrue(startWithMissingParams.contains(#""isError":true"#)) + XCTAssertTrue(startWithMissingParams.contains("tools/call params must be an object")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + + let startWithNonObjectParams = try XCTUnwrap(server.handle(line: #"{"jsonrpc":"2.0","id":6,"method":"tools/call","params":[]}"#)) + XCTAssertTrue(startWithNonObjectParams.contains(#""isError":true"#)) + XCTAssertTrue(startWithNonObjectParams.contains("tools/call params must be an object")) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("latest-session.json").path)) + XCTAssertFalse(FileManager.default.fileExists(atPath: root.appendingPathComponent("active-session.json").path)) + } + func testReadToolArgumentsAcceptsJSONObject() throws { let arguments = try readOpenComputerUseToolArguments( json: #"{"app":"TextEdit","pages":2}"#, @@ -270,6 +3278,12 @@ final class OpenComputerUseKitTests: XCTestCase { func testMacOSAppAgentProxyDecisionRoutesAutomationCommandsThroughAppBundle() { for command in [ OpenComputerUseCLICommand.mcp, + .eventStream(.mcp), + .eventStream(.start(json: true)), + .eventStream(.status(json: true)), + .eventStream(.stop(json: true)), + .eventStream(.cancel(json: true)), + .eventStream(.wait(json: true, sessionID: nil, timeout: 0, notifyCommand: nil)), .doctor, .listApps, .snapshot(app: "TextEdit"), @@ -287,6 +3301,8 @@ final class OpenComputerUseKitTests: XCTestCase { func testMacOSAppAgentProxyDecisionKeepsNonAutomationCommandsLocal() { for command in [ OpenComputerUseCLICommand.turnEnded(payload: nil), + .eventStream(.summarize(inputPath: "/tmp/session", includeText: false, requireAction: false)), + .eventStream(.validate(inputPath: "/tmp/session", strictOCU: false, requiredEventTypes: [], requireSkillDraft: false)), .help(command: nil), .version, ] { @@ -1259,6 +4275,24 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertEqual(point, CGPoint(x: -1311, y: 701)) } + func testEventStreamScreenStatePointConvertsFromAppKitGlobalPoint() { + let point = eventStreamScreenStatePoint( + fromAppKitGlobalPoint: CGPoint(x: -1311, y: 281), + screenMappings: [ + VisualCursorScreenMapping( + screenStateFrame: CGRect(x: 0, y: 0, width: 2560, height: 1440), + appKitFrame: CGRect(x: 0, y: 0, width: 2560, height: 1440) + ), + VisualCursorScreenMapping( + screenStateFrame: CGRect(x: -1512, y: 458, width: 1512, height: 982), + appKitFrame: CGRect(x: -1512, y: 0, width: 1512, height: 982) + ), + ] + ) + + XCTAssertEqual(point, CGPoint(x: -1311, y: 1159)) + } + func testScreenshotPixelScaleUsesRetinaSizedImageAgainstWindowBounds() { let scale = screenshotPixelScale( screenshotPixelSize: CGSize(width: 2048, height: 1266), @@ -1717,7 +4751,12 @@ final class OpenComputerUseKitTests: XCTestCase { XCTAssertGreaterThan(abs(negativePose.angleOffset), 0.08) } - private func makeSnapshot(treeLines: [String], focusedSummary: String?, selectedText: String? = nil) -> AppSnapshot { + private func makeSnapshot( + treeLines: [String], + focusedSummary: String?, + selectedText: String? = nil, + screenshotPNGData: Data? = nil + ) -> AppSnapshot { AppSnapshot( app: RunningAppDescriptor( name: "Sample Chat", @@ -1729,7 +4768,7 @@ final class OpenComputerUseKitTests: XCTestCase { windowBounds: nil, targetWindowID: nil, targetWindowLayer: nil, - screenshotPNGData: nil, + screenshotPNGData: screenshotPNGData, mode: .accessibility, treeLines: treeLines, focusedSummary: focusedSummary, @@ -1843,3 +4882,196 @@ final class OpenComputerUseKitTests: XCTestCase { return (width, height) } } + +private func makeTemporaryDirectory() throws -> URL { + let url = FileManager.default.temporaryDirectory + .appendingPathComponent("open-computer-use-tests", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + return url +} + +private struct EventStreamSummaryFixture { + let sessionDirectory: URL + let metadataURL: URL + let eventsURL: URL +} + +private func makeEventStreamSummaryFixture( + state: String = "stopped", + endReason: String = "recording_controls_stopped", + includeBlockingDiagnostic: Bool = false, + includeSessionEnded: Bool = true +) throws -> EventStreamSummaryFixture { + let root = try makeTemporaryDirectory() + let eventsURL = root.appendingPathComponent("events.jsonl") + let metadataURL = root.appendingPathComponent("metadata.json") + let suppressedURL = root.appendingPathComponent("suppressed.jsonl") + let sessionURL = root.appendingPathComponent("session.json") + + var events: [[String: Any]] = [ + [ + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.000Z", + "sessionId": "summary-fixture", + ], + [ + "type": "window.changed", + "timestamp": "2026-06-26T00:00:01.000Z", + "sessionId": "summary-fixture", + "appName": "Fixture", + "bundleIdentifier": "dev.opencomputeruse.fixture", + "windowTitle": "Demo", + ], + [ + "type": "mouse.click", + "timestamp": "2026-06-26T00:00:02.000Z", + "sessionId": "summary-fixture", + "location": ["x": 10, "y": 20], + "targetAccessibilityElement": [ + "role": "AXButton", + "title": "Save", + "value": "secret-value", + "selectedText": "selected-secret", + "actions": ["AXPress"], + ], + ], + [ + "type": "keyboard.text_input", + "timestamp": "2026-06-26T00:00:03.000Z", + "sessionId": "summary-fixture", + "textLength": 6, + "secureInput": true, + "focusedAccessibilityElement": [ + "role": "AXTextField", + "label": "Token", + "secureInput": true, + "value": "token-secret", + ], + ], + [ + "type": "selection.changed", + "timestamp": "2026-06-26T00:00:04.000Z", + "sessionId": "summary-fixture", + "selectedText": "selection-event-secret", + "focusedAccessibilityElement": [ + "role": "AXTextArea", + "selectedText": "selected-secret", + ], + ], + ] + if includeBlockingDiagnostic { + events.append([ + "type": "debug.error", + "timestamp": "2026-06-26T00:00:04.750Z", + "sessionId": "summary-fixture", + "subsystem": "inputMonitoring", + "reason": "inputMonitorsUnavailable", + "errorType": "permission", + ]) + } + if includeSessionEnded { + events.append([ + "type": "session.ended", + "timestamp": "2026-06-26T00:00:05.000Z", + "sessionId": "summary-fixture", + "endReason": endReason, + ]) + } + + let lines = try events.map { event -> String in + let data = try JSONSerialization.data(withJSONObject: event, options: [.withoutEscapingSlashes]) + return String(data: data, encoding: .utf8) ?? "{}" + }.joined(separator: "\n") + try (lines + "\n").write(to: eventsURL, atomically: true, encoding: .utf8) + try "".write(to: suppressedURL, atomically: true, encoding: .utf8) + + var metadata: [String: Any] = [ + "sessionId": "summary-fixture", + "state": state, + "active": state == "recording", + "endReason": endReason, + "eventCount": events.count, + "suppressedEventCount": 0, + "eventsPath": eventsURL.path, + "metadataPath": metadataURL.path, + "sessionPath": sessionURL.path, + "suppressedEventsPath": suppressedURL.path, + ] + if state == "recording" { + metadata["currentSegmentEventsPath"] = eventsURL.path + metadata["currentSegmentMetadataPath"] = metadataURL.path + } + try metadata.writeJSONObject(to: metadataURL) + try metadata.writeJSONObject(to: sessionURL) + return EventStreamSummaryFixture(sessionDirectory: root, metadataURL: metadataURL, eventsURL: eventsURL) +} + +private extension Dictionary where Key == String, Value == Any { + func writeJSONObject(to url: URL) throws { + let data = try JSONSerialization.data( + withJSONObject: self, + options: [.prettyPrinted, .sortedKeys, .withoutEscapingSlashes] + ) + try data.write(to: url, options: [.atomic]) + } +} + +private func repositoryRootURL(filePath: String = #filePath) -> URL { + URL(fileURLWithPath: filePath) + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() + .deletingLastPathComponent() +} + +private func jsonObject(contentsOf url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + return try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) +} + +private func jsonObjects(contentsOf url: URL) throws -> [[String: Any]] { + let text = try String(contentsOf: url, encoding: .utf8) + return try text + .split(separator: "\n") + .map { line in + let data = Data(line.utf8) + return try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any]) + } +} + +private func writeEventStreamFixtureEvents( + _ events: [[String: Any]], + fixture: EventStreamSummaryFixture +) throws { + let lines = try events.map { event -> String in + let data = try JSONSerialization.data(withJSONObject: event, options: [.withoutEscapingSlashes]) + return String(data: data, encoding: .utf8) ?? "{}" + }.joined(separator: "\n") + try (lines + "\n").write(to: fixture.eventsURL, atomically: true, encoding: .utf8) + + var metadata = try jsonObject(contentsOf: fixture.metadataURL) + metadata["eventCount"] = events.count + try metadata.writeJSONObject(to: fixture.metadataURL) + try metadata.writeJSONObject(to: fixture.sessionDirectory.appendingPathComponent("session.json")) +} + +private func jsonObject(text: String) throws -> [String: Any] { + try XCTUnwrap(JSONSerialization.jsonObject(with: Data(text.utf8)) as? [String: Any]) +} + +private func canonicalJSON(_ object: Any) throws -> String { + let data = try JSONSerialization.data( + withJSONObject: object, + options: [.sortedKeys, .withoutEscapingSlashes] + ) + return try XCTUnwrap(String(data: data, encoding: .utf8)) +} + +private func mcpPrimaryText(_ response: String) throws -> String { + let object = try XCTUnwrap(JSONSerialization.jsonObject(with: Data(response.utf8)) as? [String: Any]) + let result = try XCTUnwrap(object["result"] as? [String: Any]) + let content = try XCTUnwrap(result["content"] as? [[String: Any]]) + return try XCTUnwrap(content.first?["text"] as? String) +} diff --git a/scripts/build-record-and-replay-baseline-summary.py b/scripts/build-record-and-replay-baseline-summary.py new file mode 100755 index 00000000..714633ac --- /dev/null +++ b/scripts/build-record-and-replay-baseline-summary.py @@ -0,0 +1,1266 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import sys + +from record_and_replay_baseline_contract import REQUIRED_BASELINE_CHECKS + + +EXPECTED_NO_ACTIVE_RESPONSE = {"isRecording": False, "maxDurationSeconds": 1800} +REQUIRED_MATRIX_MODES = { + "no-active", + "timeout", + "wait-timeout", + "approval", + "mcp-elicitation", + "app-agent-wait", +} +REQUIRED_ACTION_EVENT_TYPES = { + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.click", + "session.ended", +} +REQUIRED_ACTION_SCENARIO_EVENT_TYPES = { + "simple-action-stop": { + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.click", + "session.ended", + }, + "drag-stop": { + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.drag", + "session.ended", + }, +} + + +def load_json(path: pathlib.Path) -> dict: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def json_lines(path: pathlib.Path) -> list[dict]: + records = [] + for line in path.read_text().splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + value = json.loads(line) + if isinstance(value, dict): + records.append(value) + return records + + +def find_record(records: list[dict], predicate, description: str) -> dict: + matches = [record for record in records if predicate(record)] + if not matches: + raise ValueError(f"missing {description}") + return matches[-1] + + +def action_record_ok(record: dict, required_event_types: set[str]) -> tuple[bool, list[str]]: + event_types = set(record.get("eventTypes", [])) + missing_event_types = sorted(required_event_types - event_types) + ok = ( + record.get("ok") is True + and not missing_event_types + and record.get("checkedMcpResponseShapesCaptured") is True + and record.get("checkedSkillReadinessCanCreateDraft") is True + and record.get("checkedSkillCreatorFinalizationHandoff") is True + and record.get("checkedGeneratedSkillPathRedaction") is True + ) + return ok, missing_event_types + + +def compact_surface_result(official_surface: dict, label: str) -> dict: + result = next( + item for item in official_surface.get("results", []) if item.get("label") == label + ) + return { + "ok": result.get("ok") is True, + "protocolVersion": result.get("protocolVersion"), + "serverName": result.get("serverName"), + "toolNames": result.get("toolNames"), + } + + +def missing_false_values(prefix: str, values: dict[str, bool]) -> list[str]: + return [f"{prefix}.{key}" for key, value in values.items() if value is not True] + + +def build_summary(args: argparse.Namespace) -> dict: + standalone = load_json(args.standalone_json) + npm = load_json(args.npm_json) + official_surface = load_json(args.official_surface_json) + official_no_active = load_json(args.official_no_active_json) + official_raw_timeout = load_json(args.official_raw_timeout_json) + official_fixture_set = load_json(args.official_fixture_set_json) + official_fixture_coverage = load_json(args.official_fixture_coverage_json) + official_fixture_ingest = load_json(args.official_fixture_ingest_json) + ocu_candidate_ingest = load_json(args.ocu_candidate_ingest_json) + official_capture_preflight = load_json(args.official_capture_preflight_json) + ocu_pairing_preflight = load_json(args.ocu_pairing_preflight_json) + baseline_audit_targets = load_json(args.baseline_audit_targets_json) + baseline_contract = load_json(args.baseline_contract_json) + matrix_records = json_lines(args.matrix_jsonl) + screenshot_records = json_lines(args.screenshot_jsonl) + action_records = json_lines(args.action_jsonl) + + screenshot_record = find_record( + screenshot_records, + lambda record: record.get("ok") is True and record.get("screenshotPolicy") == "always", + "screenshot context smoke record", + ) + action_record = find_record( + action_records, + lambda record: record.get("mode") == "actions" + and record.get("actionScenario") in (None, "mixed-action-stop"), + "real input action smoke record", + ) + scenario_action_records = { + scenario: find_record( + action_records, + lambda record, scenario=scenario: ( + record.get("mode") == "actions" + and record.get("actionScenario") == scenario + ), + f"{scenario} action smoke record", + ) + for scenario in REQUIRED_ACTION_SCENARIO_EVENT_TYPES + } + official_successful_recording_golden_complete = ( + official_fixture_coverage.get("coverageOk") is True + and official_fixture_coverage.get("hasRequiredOfficialSuccessfulFixture") is True + and official_fixture_coverage.get("requiredOfficialReadinessChecked") is True + and official_fixture_coverage.get("requiredOfficialReadinessOk") is True + ) + recommended_official_recording_coverage_complete = ( + official_fixture_coverage.get("hasRecommendedOfficialScenarioCoverage") is True + ) + official_golden_gate_passed = official_successful_recording_golden_complete + official_golden_requirement_satisfied = ( + not args.require_official_golden or official_golden_gate_passed + ) + local_surface = compact_surface_result(official_surface, "local-open-computer-use") + official_surface_result = compact_surface_result(official_surface, "official-record-and-replay") + matrix_modes = sorted({record.get("mode") for record in matrix_records if record.get("mode")}) + missing_matrix_modes = sorted(REQUIRED_MATRIX_MODES - set(matrix_modes)) + missing_action_event_types = sorted( + REQUIRED_ACTION_EVENT_TYPES - set(action_record.get("eventTypes", [])) + ) + scenario_candidate_evidence = {} + for scenario, required_event_types in REQUIRED_ACTION_SCENARIO_EVENT_TYPES.items(): + scenario_record = scenario_action_records[scenario] + scenario_ok, scenario_missing_event_types = action_record_ok( + scenario_record, + required_event_types, + ) + scenario_candidate_evidence[scenario] = { + "ok": scenario_record.get("ok") is True, + "eventCount": scenario_record.get("eventCount"), + "eventTypes": scenario_record.get("eventTypes"), + "checkedRequiredEventTypes": not scenario_missing_event_types, + "missingEventTypes": scenario_missing_event_types, + "checkedMcpResponseShapesCaptured": ( + scenario_record.get("checkedMcpResponseShapesCaptured") is True + ), + "checkedSkillReadinessCanCreateDraft": ( + scenario_record.get("checkedSkillReadinessCanCreateDraft") is True + ), + "checkedSkillCreatorFinalizationHandoff": ( + scenario_record.get("checkedSkillCreatorFinalizationHandoff") is True + ), + "checkedGeneratedSkillPathRedaction": ( + scenario_record.get("checkedGeneratedSkillPathRedaction") is True + ), + "checkedScenarioCandidate": scenario_ok, + } + + event_stream_matrix_evidence = { + "ok": any( + record.get("ok") is True and record.get("matrix") == "event-stream" + for record in matrix_records + ), + "checkedDefaultLifecycleHandoff": any( + record.get("ok") is True and record.get("handoffChecked") is True + for record in matrix_records + ), + "checkedRequiredModes": not missing_matrix_modes, + "checkedModes": matrix_modes, + "missingModes": missing_matrix_modes, + } + screenshot_context_evidence = { + "ok": screenshot_record.get("ok") is True, + "checkedScreenshotPolicyAlways": screenshot_record.get("screenshotPolicy") == "always", + "checkedScreenshotNeededForContext": ( + screenshot_record.get("screenshotNeededForContextCount", 0) > 0 + ), + "screenshotAvailableCount": screenshot_record.get("screenshotAvailableCount"), + "screenshotPathCount": screenshot_record.get("screenshotPathCount"), + "checkedScreenshotPathWhenAvailable": ( + screenshot_record.get("screenshotAvailableCount", 0) <= 0 + or screenshot_record.get("screenshotPathCount", 0) > 0 + ), + } + real_input_action_evidence = { + "ok": action_record.get("ok") is True, + "eventCount": action_record.get("eventCount"), + "eventTypes": action_record.get("eventTypes"), + "checkedRequiredEventTypes": not missing_action_event_types, + "missingEventTypes": missing_action_event_types, + "checkedSkillDraftGenerated": bool(action_record.get("skillPath")), + "checkedMcpTranscriptCaptured": bool(action_record.get("mcpTranscriptPath")), + "checkedMcpResponseShapesCaptured": ( + action_record.get("checkedMcpResponseShapesCaptured") is True + ), + "checkedSkillReadinessCanCreateDraft": ( + action_record.get("checkedSkillReadinessCanCreateDraft") is True + ), + "checkedSkillCreatorFinalizationHandoff": ( + action_record.get("checkedSkillCreatorFinalizationHandoff") is True + ), + "checkedGeneratedSkillPathRedaction": ( + action_record.get("checkedGeneratedSkillPathRedaction") is True + ), + "checkedSimpleActionStopCandidate": scenario_candidate_evidence[ + "simple-action-stop" + ]["checkedScenarioCandidate"], + "checkedDragStopCandidate": scenario_candidate_evidence["drag-stop"][ + "checkedScenarioCandidate" + ], + "scenarioCandidates": scenario_candidate_evidence, + } + official_surface_evidence = { + "fixture": "record-and-replay-event-stream-surface-1.0.857.json", + "local": local_surface, + "official": official_surface_result, + } + official_no_active_evidence = { + "ok": official_no_active.get("ok") is True, + "fixture": pathlib.Path(official_no_active.get("fixture", "")).name, + "checkedTools": official_no_active.get("checkedTools"), + "checkedNoSessionFilesCreated": ( + official_no_active.get("createdSessionFiles") is False + ), + "checkedStatusShape": ( + official_no_active.get("actual", {}).get("event_stream_status") + == EXPECTED_NO_ACTIVE_RESPONSE + ), + "checkedStopShape": ( + official_no_active.get("actual", {}).get("event_stream_stop") + == EXPECTED_NO_ACTIVE_RESPONSE + ), + "status": official_no_active.get("actual", {}).get("event_stream_status"), + "stop": official_no_active.get("actual", {}).get("event_stream_stop"), + } + raw_timeout_fixtures = official_raw_timeout.get("fixtures") or [] + raw_timeout_fixture_name = next( + ( + pathlib.Path(item).name + for item in raw_timeout_fixtures + if "raw-start-timeout" in pathlib.Path(item).name + ), + None, + ) + official_raw_timeout_evidence = { + "ok": official_raw_timeout.get("ok") is True, + "fixture": raw_timeout_fixture_name, + "checkedOfficialRawStartTimeoutBoundary": ( + official_raw_timeout.get("checkedOfficialRawStartTimeoutBoundary") is True + ), + "checkedOfficialRawStartStatusStopTimeout": ( + official_raw_timeout.get("checkedOfficialRawStartStatusStopTimeout") is True + ), + "checkedOfficialRawStartDoesNotReturnRecordingPaths": ( + official_raw_timeout.get("checkedOfficialRawStartDoesNotReturnRecordingPaths") + is True + ), + "checkedOfficialRawSurface": ( + official_raw_timeout.get("checkedOfficialRawSurface") is True + ), + "checkedRawProbeFixtureRedaction": ( + official_raw_timeout.get("checkedRawProbeFixtureRedaction") is True + ), + "officialRawToolCallCount": official_raw_timeout.get("officialRawToolCallCount"), + "officialRawTimeoutCount": official_raw_timeout.get("officialRawTimeoutCount"), + } + official_fixture_set_evidence = { + "ok": official_fixture_set.get("ok") is True, + "checkedFixtureSetGate": official_fixture_set.get("checkedFixtureSetGate") is True, + "checkedRequiredSimpleActionStopScenario": ( + official_fixture_set.get("checkedRequiredSimpleActionStopScenario") is True + ), + "checkedCandidatePairing": ( + official_fixture_set.get("checkedCandidatePairing") is True + ), + "checkedCandidateReadinessFailure": ( + official_fixture_set.get("checkedCandidateReadinessFailure") is True + ), + "checkedAxDiffComparisonPolicy": ( + official_fixture_set.get("checkedAxDiffComparisonPolicy") is True + ), + "checkedSuppressedStreamComparisonPolicy": ( + official_fixture_set.get("checkedSuppressedStreamComparisonPolicy") is True + ), + "checkedAxDiffComparisonFailure": ( + official_fixture_set.get("checkedAxDiffComparisonFailure") is True + ), + "checkedStopEndReasonPolicy": ( + official_fixture_set.get("checkedStopEndReasonPolicy") is True + ), + "checkedCancelScenarioPolicy": ( + official_fixture_set.get("checkedCancelScenarioPolicy") is True + ), + "checkedKeyboardInputScenarioPolicy": ( + official_fixture_set.get("checkedKeyboardInputScenarioPolicy") is True + ), + "checkedDragScenarioPolicy": ( + official_fixture_set.get("checkedDragScenarioPolicy") is True + ), + "checkedTimeoutScenarioPolicy": ( + official_fixture_set.get("checkedTimeoutScenarioPolicy") is True + ), + "coverageOk": official_fixture_coverage.get("coverageOk") is True, + "scenarioCoverageOk": official_fixture_coverage.get("scenarioCoverageOk") is True, + "requiredOfficialScenarios": official_fixture_coverage.get("requiredOfficialScenarios"), + "recommendedOfficialScenarios": official_fixture_coverage.get( + "recommendedOfficialScenarios" + ), + "requiredOfficialReadinessChecked": ( + official_fixture_coverage.get("requiredOfficialReadinessChecked") is True + ), + "requiredOfficialReadinessOk": ( + official_fixture_coverage.get("requiredOfficialReadinessOk") is True + ), + "repoAvailableOfficialScenarios": official_fixture_coverage.get( + "availableOfficialScenarios" + ), + "repoHasRequiredOfficialSuccessfulFixture": ( + official_fixture_coverage.get("hasRequiredOfficialSuccessfulFixture") is True + ), + "repoHasRecommendedOfficialScenarioCoverage": ( + official_fixture_coverage.get("hasRecommendedOfficialScenarioCoverage") is True + ), + "missingRepoOfficialScenarios": official_fixture_coverage.get( + "missingOfficialScenarios" + ), + "notReadyRepoOfficialScenarios": official_fixture_coverage.get( + "notReadyOfficialScenarios" + ), + "missingRepoRecommendedOfficialScenarios": official_fixture_coverage.get( + "missingRecommendedOfficialScenarios" + ), + "coverageErrors": official_fixture_coverage.get("errors") or [], + } + fixture_ingest_evidence = { + "checkedOfficialFixtureIngest": ( + official_fixture_ingest.get("checkedOfficialFixtureIngest") is True + ), + "checkedOfficialFixtureInspectOnly": ( + official_fixture_ingest.get("checkedOfficialFixtureInspectOnly") is True + ), + "checkedOfficialSessionDirectoryPathHandoff": ( + official_fixture_ingest.get("checkedOfficialSessionDirectoryPathHandoff") is True + ), + "checkedPostIngestCoverageReport": ( + official_fixture_ingest.get("checkedPostIngestCoverageReport") is True + ), + "checkedPostIngestCoverageReadiness": ( + official_fixture_ingest.get("checkedPostIngestCoverageReadiness") is True + ), + "checkedPostIngestRequireCoverageFailure": ( + official_fixture_ingest.get("checkedPostIngestRequireCoverageFailure") is True + ), + "checkedOcuCandidateIngest": ( + ocu_candidate_ingest.get("checkedOcuCandidateIngest") is True + ), + "checkedOcuCandidateIngestHandoffCommands": ( + ocu_candidate_ingest.get("checkedCandidateIngestHandoffCommands") is True + ), + "checkedOcuSmokeJsonImport": ( + ocu_candidate_ingest.get("checkedSmokeJsonImport") is True + ), + "checkedOcuOfficialCandidatePairing": ( + ocu_candidate_ingest.get("checkedOfficialCandidatePairing") is True + ), + "checkedOcuCandidateRedaction": ( + ocu_candidate_ingest.get("checkedCandidateRedaction") is True + ), + "checkedOcuKeyboardInputScenarioReadiness": ( + ocu_candidate_ingest.get("checkedKeyboardInputScenarioReadiness") is True + ), + "checkedOcuDragScenarioReadiness": ( + ocu_candidate_ingest.get("checkedDragScenarioReadiness") is True + ), + } + preflight_evidence = { + "checkedOfficialCaptureMissingScenarioPreflight": ( + official_capture_preflight.get("checkedMissingScenarioPreflight") is True + ), + "checkedOfficialCapturePacket": ( + official_capture_preflight.get("checkedCapturePacket") is True + ), + "checkedOfficialCapturePacketPostCaptureWorkflow": ( + official_capture_preflight.get("checkedCapturePacketPostCaptureWorkflow") + is True + ), + "checkedOfficialCapturePacketWorkflowVerifier": ( + official_capture_preflight.get("checkedCapturePacketWorkflowVerifier") + is True + ), + "checkedOfficialCapturePacketHandoffScripts": ( + official_capture_preflight.get("checkedCapturePacketHandoffScripts") + is True + ), + "checkedOfficialCapturePacketStrictAuditHandoff": ( + official_capture_preflight.get("checkedCapturePacketStrictAuditHandoff") + is True + ), + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff": ( + official_capture_preflight.get( + "checkedCapturePacketStrictExpectedFailureAuditHandoff" + ) + is True + ), + "checkedOfficialCapturePacketOcuCandidateOutputDir": ( + official_capture_preflight.get("checkedCapturePacketOcuCandidateOutputDir") + is True + ), + "checkedOfficialCapturePacketNoTranscript": ( + official_capture_preflight.get("checkedCapturePacketNoTranscript") + is True + ), + "checkedOfficialCapturePacketSet": ( + official_capture_preflight.get("checkedCapturePacketSet") is True + ), + "checkedOfficialCapturePacketSetPostCaptureWorkflow": ( + official_capture_preflight.get("checkedCapturePacketSetPostCaptureWorkflow") + is True + ), + "checkedOfficialCapturePacketSetWorkflowVerifier": ( + official_capture_preflight.get("checkedCapturePacketSetWorkflowVerifier") + is True + ), + "checkedOfficialCapturePacketSetNoTranscript": ( + official_capture_preflight.get("checkedCapturePacketSetNoTranscript") + is True + ), + "checkedOfficialCapturePacketSetOcuCandidateHandoff": ( + official_capture_preflight.get("checkedCapturePacketSetOcuCandidateHandoff") + is True + ), + "checkedOfficialCapturePacketSetContractManifest": ( + official_capture_preflight.get("checkedCapturePacketSetContractManifest") + is True + ), + "checkedOfficialCapturePacketTranscriptManifest": ( + official_capture_preflight.get("checkedCapturePacketTranscriptManifest") + is True + ), + "checkedOfficialCapturePacketMakeTargets": ( + official_capture_preflight.get("checkedMakeCapturePacketTargets") is True + ), + "checkedOfficialCapturePacketInputSemanticGuard": ( + official_capture_preflight.get("checkedCapturePacketInputSemanticGuard") + is True + ), + "checkedOfficialCapturePacketPlaceholderGuard": ( + official_capture_preflight.get("checkedCapturePacketPlaceholderGuard") is True + ), + "checkedOfficialCapturePacketVerifyInputs": ( + official_capture_preflight.get("checkedCapturePacketVerifyInputs") is True + ), + "checkedOfficialCapturePacketImportPlaceholderGuard": ( + official_capture_preflight.get("checkedCapturePacketImportPlaceholderGuard") + is True + ), + "checkedOfficialCapturePacketTranscriptPlaceholderGuard": ( + official_capture_preflight.get("checkedCapturePacketTranscriptPlaceholderGuard") + is True + ), + "checkedOfficialCapturePacketSetRootPlaceholderGuard": ( + official_capture_preflight.get("checkedCapturePacketSetRootPlaceholderGuard") + is True + ), + "checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard": ( + official_capture_preflight.get( + "checkedCapturePacketSetRootPreflightPlaceholderGuard" + ) + is True + ), + "checkedOfficialCapturePacketSetVerifyAll": ( + official_capture_preflight.get("checkedCapturePacketSetVerifyAll") is True + ), + "checkedOfficialCaptureRequireReadyFailure": ( + official_capture_preflight.get("checkedRequireReadyFailure") is True + ), + "checkedOfficialCaptureMissingPluginFailure": ( + official_capture_preflight.get("checkedMissingPluginFailure") is True + ), + "checkedOfficialCaptureKeyboardScenario": ( + official_capture_preflight.get("checkedAllowedMissingPluginKeyboardScenario") + is True + ), + "checkedOfficialCaptureCoverageErrorFailure": ( + official_capture_preflight.get("checkedCoverageErrorFailure") is True + ), + "checkedOcuPairingMissingOfficialPreflight": ( + ocu_pairing_preflight.get("checkedMissingOfficialPreflight") is True + ), + "checkedOcuPairingNoCandidatePreflight": ( + ocu_pairing_preflight.get("checkedNoCandidatePreflight") is True + ), + "checkedOcuPairingPairedCandidatePreflight": ( + ocu_pairing_preflight.get("checkedPairedCandidatePreflight") is True + ), + "checkedOcuPairingKeyboardRecordingRequiredScenario": ( + ocu_pairing_preflight.get("checkedKeyboardRecordingRequiredScenario") is True + ), + "checkedOcuPairingCoverageErrorReport": ( + ocu_pairing_preflight.get("checkedCoverageErrorReport") is True + ), + "checkedBaselineAuditMakeTargets": ( + baseline_audit_targets.get("checkedBaselineAuditMakeTarget") is True + ), + "checkedBaselineAuditDefaultSummaryPath": ( + baseline_audit_targets.get("checkedBaselineAuditDefaultSummaryPath") is True + ), + "checkedBaselineAuditCustomSummaryPath": ( + baseline_audit_targets.get("checkedBaselineAuditCustomSummaryPath") is True + ), + "checkedBaselineAuditIgnoresStrictSummaryVar": ( + baseline_audit_targets.get("checkedBaselineAuditIgnoresStrictSummaryVar") + is True + ), + "checkedBaselineAuditStrictOfficialGoldenTarget": ( + baseline_audit_targets.get("checkedStrictOfficialGoldenAuditMakeTarget") is True + ), + "checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPath": ( + baseline_audit_targets.get( + "checkedStrictOfficialGoldenAuditDefaultSummaryPath" + ) + is True + ), + "checkedBaselineAuditStrictOfficialGoldenCustomSummaryPath": ( + baseline_audit_targets.get( + "checkedStrictOfficialGoldenAuditCustomSummaryPath" + ) + is True + ), + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar": ( + baseline_audit_targets.get( + "checkedStrictOfficialGoldenAuditIgnoresBaselineSummaryVar" + ) + is True + ), + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath": ( + baseline_audit_targets.get( + "checkedStrictOfficialGoldenAuditSeparateSummaryPath" + ) + is True + ), + } + baseline_contract_evidence = { + "ok": baseline_contract.get("ok") is True, + "checkedRequiredBaselineChecks": ( + baseline_contract.get("checkedRequiredBaselineChecks") is True + ), + "checkedNoDuplicateRequiredBaselineChecks": ( + baseline_contract.get("checkedNoDuplicateRequiredBaselineChecks") is True + ), + "checkedStandaloneSmokeRequiredKeys": ( + baseline_contract.get("checkedStandaloneSmokeRequiredKeys") is True + ), + "checkedStandaloneSummaryEvidenceKeys": ( + baseline_contract.get("checkedStandaloneSummaryEvidenceKeys") is True + ), + "checkedStandaloneLifecycleSummaryRenames": ( + baseline_contract.get("checkedStandaloneLifecycleSummaryRenames") is True + ), + "checkedNpmStagedSmokeRequiredKeys": ( + baseline_contract.get("checkedNpmStagedSmokeRequiredKeys") is True + ), + "checkedNpmStagedSummaryEvidenceKeys": ( + baseline_contract.get("checkedNpmStagedSummaryEvidenceKeys") is True + ), + "checkedNpmStagedSummaryMatchesSmoke": ( + baseline_contract.get("checkedNpmStagedSummaryMatchesSmoke") is True + ), + } + standalone_evidence = { + "checkedGeneratedRepoSelfCheck": ( + standalone.get("checkedGeneratedRepoSelfCheck") is True + ), + "checkedManifestContract": standalone.get("checkedManifestContract") is True, + "checkedReadmeHandoffContract": ( + standalone.get("checkedReadmeHandoffContract") is True + ), + "checkedReadmeOfficialEvidenceHandoff": ( + standalone.get("checkedReadmeOfficialEvidenceHandoff") is True + ), + "checkedReadmeOfficialGoldenGap": ( + standalone.get("checkedReadmeOfficialGoldenGap") is True + ), + "checkedReadmeWaitNotifyBoundary": ( + standalone.get("checkedReadmeWaitNotifyBoundary") is True + ), + "checkedGeneratedReadmePrerequisites": ( + standalone.get("checkedGeneratedReadmePrerequisites") is True + ), + "checkedGeneratedReadmeScenarioList": ( + standalone.get("checkedGeneratedReadmeScenarioList") is True + ), + "checkedOfficialEvidenceManifest": ( + standalone.get("checkedOfficialEvidenceManifest") is True + ), + "checkedOfficialEvidenceScenarioManifest": ( + standalone.get("checkedOfficialEvidenceScenarioManifest") is True + ), + "checkedOfficialEvidencePreflightManifest": ( + standalone.get("checkedOfficialEvidencePreflightManifest") is True + ), + "checkedOfficialEvidenceAuditManifest": ( + standalone.get("checkedOfficialEvidenceAuditManifest") is True + ), + "checkedOfficialFixtureSetComparePolicyManifest": ( + standalone.get("checkedOfficialFixtureSetComparePolicyManifest") is True + ), + "checkedSourceBaselineSummaryEvidence": ( + standalone.get("checkedSourceBaselineSummaryEvidence") is True + ), + "checkedSourceBaselineSummaryOfficialGoldenState": ( + standalone.get("checkedSourceBaselineSummaryOfficialGoldenState") is True + ), + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": ( + standalone.get( + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff" + ) + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": ( + standalone.get("checkedSourceBaselineSummaryCapturePacketSetContractManifest") + is True + ), + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": ( + standalone.get("checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow") + is True + ), + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": ( + standalone.get("checkedSourceBaselineSummaryCapturePacketWorkflowVerifier") + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": ( + standalone.get( + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + ) + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": ( + standalone.get("checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier") + is True + ), + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": ( + standalone.get("checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff") + is True + ), + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": ( + standalone.get( + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff" + ) + is True + ), + "checkedPackageArtifact": standalone.get("checkedPackageArtifact") is True, + "checkedRuntimeContract": standalone.get("checkedRuntimeContract") is True, + "checkedRuntimeTimeoutDiagnostics": ( + standalone.get("checkedRuntimeTimeoutDiagnostics") is True + ), + "checkedInitializeSurfaceContract": ( + standalone.get("checkedInitializeSurfaceContract") is True + ), + "checkedToolMetadataContract": ( + standalone.get("checkedToolMetadataContract") is True + ), + "checkedToolInputSchemaNoArguments": ( + standalone.get("checkedToolInputSchemaNoArguments") is True + ), + "checkedNoActiveStatusStop": standalone.get("checkedNoActiveStatusStop") is True, + "checkedRequiresObjectParams": standalone.get("checkedRequiresObjectParams") is True, + "checkedRequiresStringToolName": ( + standalone.get("checkedRequiresStringToolName") is True + ), + "checkedRequiresObjectArguments": ( + standalone.get("checkedRequiresObjectArguments") is True + ), + "checkedRejectsUnexpectedArguments": ( + standalone.get("checkedRejectsUnexpectedArguments") is True + ), + "checkedRejectsNonObjectArguments": ( + standalone.get("checkedRejectsNonObjectArguments") is True + ), + "checkedRejectedRequestsDoNotCreateSessionFiles": ( + standalone.get("checkedRejectedRequestsDoNotCreateSessionFiles") is True + ), + "checkedStatusNotUsedAsWaitLoopGuard": ( + standalone.get("checkedStatusNotUsedAsWaitLoopGuard") is True + ), + "checkedMcpNoDirectEventContentsGuard": ( + standalone.get("checkedMcpNoDirectEventContentsGuard") is True + ), + "checkedWaitNotifyContract": standalone.get("checkedWaitNotifyContract") is True, + "checkedNotifySuppressedEventsPathEnv": ( + standalone.get("checkedNotifySuppressedEventsPathEnv") is True + ), + "checkedNotifyCallbackFailureExit": ( + standalone.get("checkedNotifyCallbackFailureExit") is True + ), + "checkedNotifyCallbackFailureReason": ( + standalone.get("checkedNotifyCallbackFailureReason") is True + ), + "checkedNotifyCallbackTimeoutFailureExit": ( + standalone.get("checkedNotifyCallbackTimeoutFailureExit") is True + ), + "checkedNotifyCallbackTimeoutReason": ( + standalone.get("checkedNotifyCallbackTimeoutReason") is True + ), + "checkedRecordingToSkillHandoff": ( + standalone.get("checkedRecordingToSkillHandoff") is True + ), + "checkedStrictValidation": standalone.get("checkedStrictValidation") is True, + "checkedDeclaredHandoffPaths": standalone.get("checkedDeclaredHandoffPaths") is True, + "checkedScreenshotPathContainment": ( + standalone.get("checkedScreenshotPathContainment") is True + ), + "checkedEventsOnlyValidation": ( + standalone.get("checkedEventsOnlyValidation") is True + ), + "checkedScaffoldSkill": standalone.get("checkedScaffoldSkill") is True, + "checkedScaffoldSkillFailureExit": ( + standalone.get("checkedScaffoldSkillFailureExit") is True + ), + "checkedSkillCreatorHandoff": ( + standalone.get("checkedSkillCreatorHandoff") is True + ), + "checkedCancelledRecordingContract": ( + standalone.get("checkedCancelledRecordingContract") is True + ), + "checkedCancelledRecordingRejected": ( + standalone.get("checkedCancelledRecordingRejected") is True + ), + "checkedLifecycleSmoke": standalone.get("checkedLifecycleSmoke") is True, + "checkedLifecycleOneActive": standalone.get("checkedOneActive") is True, + "checkedLifecycleIdempotentStop": ( + standalone.get("checkedIdempotentStop") is True + ), + "checkedLifecycleFinalStatus": standalone.get("checkedFinalStatus") is True, + } + npm_evidence = { + "checkedNpmLauncher": npm.get("checkedNpmLauncher") is True, + "checkedNpmPythonLauncherDiagnostics": ( + npm.get("checkedNpmPythonLauncherDiagnostics") is True + ), + "checkedStandaloneRepoScaffold": npm.get("checkedStandaloneRepoScaffold") is True, + "checkedManifestContract": npm.get("checkedManifestContract") is True, + "checkedReadmeHandoffContract": npm.get("checkedReadmeHandoffContract") is True, + "checkedReadmeOfficialEvidenceHandoff": ( + npm.get("checkedReadmeOfficialEvidenceHandoff") is True + ), + "checkedReadmeOfficialGoldenGap": ( + npm.get("checkedReadmeOfficialGoldenGap") is True + ), + "checkedReadmeWaitNotifyBoundary": ( + npm.get("checkedReadmeWaitNotifyBoundary") is True + ), + "checkedGeneratedReadmePrerequisites": ( + npm.get("checkedGeneratedReadmePrerequisites") is True + ), + "checkedGeneratedReadmeScenarioList": ( + npm.get("checkedGeneratedReadmeScenarioList") is True + ), + "checkedOfficialEvidenceManifest": ( + npm.get("checkedOfficialEvidenceManifest") is True + ), + "checkedOfficialEvidenceScenarioManifest": ( + npm.get("checkedOfficialEvidenceScenarioManifest") is True + ), + "checkedOfficialEvidencePreflightManifest": ( + npm.get("checkedOfficialEvidencePreflightManifest") is True + ), + "checkedOfficialEvidenceAuditManifest": ( + npm.get("checkedOfficialEvidenceAuditManifest") is True + ), + "checkedOfficialFixtureSetComparePolicyManifest": ( + npm.get("checkedOfficialFixtureSetComparePolicyManifest") is True + ), + "checkedSourceBaselineSummaryEvidence": ( + npm.get("checkedSourceBaselineSummaryEvidence") is True + ), + "checkedSourceBaselineSummaryOfficialGoldenState": ( + npm.get("checkedSourceBaselineSummaryOfficialGoldenState") is True + ), + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": ( + npm.get("checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff") + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": ( + npm.get("checkedSourceBaselineSummaryCapturePacketSetContractManifest") + is True + ), + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": ( + npm.get("checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow") + is True + ), + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": ( + npm.get("checkedSourceBaselineSummaryCapturePacketWorkflowVerifier") + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": ( + npm.get("checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow") + is True + ), + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": ( + npm.get("checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier") + is True + ), + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": ( + npm.get("checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff") + is True + ), + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": ( + npm.get( + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff" + ) + is True + ), + "checkedPackageArtifact": npm.get("checkedPackageArtifact") is True, + "checkedGeneratedRepoSelfCheck": npm.get("checkedGeneratedRepoSelfCheck") is True, + "checkedRuntimeTimeoutDiagnostics": ( + npm.get("checkedRuntimeTimeoutDiagnostics") is True + ), + "checkedInitializeSurfaceContract": ( + npm.get("checkedInitializeSurfaceContract") is True + ), + "checkedToolMetadataContract": npm.get("checkedToolMetadataContract") is True, + "checkedToolInputSchemaNoArguments": ( + npm.get("checkedToolInputSchemaNoArguments") is True + ), + "checkedNoActiveStatusStop": npm.get("checkedNoActiveStatusStop") is True, + "checkedRequiresObjectParams": npm.get("checkedRequiresObjectParams") is True, + "checkedRequiresStringToolName": ( + npm.get("checkedRequiresStringToolName") is True + ), + "checkedRequiresObjectArguments": ( + npm.get("checkedRequiresObjectArguments") is True + ), + "checkedRejectsUnexpectedArguments": ( + npm.get("checkedRejectsUnexpectedArguments") is True + ), + "checkedRejectsNonObjectArguments": ( + npm.get("checkedRejectsNonObjectArguments") is True + ), + "checkedRejectedRequestsDoNotCreateSessionFiles": ( + npm.get("checkedRejectedRequestsDoNotCreateSessionFiles") is True + ), + "checkedSkillWorkflow": npm.get("checkedSkillWorkflow") is True, + "checkedStatusNotUsedAsWaitLoopGuard": ( + npm.get("checkedStatusNotUsedAsWaitLoopGuard") is True + ), + "checkedMcpNoDirectEventContentsGuard": ( + npm.get("checkedMcpNoDirectEventContentsGuard") is True + ), + "checkedWaitNotifyContract": npm.get("checkedWaitNotifyContract") is True, + "checkedNotifySuppressedEventsPathEnv": ( + npm.get("checkedNotifySuppressedEventsPathEnv") is True + ), + "checkedNotifyCallbackFailureExit": ( + npm.get("checkedNotifyCallbackFailureExit") is True + ), + "checkedNotifyCallbackFailureReason": ( + npm.get("checkedNotifyCallbackFailureReason") is True + ), + "checkedNotifyCallbackTimeoutFailureExit": ( + npm.get("checkedNotifyCallbackTimeoutFailureExit") is True + ), + "checkedNotifyCallbackTimeoutReason": ( + npm.get("checkedNotifyCallbackTimeoutReason") is True + ), + "checkedSkillPackaging": npm.get("checkedSkillPackaging") is True, + "checkedRecordingToSkillManifestContract": ( + npm.get("checkedRecordingToSkillManifestContract") is True + ), + "checkedCancelledRecordingContract": ( + npm.get("checkedCancelledRecordingContract") is True + ), + "checkedStrictValidation": npm.get("checkedStrictValidation") is True, + "checkedDeclaredHandoffPaths": npm.get("checkedDeclaredHandoffPaths") is True, + "checkedScreenshotPathContainment": ( + npm.get("checkedScreenshotPathContainment") is True + ), + "checkedEventsOnlyValidation": npm.get("checkedEventsOnlyValidation") is True, + "checkedScaffoldSkill": npm.get("checkedScaffoldSkill") is True, + "checkedScaffoldSkillFailureExit": ( + npm.get("checkedScaffoldSkillFailureExit") is True + ), + "checkedCancelledRecordingRejected": ( + npm.get("checkedCancelledRecordingRejected") is True + ), + "checkedRecordingToSkillHandoff": ( + npm.get("checkedRecordingToSkillHandoff") is True + ), + "checkedSkillCreatorHandoff": npm.get("checkedSkillCreatorHandoff") is True, + } + + official_non_recording_baseline_verified = ( + local_surface.get("ok") is True + and official_surface_result.get("ok") is True + and official_no_active_evidence["ok"] is True + and official_no_active_evidence["checkedNoSessionFilesCreated"] is True + and official_no_active_evidence["checkedStatusShape"] is True + and official_no_active_evidence["checkedStopShape"] is True + and not missing_false_values( + "officialRawStartTimeout", + { + key: value + for key, value in official_raw_timeout_evidence.items() + if key.startswith("checked") or key == "ok" + }, + ) + ) + standalone_repo_scaffold_baseline_verified = ( + not missing_false_values("standaloneSkillRepo", standalone_evidence) + and not missing_false_values("npmStagedSkillRepo", npm_evidence) + ) + usable_baseline_missing_evidence = ( + missing_false_values( + "baselineContract", + { + key: value + for key, value in baseline_contract_evidence.items() + if key.startswith("checked") or key == "ok" + }, + ) + + missing_false_values( + "eventStreamMatrix", + { + "ok": event_stream_matrix_evidence["ok"], + "checkedDefaultLifecycleHandoff": event_stream_matrix_evidence[ + "checkedDefaultLifecycleHandoff" + ], + "checkedRequiredModes": event_stream_matrix_evidence["checkedRequiredModes"], + }, + ) + + missing_false_values( + "screenshotContextSmoke", + { + "ok": screenshot_context_evidence["ok"], + "checkedScreenshotPolicyAlways": screenshot_context_evidence[ + "checkedScreenshotPolicyAlways" + ], + "checkedScreenshotNeededForContext": screenshot_context_evidence[ + "checkedScreenshotNeededForContext" + ], + "checkedScreenshotPathWhenAvailable": screenshot_context_evidence[ + "checkedScreenshotPathWhenAvailable" + ], + }, + ) + + missing_false_values( + "realInputActionSmoke", + { + "ok": real_input_action_evidence["ok"], + "checkedRequiredEventTypes": real_input_action_evidence[ + "checkedRequiredEventTypes" + ], + "checkedSkillDraftGenerated": real_input_action_evidence[ + "checkedSkillDraftGenerated" + ], + "checkedMcpTranscriptCaptured": real_input_action_evidence[ + "checkedMcpTranscriptCaptured" + ], + "checkedMcpResponseShapesCaptured": real_input_action_evidence[ + "checkedMcpResponseShapesCaptured" + ], + "checkedSkillReadinessCanCreateDraft": real_input_action_evidence[ + "checkedSkillReadinessCanCreateDraft" + ], + "checkedSkillCreatorFinalizationHandoff": real_input_action_evidence[ + "checkedSkillCreatorFinalizationHandoff" + ], + "checkedGeneratedSkillPathRedaction": real_input_action_evidence[ + "checkedGeneratedSkillPathRedaction" + ], + "checkedSimpleActionStopCandidate": real_input_action_evidence[ + "checkedSimpleActionStopCandidate" + ], + "checkedDragStopCandidate": real_input_action_evidence[ + "checkedDragStopCandidate" + ], + }, + ) + + missing_false_values( + "officialSurfaceCompare", + { + "localOk": local_surface.get("ok") is True, + "officialOk": official_surface_result.get("ok") is True, + }, + ) + + missing_false_values( + "officialNoActiveResponse", + { + "ok": official_no_active_evidence["ok"], + "checkedNoSessionFilesCreated": official_no_active_evidence[ + "checkedNoSessionFilesCreated" + ], + "checkedStatusShape": official_no_active_evidence["checkedStatusShape"], + "checkedStopShape": official_no_active_evidence["checkedStopShape"], + }, + ) + + missing_false_values( + "officialRawStartTimeout", + { + key: value + for key, value in official_raw_timeout_evidence.items() + if key.startswith("checked") or key == "ok" + }, + ) + + missing_false_values( + "officialFixtureSetGate", + { + key: value + for key, value in official_fixture_set_evidence.items() + if key.startswith("checked") or key == "ok" + }, + ) + + missing_false_values("fixtureIngestPipelines", fixture_ingest_evidence) + + missing_false_values("preflightPipelines", preflight_evidence) + + missing_false_values("standaloneSkillRepo", standalone_evidence) + + missing_false_values("npmStagedSkillRepo", npm_evidence) + ) + usable_baseline = not usable_baseline_missing_evidence + standalone_repo_baseline_ready = ( + usable_baseline and standalone_repo_scaffold_baseline_verified + ) + official_successful_recording_equivalence_ready = ( + usable_baseline and official_successful_recording_golden_complete + ) + missing_required_official_scenarios = ( + official_fixture_coverage.get("missingOfficialScenarios") or [] + ) + not_ready_required_official_scenarios = ( + official_fixture_coverage.get("notReadyOfficialScenarios") or [] + ) + missing_recommended_official_scenarios = ( + official_fixture_coverage.get("missingRecommendedOfficialScenarios") or [] + ) + next_actions = [] + if not usable_baseline: + next_actions.append( + { + "kind": "fix-usable-baseline-evidence", + "reason": "required baseline evidence is incomplete", + "missingEvidence": usable_baseline_missing_evidence, + "commands": ["make record-and-replay-baseline-smoke"], + } + ) + if not official_successful_recording_golden_complete: + scenario = ( + missing_required_official_scenarios[0] + if missing_required_official_scenarios + else "simple-action-stop" + ) + next_actions.append( + { + "kind": "capture-official-successful-recording-golden", + "reason": "required official successful recording fixture is missing or not ready", + "scenario": scenario, + "missingRequiredScenarios": missing_required_official_scenarios, + "notReadyRequiredScenarios": not_ready_required_official_scenarios, + "commands": [ + ( + "make record-and-replay-official-golden-capture-packet " + f"RNR_SCENARIO={scenario} RNR_PACKET_DIR=" + ), + "cd && ./verify-inputs.sh", + "cd && ./inspect-only.sh", + "cd && ./import-fixture.sh", + ( + "./scripts/ingest-official-record-and-replay-fixture.py " + "--status-json " + f"--name official-{scenario}-1.0.857 " + f"--scenario {scenario} --inspect-only" + ), + "make record-and-replay-official-golden-gate-audit", + ], + } + ) + if missing_recommended_official_scenarios: + next_actions.append( + { + "kind": "capture-recommended-official-golden-set", + "reason": "recommended calibration scenarios are not fully covered", + "missingRecommendedScenarios": missing_recommended_official_scenarios, + "commands": [ + ( + "make record-and-replay-official-golden-capture-packet-set " + "RNR_PACKET_DIR=" + ), + "cd && ./verify-all.sh", + "cd && ./inspect-all.sh", + "cd && ./import-all.sh", + "cd && ./ingest-ocu-candidates.sh", + ], + } + ) + if standalone_repo_baseline_ready: + next_actions.append( + { + "kind": "scaffold-standalone-record-and-replay-repo", + "reason": "standalone repo baseline is ready, but official equivalence still depends on golden fixtures", + "commands": [ + "make record-and-replay-baseline-audit", + "open-computer-use scaffold-record-and-replay-skill-repo --output-dir ", + "cd && ./scripts/check.sh", + "cd && ./scripts/recording-lifecycle-smoke.py", + ], + } + ) + + return { + "ok": usable_baseline and official_golden_requirement_satisfied, + "baseline": "record-and-replay", + "checks": list(REQUIRED_BASELINE_CHECKS), + "status": { + "usableBaseline": usable_baseline, + "missingUsableBaselineEvidence": usable_baseline_missing_evidence, + "strictOfficialGoldenRequired": args.require_official_golden, + "officialGoldenRequirementSatisfied": official_golden_requirement_satisfied, + "officialGoldenGatePassed": official_golden_gate_passed, + "officialNonRecordingBaselineVerified": official_non_recording_baseline_verified, + "officialRawStartTimeoutBoundaryVerified": not missing_false_values( + "officialRawStartTimeout", + { + key: value + for key, value in official_raw_timeout_evidence.items() + if key.startswith("checked") or key == "ok" + }, + ), + "standaloneRepoScaffoldBaselineVerified": ( + standalone_repo_scaffold_baseline_verified + ), + "standaloneRepoBaselineReady": standalone_repo_baseline_ready, + "officialSuccessfulRecordingGoldenComplete": ( + official_successful_recording_golden_complete + ), + "officialSuccessfulRecordingEquivalenceReady": ( + official_successful_recording_equivalence_ready + ), + "recommendedOfficialRecordingCoverageComplete": ( + recommended_official_recording_coverage_complete + ), + "requiresOfficialGoldenCapture": ( + not official_successful_recording_golden_complete + ), + "missingRequiredOfficialSuccessfulRecordingScenarios": ( + missing_required_official_scenarios + ), + "notReadyRequiredOfficialSuccessfulRecordingScenarios": ( + not_ready_required_official_scenarios + ), + "missingRecommendedOfficialSuccessfulRecordingScenarios": ( + missing_recommended_official_scenarios + ), + "officialFixtureCoverageErrors": official_fixture_coverage.get("errors") or [], + }, + "nextActions": next_actions, + "evidence": { + "baselineContract": baseline_contract_evidence, + "eventStreamMatrix": event_stream_matrix_evidence, + "screenshotContextSmoke": screenshot_context_evidence, + "realInputActionSmoke": real_input_action_evidence, + "officialSurfaceCompare": official_surface_evidence, + "officialNoActiveResponse": official_no_active_evidence, + "officialRawStartTimeout": official_raw_timeout_evidence, + "officialFixtureSetGate": official_fixture_set_evidence, + "fixtureIngestPipelines": fixture_ingest_evidence, + "preflightPipelines": preflight_evidence, + "standaloneSkillRepo": standalone_evidence, + "npmStagedSkillRepo": npm_evidence, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Build the final machine-readable Record & Replay baseline summary." + ) + parser.add_argument("--standalone-json", required=True, type=pathlib.Path) + parser.add_argument("--npm-json", required=True, type=pathlib.Path) + parser.add_argument("--official-surface-json", required=True, type=pathlib.Path) + parser.add_argument("--official-no-active-json", required=True, type=pathlib.Path) + parser.add_argument("--official-raw-timeout-json", required=True, type=pathlib.Path) + parser.add_argument("--official-fixture-set-json", required=True, type=pathlib.Path) + parser.add_argument("--official-fixture-coverage-json", required=True, type=pathlib.Path) + parser.add_argument("--official-fixture-ingest-json", required=True, type=pathlib.Path) + parser.add_argument("--ocu-candidate-ingest-json", required=True, type=pathlib.Path) + parser.add_argument("--official-capture-preflight-json", required=True, type=pathlib.Path) + parser.add_argument("--ocu-pairing-preflight-json", required=True, type=pathlib.Path) + parser.add_argument("--baseline-audit-targets-json", required=True, type=pathlib.Path) + parser.add_argument("--baseline-contract-json", required=True, type=pathlib.Path) + parser.add_argument("--matrix-jsonl", required=True, type=pathlib.Path) + parser.add_argument("--screenshot-jsonl", required=True, type=pathlib.Path) + parser.add_argument("--action-jsonl", required=True, type=pathlib.Path) + parser.add_argument( + "--require-official-golden", + action="store_true", + help="Return non-zero unless required official successful recording fixtures exist.", + ) + args = parser.parse_args() + + summary = build_summary(args) + print(json.dumps(summary, sort_keys=True), flush=True) + if not summary["status"]["usableBaseline"]: + missing = summary["status"].get("missingUsableBaselineEvidence") or [] + print( + "Record & Replay usable baseline evidence is incomplete: " + ", ".join(missing), + file=sys.stderr, + ) + if not summary["status"]["officialGoldenRequirementSatisfied"]: + missing = summary["status"].get("missingRequiredOfficialSuccessfulRecordingScenarios") or [] + not_ready = ( + summary["status"].get("notReadyRequiredOfficialSuccessfulRecordingScenarios") + or [] + ) + details = [] + if missing: + details.append("missing=" + ", ".join(missing)) + if not_ready: + details.append("notReady=" + ", ".join(not_ready)) + coverage_errors = ( + summary["status"].get("officialFixtureCoverageErrors") + or [] + ) + if coverage_errors: + details.append("coverageErrors=" + " | ".join(coverage_errors)) + if not details: + details.append("readiness=failed") + print( + "official successful recording golden fixture coverage/readiness is incomplete: " + + "; ".join(details), + file=sys.stderr, + ) + if not summary["ok"]: + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-docs.sh b/scripts/check-docs.sh index d0073ee0..58689c1f 100755 --- a/scripts/check-docs.sh +++ b/scripts/check-docs.sh @@ -21,13 +21,22 @@ required_files=( "docs/SUPPLY_CHAIN_SECURITY.md" "docs/design-docs/core-beliefs.md" "docs/design-docs/index.md" + "docs/design-docs/record-and-replay-handoff.md" + "docs/design-docs/record-and-replay-official-golden-capture.md" + "docs/design-docs/record-and-replay-replication.md" "docs/product-specs/index.md" "docs/references/README.md" + "docs/references/codex-computer-use-reverse-engineering/record-and-replay-event-stream.md" + "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json" + "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json" + "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json" "docs/generated/README.md" "docs/exec-plans/templates/execution-plan.md" "docs/exec-plans/tech-debt-tracker.md" "docs/histories/template.md" "docs/releases/feature-release-notes.md" + "skills/open-computer-use-record-and-replay/SKILL.md" + "skills/open-computer-use/references/record-and-replay.md" ) missing=0 @@ -51,6 +60,195 @@ if ! grep -q "docs/" "${repo_root}/AGENTS.md"; then missing=1 fi +doc_contracts=( + $'README.md\trecord-and-replay-baseline-audit\t英文 README 应声明 Record & Replay baseline audit artifact 入口' + $'README.md\trecord-and-replay-official-golden-capture-packet\t英文 README 应声明 official golden capture packet 入口' + $'README.md\tfinalize-record-and-replay-official-capture-packet.py\t英文 README 应声明 hosted official capture finalizer 入口' + $'README.md\t--allow-strict-official-golden-missing\t英文 README 应声明 strict expected-failure summary audit 入口' + $'README.md\tstrict-expected-failure-audit.sh\t英文 README 应声明 capture packet strict expected-failure wrapper' + $'README.zh-CN.md\trecord-and-replay-baseline-audit\t中文 README 应声明 Record & Replay baseline audit artifact 入口' + $'README.zh-CN.md\trecord-and-replay-official-golden-capture-packet\t中文 README 应声明 official golden capture packet 入口' + $'README.zh-CN.md\tfinalize-record-and-replay-official-capture-packet.py\t中文 README 应声明 hosted official capture finalizer 入口' + $'README.zh-CN.md\t--allow-strict-official-golden-missing\t中文 README 应声明 strict expected-failure summary audit 入口' + $'README.zh-CN.md\tstrict-expected-failure-audit.sh\t中文 README 应声明 capture packet strict expected-failure wrapper' + $'docs/design-docs/record-and-replay-handoff.md\tingest-ocu-candidates.sh\tRecord & Replay handoff 短入口应声明 recommended packet set 的 OCU candidate 批量 handoff' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tingest-ocu-candidates.sh\t官方 golden 采集入口应声明 recommended packet set 的 OCU candidate 批量 handoff' + $'docs/design-docs/record-and-replay-official-golden-capture.md\trecord-and-replay-official-golden-gate-audit\t官方 golden 采集入口应声明采样后刷新 strict audit artifact' + $'docs/design-docs/record-and-replay-replication.md\tingest-ocu-candidates.sh\tRecord & Replay 设计文档应声明 recommended packet set 的 OCU candidate handoff' + $'docs/CICD.md\tingest-ocu-candidates.sh\tCI 文档应声明 baseline artifact audit 会守住 recommended packet set 的 OCU candidate handoff' + $'docs/design-docs/record-and-replay-handoff.md\tsourceRepoBaselineAudit\tRecord & Replay handoff 短入口应声明 standalone source repo audit manifest handoff' + $'docs/design-docs/record-and-replay-handoff.md\tverifiesSummaryEnvVarIsolation\tRecord & Replay handoff 应声明 standalone source repo audit manifest 的变量隔离合同' + $'docs/design-docs/record-and-replay-handoff.md\trecord-and-replay-official-golden-gate-summary.json\tRecord & Replay handoff 应声明 strict audit 使用独立 summary artifact' + $'docs/design-docs/record-and-replay-handoff.md\tstrictOfficialGoldenExpectedFailureAudit\tRecord & Replay handoff 应声明 standalone manifest 的 strict expected-failure audit 字段' + $'docs/design-docs/record-and-replay-handoff.md\tverify-manifest.py\tRecord & Replay handoff 应声明 standalone manifest self-check 脚本' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedManifestContract\tRecord & Replay handoff 应声明 standalone manifest contract evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialFixtureSetComparePolicyManifest\tRecord & Replay handoff 应声明 standalone official fixture set compare policy evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryEvidence\tRecord & Replay handoff 应声明 standalone source baseline summary evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryOfficialGoldenState\tRecord & Replay handoff 应声明 standalone source summary 会守住 official golden 状态自洽 evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryCapturePacketSetContractManifest\tRecord & Replay handoff 应声明 standalone source summary 会守住 capture packet set contract evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\tRecord & Replay handoff 应声明 standalone source summary 会守住 capture packet post-capture workflow evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\tRecord & Replay handoff 应声明 standalone source summary 会守住 packet set post-capture workflow evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\tRecord & Replay handoff 应声明 standalone source summary 会守住 capture packet strict audit handoff evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\tRecord & Replay handoff 应声明 standalone source summary 会守住 capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-handoff.md\tevidence/source-baseline-summary.json\tRecord & Replay handoff 应声明 standalone copied baseline summary evidence' + $'docs/design-docs/record-and-replay-handoff.md\tofficialFixtureSetGate.sameScenarioComparePolicy\tRecord & Replay handoff 应声明 standalone official fixture set compare policy manifest' + $'docs/design-docs/record-and-replay-replication.md\tsourceRepoBaselineAudit\tRecord & Replay 设计文档应声明 standalone source repo audit manifest handoff' + $'docs/design-docs/record-and-replay-replication.md\tverifiesSummaryEnvVarIsolation\tRecord & Replay 设计文档应声明 standalone source repo audit manifest 的变量隔离合同' + $'docs/design-docs/record-and-replay-replication.md\tRNR_OFFICIAL_GOLDEN_SUMMARY_JSON\tRecord & Replay 设计文档应声明 strict audit summary 覆盖变量' + $'docs/design-docs/record-and-replay-replication.md\tstrictOfficialGoldenExpectedFailureAudit\tRecord & Replay 设计文档应声明 standalone manifest 的 strict expected-failure audit 字段' + $'docs/design-docs/record-and-replay-replication.md\tmanifestContract\tRecord & Replay 设计文档应声明 standalone manifest contract check' + $'docs/design-docs/record-and-replay-replication.md\tcheckedManifestContract\tRecord & Replay 设计文档应声明 standalone manifest contract evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialFixtureSetComparePolicyManifest\tRecord & Replay 设计文档应声明 standalone official fixture set compare policy evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryEvidence\tRecord & Replay 设计文档应声明 standalone source baseline summary evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryOfficialGoldenState\tRecord & Replay 设计文档应声明 standalone source summary 会守住 official golden 状态自洽 evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryCapturePacketSetContractManifest\tRecord & Replay 设计文档应声明 standalone source summary 会守住 capture packet set contract evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\tRecord & Replay 设计文档应声明 standalone source summary 会守住 capture packet post-capture workflow evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\tRecord & Replay 设计文档应声明 standalone source summary 会守住 packet set post-capture workflow evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\tRecord & Replay 设计文档应声明 standalone source summary 会守住 capture packet strict audit handoff evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\tRecord & Replay 设计文档应声明 standalone source summary 会守住 capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-replication.md\tcopiedBaselineSummaryEvidence\tRecord & Replay 设计文档应声明 standalone copied baseline summary evidence manifest' + $'docs/design-docs/record-and-replay-replication.md\tofficialFixtureSetGate.sameScenarioComparePolicy\tRecord & Replay 设计文档应声明 standalone official fixture set compare policy manifest' + $'docs/design-docs/record-and-replay-handoff.md\t--allow-strict-official-golden-missing\tRecord & Replay handoff 应声明 strict expected-failure summary audit 入口' + $'docs/design-docs/record-and-replay-replication.md\t--allow-strict-official-golden-missing\tRecord & Replay 设计文档应声明 strict expected-failure summary audit 入口' + $'docs/ARCHITECTURE.md\trecord-and-replay-baseline-audit\t架构文档应声明 standalone repo source baseline audit handoff' + $'docs/ARCHITECTURE.md\tverifiesSummaryEnvVarIsolation\t架构文档应声明 standalone source repo audit manifest 的变量隔离合同' + $'docs/ARCHITECTURE.md\tstrictOfficialGoldenExpectedFailureAudit\t架构文档应声明 standalone manifest 的 strict expected-failure audit 字段' + $'docs/ARCHITECTURE.md\tverify-manifest.py\t架构文档应声明 standalone manifest self-check 脚本' + $'docs/ARCHITECTURE.md\tmanifestContract\t架构文档应声明 standalone manifest contract check' + $'docs/ARCHITECTURE.md\tcheckedOfficialFixtureSetComparePolicyManifest\t架构文档应声明 standalone official fixture set compare policy evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryEvidence\t架构文档应声明 standalone source baseline summary evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryOfficialGoldenState\t架构文档应声明 standalone source summary 会守住 official golden 状态自洽 evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryCapturePacketSetContractManifest\t架构文档应声明 standalone source summary 会守住 capture packet set contract evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\t架构文档应声明 standalone source summary 会守住 capture packet post-capture workflow evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\t架构文档应声明 standalone source summary 会守住 packet set post-capture workflow evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\t架构文档应声明 standalone source summary 会守住 capture packet strict audit handoff evidence' + $'docs/ARCHITECTURE.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\t架构文档应声明 standalone source summary 会守住 capture packet strict expected-failure audit handoff evidence' + $'docs/ARCHITECTURE.md\tverify-package-artifact.py\t架构文档应声明 standalone generated repo 会运行 package artifact self-check' + $'docs/ARCHITECTURE.md\tcheckedPackageArtifact\t架构文档应声明 baseline artifact audit 会守住 standalone package artifact evidence' + $'docs/ARCHITECTURE.md\tcopiedBaselineSummaryEvidence\t架构文档应声明 standalone copied baseline summary evidence manifest' + $'docs/ARCHITECTURE.md\tofficialFixtureSetGate.sameScenarioComparePolicy\t架构文档应声明 standalone official fixture set compare policy manifest' + $'docs/ARCHITECTURE.md\t--allow-strict-official-golden-missing\t架构文档应声明 strict expected-failure summary audit 入口' + $'docs/CICD.md\tcheckedOfficialEvidenceAuditManifest\tCI 文档应声明 baseline artifact audit 会守住 standalone source audit manifest evidence' + $'docs/CICD.md\tcheckedManifestContract\tCI 文档应声明 baseline artifact audit 会守住 standalone manifest contract evidence' + $'docs/CICD.md\tcheckedOfficialFixtureSetComparePolicyManifest\tCI 文档应声明 baseline artifact audit 会守住 standalone official fixture set compare policy evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryEvidence\tCI 文档应声明 baseline artifact audit 会守住 standalone source baseline summary evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryOfficialGoldenState\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary official golden 状态自洽 evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryCapturePacketSetContractManifest\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary capture packet set contract evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary capture packet post-capture workflow evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary packet set post-capture workflow evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary capture packet strict audit handoff evidence' + $'docs/CICD.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\tCI 文档应声明 baseline artifact audit 会守住 standalone source summary capture packet strict expected-failure audit handoff evidence' + $'docs/CICD.md\tverify-package-artifact.py\tCI 文档应声明 standalone generated repo 会运行 package artifact self-check' + $'docs/CICD.md\tcheckedPackageArtifact\tCI 文档应声明 baseline artifact audit 会守住 standalone package artifact evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryOfficialGoldenState\t质量文档应声明 baseline artifact audit 会守住 standalone source summary official golden 状态自洽 evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryCapturePacketSetContractManifest\t质量文档应声明 baseline artifact audit 会守住 standalone source summary capture packet set contract evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\t质量文档应声明 baseline artifact audit 会守住 standalone source summary capture packet post-capture workflow evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\t质量文档应声明 baseline artifact audit 会守住 standalone source summary packet set post-capture workflow evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\t质量文档应声明 baseline artifact audit 会守住 standalone source summary capture packet strict audit handoff evidence' + $'docs/QUALITY_SCORE.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\t质量文档应声明 baseline artifact audit 会守住 standalone source summary capture packet strict expected-failure audit handoff evidence' + $'docs/QUALITY_SCORE.md\tverify-package-artifact.py\t质量文档应声明 standalone generated repo 会运行 package artifact self-check' + $'docs/QUALITY_SCORE.md\tcheckedPackageArtifact\t质量文档应声明 baseline artifact audit 会守住 standalone package artifact evidence' + $'docs/CICD.md\tcopiedBaselineSummaryEvidence\tCI 文档应声明 standalone copied baseline summary evidence manifest' + $'docs/CICD.md\tofficialFixtureSetGate.sameScenarioComparePolicy\tCI 文档应声明 standalone official fixture set compare policy manifest' + $'docs/CICD.md\tverify-source-baseline-summary.py\tCI 文档应声明 standalone generated repo 会运行 source baseline summary self-check' + $'docs/design-docs/record-and-replay-handoff.md\tverify-package-artifact.py\tRecord & Replay handoff 应声明 standalone generated repo 会运行 package artifact self-check' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedPackageArtifact\tRecord & Replay handoff 应声明 standalone package artifact evidence' + $'docs/design-docs/record-and-replay-replication.md\tverify-package-artifact.py\tRecord & Replay 设计文档应声明 standalone generated repo 会运行 package artifact self-check' + $'docs/design-docs/record-and-replay-replication.md\tcheckedPackageArtifact\tRecord & Replay 设计文档应声明 standalone package artifact evidence' + $'docs/CICD.md\tMissing Record & Replay baseline summary artifact\tCI 文档应声明 npm package build 缺 baseline summary artifact 时失败' + $'docs/design-docs/record-and-replay-handoff.md\tMissing Record & Replay baseline summary artifact\tRecord & Replay handoff 应声明 npm package build 缺 baseline summary artifact 时失败' + $'docs/design-docs/record-and-replay-replication.md\tMissing Record & Replay baseline summary artifact\tRecord & Replay 设计文档应声明 npm package build 缺 baseline summary artifact 时失败' + $'docs/CICD.md\tverify-manifest.py\tCI 文档应声明 standalone generated repo 会运行 manifest self-check' + $'docs/CICD.md\tverifiesSummaryEnvVarIsolation\tCI 文档应声明 standalone source audit manifest 的变量隔离合同' + $'docs/CICD.md\tcheckedStandaloneNextActionBaselineAuditCommand\tCI 文档应声明 baseline artifact audit 会守住 standalone nextActions 的 source baseline audit 命令' + $'docs/QUALITY_SCORE.md\tcheckedMissingGoldenNextActionStrictAuditStep\t质量文档应声明 baseline artifact audit 会守住 missing golden nextActions 的 strict audit 命令' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedMissingGoldenNextActionStrictAuditStep\t执行计划应声明 baseline artifact audit 会守住 missing golden nextActions 的 strict audit 命令' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcapture-contract.json\t官方 golden 采集入口应声明 capture packet semantic contract' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketStrictAuditHandoff\t官方 golden 采集入口应声明 capture packet strict audit handoff evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\t官方 golden 采集入口应声明 capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tpostCaptureWorkflow\t官方 golden 采集入口应声明 capture packet post-capture workflow contract' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tverify-workflow.sh\t官方 golden 采集入口应声明 capture packet workflow verifier' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tfinalize-record-and-replay-official-capture-packet.py\t官方 golden 采集入口应声明 hosted official capture finalizer' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tfinalStatusResponseShape\t官方 golden 采集入口应声明 hosted final status response shape' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\t官方 golden 采集入口应声明 capture packet post-capture workflow evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketWorkflowVerifier\t官方 golden 采集入口应声明 capture packet workflow verifier evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\t官方 golden 采集入口应声明 packet set post-capture workflow evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\t官方 golden 采集入口应声明 packet set workflow verifier evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketInputSemanticGuard\tRecord & Replay 设计文档应声明 capture packet semantic guard evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketStrictAuditHandoff\tRecord & Replay 设计文档应声明 capture packet strict audit handoff evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\tRecord & Replay 设计文档应声明 capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\tRecord & Replay 设计文档应声明 capture packet post-capture workflow evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketWorkflowVerifier\tRecord & Replay 设计文档应声明 capture packet workflow verifier evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\tRecord & Replay 设计文档应声明 packet set post-capture workflow evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\tRecord & Replay 设计文档应声明 packet set workflow verifier evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketInputSemanticGuard\tRecord & Replay handoff 应声明 capture packet semantic guard evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketStrictAuditHandoff\tRecord & Replay handoff 应声明 capture packet strict audit handoff evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\tRecord & Replay handoff 应声明 capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\tRecord & Replay handoff 应声明 capture packet post-capture workflow evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketWorkflowVerifier\tRecord & Replay handoff 应声明 capture packet workflow verifier evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\tRecord & Replay handoff 应声明 packet set post-capture workflow evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\tRecord & Replay handoff 应声明 packet set workflow verifier evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketInputSemanticGuard\t架构文档应声明 capture packet semantic guard evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketStrictAuditHandoff\t架构文档应声明 capture packet strict audit handoff evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\t架构文档应声明 capture packet post-capture workflow evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketWorkflowVerifier\t架构文档应声明 capture packet workflow verifier evidence' + $'docs/ARCHITECTURE.md\tfinalize-record-and-replay-official-capture-packet.py\t架构文档应声明 hosted official capture finalizer' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\t架构文档应声明 packet set post-capture workflow evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\t架构文档应声明 packet set workflow verifier evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketInputSemanticGuard\tCI 文档应声明 baseline artifact audit 会守住 capture packet semantic guard evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketStrictAuditHandoff\tCI 文档应声明 baseline artifact audit 会守住 capture packet strict audit handoff evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\tCI 文档应声明 baseline artifact audit 会守住 capture packet strict expected-failure audit handoff evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\tCI 文档应声明 baseline artifact audit 会守住 capture packet post-capture workflow evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketWorkflowVerifier\tCI 文档应声明 baseline artifact audit 会守住 capture packet workflow verifier evidence' + $'docs/CICD.md\trecord-and-replay-official-capture-finalizer-smoke\tCI 文档应声明 hosted official capture finalizer smoke' + $'docs/CICD.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\tCI 文档应声明 baseline artifact audit 会守住 packet set post-capture workflow evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\tCI 文档应声明 baseline artifact audit 会守住 packet set workflow verifier evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketInputSemanticGuard\t质量文档应声明 baseline artifact audit 会守住 capture packet semantic guard evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketStrictAuditHandoff\t质量文档应声明 baseline artifact audit 会守住 capture packet strict audit handoff evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\t质量文档应声明 baseline artifact audit 会守住 capture packet strict expected-failure audit handoff evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\t质量文档应声明 baseline artifact audit 会守住 capture packet post-capture workflow evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketWorkflowVerifier\t质量文档应声明 baseline artifact audit 会守住 capture packet workflow verifier evidence' + $'docs/QUALITY_SCORE.md\tfinalize-record-and-replay-official-capture-packet.py\t质量文档应声明 hosted official capture finalizer' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\t质量文档应声明 baseline artifact audit 会守住 packet set post-capture workflow evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\t质量文档应声明 baseline artifact audit 会守住 packet set workflow verifier evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketInputSemanticGuard\t执行计划应声明 capture packet semantic guard evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketStrictAuditHandoff\t执行计划应声明 capture packet strict audit handoff evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketStrictExpectedFailureAuditHandoff\t执行计划应声明 capture packet strict expected-failure audit handoff evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketPostCaptureWorkflow\t执行计划应声明 capture packet post-capture workflow evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketWorkflowVerifier\t执行计划应声明 capture packet workflow verifier evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketSetPostCaptureWorkflow\t执行计划应声明 packet set post-capture workflow evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketSetWorkflowVerifier\t执行计划应声明 packet set workflow verifier evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflow\t执行计划应声明 standalone source summary capture packet post-capture workflow evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketWorkflowVerifier\t执行计划应声明 standalone source summary capture packet workflow verifier evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow\t执行计划应声明 standalone source summary packet set post-capture workflow evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketSetWorkflowVerifier\t执行计划应声明 standalone source summary packet set workflow verifier evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketStrictAuditHandoff\t执行计划应声明 standalone source summary capture packet strict audit handoff evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff\t执行计划应声明 standalone source summary capture packet strict expected-failure audit handoff evidence' + $'docs/design-docs/record-and-replay-official-golden-capture.md\tcheckedOfficialCapturePacketSetContractManifest\t官方 golden 采集入口应声明 capture packet set contract manifest evidence' + $'docs/design-docs/record-and-replay-replication.md\tcheckedOfficialCapturePacketSetContractManifest\tRecord & Replay 设计文档应声明 capture packet set contract manifest evidence' + $'docs/design-docs/record-and-replay-handoff.md\tcheckedOfficialCapturePacketSetContractManifest\tRecord & Replay handoff 应声明 capture packet set contract manifest evidence' + $'docs/ARCHITECTURE.md\tcheckedOfficialCapturePacketSetContractManifest\t架构文档应声明 capture packet set contract manifest evidence' + $'docs/CICD.md\tcheckedOfficialCapturePacketSetContractManifest\tCI 文档应声明 baseline artifact audit 会守住 capture packet set contract manifest evidence' + $'docs/QUALITY_SCORE.md\tcheckedOfficialCapturePacketSetContractManifest\t质量文档应声明 baseline artifact audit 会守住 capture packet set contract manifest evidence' + $'docs/exec-plans/active/20260626-record-and-replay-event-stream.md\tcheckedOfficialCapturePacketSetContractManifest\t执行计划应声明 capture packet set contract manifest evidence' + $'docs/CICD.md\tstrictOfficialGoldenExpectedFailureAudit\tCI 文档应声明 standalone manifest 的 strict expected-failure audit 字段' + $'docs/CICD.md\t--allow-strict-official-golden-missing\tCI 文档应声明 strict expected-failure summary audit 入口' + $'docs/design-docs/record-and-replay-handoff.md\tOPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH\tRecord & Replay handoff 应声明 notify callback 传递 suppressed events path' + $'docs/design-docs/record-and-replay-replication.md\tcheckedNotifySuppressedEventsPathEnv\tRecord & Replay 设计文档应声明 baseline 会守住 notify suppressed events path evidence' + $'docs/CICD.md\tcheckedNotifySuppressedEventsPathEnv\tCI 文档应声明 baseline artifact audit 会守住 notify suppressed events path evidence' +) + +for contract in "${doc_contracts[@]}"; do + IFS=$'\t' read -r path needle message <<<"${contract}" + if [[ ! -f "${repo_root}/${path}" ]]; then + continue + fi + if ! grep -Fq -- "${needle}" "${repo_root}/${path}"; then + echo "${message}: ${path} 缺少 ${needle}" + missing=1 + fi +done + if [[ "${missing}" -ne 0 ]]; then exit 1 fi diff --git a/scripts/check-event-stream-golden-readiness.py b/scripts/check-event-stream-golden-readiness.py new file mode 100755 index 00000000..0fea2ed9 --- /dev/null +++ b/scripts/check-event-stream-golden-readiness.py @@ -0,0 +1,697 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import pathlib +import sys +from collections import Counter +from typing import Any + + +ACTION_EVENT_TYPES = { + "keyboard.shortcut", + "keyboard.submit", + "keyboard.text_input", + "mouse.click", + "mouse.context_menu", + "mouse.drag", + "terminal.value_changed", +} + +FINAL_STATES = {"stopped", "cancelled"} + +MCP_TOOL_EVIDENCE_DEFAULTS = { + "requested": False, + "responded": False, + "hasResult": False, + "hasError": False, + "timedOut": False, +} + +MCP_RESPONSE_SHAPES = { + "startResponseShape": "event_stream_start", + "repeatStartResponseShape": "event_stream_start", + "statusResponseShape": "event_stream_status", + "stopResponseShape": "event_stream_stop", + "repeatStopResponseShape": "event_stream_stop", + "finalStatusResponseShape": "event_stream_status", +} + +MCP_RESPONSE_SHAPE_ALIASES = { + "startResponse": "startResponseShape", + "repeatStartResponse": "repeatStartResponseShape", + "statusResponse": "statusResponseShape", + "stopResponse": "stopResponseShape", + "repeatStopResponse": "repeatStopResponseShape", + "finalStatusResponse": "finalStatusResponseShape", +} + +HANDOFF_PATH_KEYS = ( + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", +) + + +def load_json(path: pathlib.Path) -> dict[str, Any]: + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def read_jsonl(path: pathlib.Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + if not path.exists(): + raise ValueError(f"missing JSONL file: {path}") + for index, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSONL in {path}:{index}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}:{index}") + records.append(value) + return records + + +def event_kind(event: dict[str, Any]) -> str | None: + kind = event.get("kind") + if isinstance(kind, str): + return kind + event_type = event.get("type") + if isinstance(event_type, str): + return event_type + return None + + +def resolve_path(base: pathlib.Path, raw: Any, fallback: pathlib.Path) -> pathlib.Path: + if isinstance(raw, str) and raw: + candidate = pathlib.Path(raw) + if candidate.is_absolute(): + return candidate + return base / candidate + return fallback + + +def handoff_path_evidence(session_dir: pathlib.Path, metadata: dict[str, Any], key: str) -> dict[str, Any]: + raw = metadata.get(key) + evidence: dict[str, Any] = { + "value": raw, + "resolvedPath": None, + "exists": None, + } + if isinstance(raw, str) and raw: + path = pathlib.Path(raw) + resolved_path = path if path.is_absolute() else session_dir / path + evidence["resolvedPath"] = str(resolved_path) + evidence["exists"] = resolved_path.exists() + return evidence + + +def first_string(value: dict[str, Any], keys: list[str]) -> str | None: + for key in keys: + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + return None + + +def session_alias_compatible(metadata: dict[str, Any], session_alias: dict[str, Any]) -> bool: + if metadata == session_alias: + return True + + metadata_session_id = first_string(metadata, ["sessionId", "sessionID", "id"]) + session_id = first_string(session_alias, ["id", "sessionId", "sessionID"]) + if metadata_session_id is None or session_id is None or metadata_session_id != session_id: + return False + + metadata_started_at = metadata.get("startedAt") + if isinstance(metadata_started_at, str) and metadata_started_at != session_alias.get("startedAt"): + return False + + for key in ["endedAt", "endReason"]: + session_value = session_alias.get(key) + if isinstance(session_value, str) and session_value != metadata.get(key): + return False + + session_events_path = session_alias.get("eventsPath") + metadata_events_path = metadata.get("eventsPath") + if not isinstance(session_events_path, str) or not session_events_path: + return False + if not isinstance(metadata_events_path, str) or not metadata_events_path: + return False + if ( + session_events_path != metadata_events_path + and pathlib.Path(session_events_path).name != pathlib.Path(metadata_events_path).name + ): + return False + + return True + + +def resolve_recording(path: pathlib.Path) -> dict[str, Any]: + if path.is_dir(): + metadata_json_path = path / "metadata.json" + session_alias_path = path / "session.json" + metadata_json = load_json(metadata_json_path) if metadata_json_path.exists() else None + session_alias = load_json(session_alias_path) if session_alias_path.exists() else None + if metadata_json is not None: + metadata_path = metadata_json_path + metadata = metadata_json + elif session_alias is not None: + metadata_path = session_alias_path + metadata = session_alias + else: + raise ValueError(f"recording directory has no metadata.json or session.json: {path}") + session_dir = path + else: + if not path.exists(): + raise ValueError(f"path does not exist: {path}") + if path.name not in {"metadata.json", "session.json"}: + raise ValueError("input must be a recording directory, metadata.json, or session.json") + session_dir = path.parent + metadata_json_path = session_dir / "metadata.json" + session_alias_path = session_dir / "session.json" + metadata_json = load_json(metadata_json_path) if metadata_json_path.exists() else None + session_alias = load_json(session_alias_path) if session_alias_path.exists() else None + if path.name == "session.json" and metadata_json is not None: + metadata_path = metadata_json_path + metadata = metadata_json + else: + metadata_path = path + metadata = load_json(path) + + events_path = resolve_path(session_dir, metadata.get("eventsPath"), session_dir / "events.jsonl") + suppressed_path = resolve_path( + session_dir, + metadata.get("suppressedEventsPath"), + session_dir / "suppressed.jsonl", + ) + manifest_path = session_dir / "fixture-manifest.json" + manifest = load_json(manifest_path) if manifest_path.exists() else None + mcp_transcript_path = None + if isinstance(manifest, dict): + files = manifest.get("files") + if isinstance(files, dict) and isinstance(files.get("mcpTranscript"), str): + mcp_transcript_path = resolve_path(session_dir, files["mcpTranscript"], session_dir / "mcp-transcript.json") + if mcp_transcript_path is None and (session_dir / "mcp-transcript.json").exists(): + mcp_transcript_path = session_dir / "mcp-transcript.json" + + try: + events = read_jsonl(events_path) + events_error = None + except ValueError as error: + events = [] + events_error = str(error) + + return { + "sessionDir": session_dir, + "metadataPath": metadata_path, + "metadataJsonPath": metadata_json_path if metadata_json_path.exists() else None, + "sessionAliasPath": session_alias_path if session_alias_path.exists() else None, + "metadata": metadata, + "metadataJson": metadata_json, + "sessionAlias": session_alias, + "eventsPath": events_path, + "events": events, + "eventsReadError": events_error, + "suppressedPath": suppressed_path, + "suppressed": read_jsonl(suppressed_path) if suppressed_path.exists() else [], + "manifestPath": manifest_path if manifest_path.exists() else None, + "manifest": manifest, + "mcpTranscriptPath": mcp_transcript_path, + "mcpTranscript": load_json(mcp_transcript_path) if mcp_transcript_path else None, + } + + +def iter_dicts(value: Any): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from iter_dicts(child) + elif isinstance(value, list): + for child in value: + yield from iter_dicts(child) + + +def accessibility_payloads(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + payloads: list[dict[str, Any]] = [] + for event in events: + for value in iter_dicts(event): + payload = value.get("accessibilityInspectorPayload") + if isinstance(payload, dict): + payloads.append(payload) + official_ax_payload = value.get("ax") + if isinstance(official_ax_payload, dict): + payloads.append(official_ax_payload) + if event_kind(event) == "AX.focusedWindowChanged": + payload = event.get("accessibilityInspectorPayload") + if isinstance(payload, dict) and payload not in payloads: + payloads.append(payload) + official_ax_payload = event.get("ax") + if isinstance(official_ax_payload, dict) and official_ax_payload not in payloads: + payloads.append(official_ax_payload) + return payloads + + +def is_full_ax_payload(payload: dict[str, Any]) -> bool: + return ( + payload.get("kind") == "full" + or payload.get("mode") in {"full", "fullTree"} + or payload.get("diffFromPrevious") is False + or isinstance(payload.get("fullTree"), (list, str)) + ) + + +def is_full_tree_payload(payload: dict[str, Any]) -> bool: + return ( + isinstance(payload.get("fullTree"), (list, str)) and bool(payload.get("fullTree")) + ) or payload.get("mode") == "fullTree" + + +def response_evidence(response: dict[str, Any], requested: bool = True) -> dict[str, bool]: + return { + "requested": requested, + "responded": True, + "hasResult": "result" in response, + "hasError": "error" in response, + "timedOut": "timeout" in response, + } + + +def merge_tool_evidence(entry: dict[str, bool], response: dict[str, Any]) -> None: + entry["responded"] = True + entry["hasResult"] = bool(entry.get("hasResult")) or "result" in response + entry["hasError"] = bool(entry.get("hasError")) or "error" in response + saw_timeout = bool(entry.get("timedOut")) or "timeout" in response + entry["timedOut"] = saw_timeout and not (entry["hasResult"] or entry["hasError"]) + + +def mcp_response_evidence(transcript: Any) -> tuple[dict[str, Any], dict[str, Any]]: + evidence: dict[str, dict[str, bool]] = {} + shape_evidence: dict[str, dict[str, Any]] = {} + if not isinstance(transcript, dict): + return evidence, shape_evidence + + response_keys: dict[str, str] = dict(MCP_RESPONSE_SHAPES) + response_keys.update( + {raw_key: MCP_RESPONSE_SHAPES[shape_key] for raw_key, shape_key in MCP_RESPONSE_SHAPE_ALIASES.items()} + ) + for response_key, tool_name in response_keys.items(): + response = transcript.get(response_key) + if isinstance(response, dict): + entry = evidence.setdefault( + tool_name, + dict(MCP_TOOL_EVIDENCE_DEFAULTS), + ) + entry["requested"] = True + merge_tool_evidence(entry, response) + shape_key = MCP_RESPONSE_SHAPE_ALIASES.get(response_key, response_key) + if shape_key in MCP_RESPONSE_SHAPES: + shape_evidence[shape_key] = { + **response_evidence(response), + "toolName": tool_name, + } + + request_id_to_tool: dict[Any, str] = {} + for event in transcript.get("transcript", []): + if not isinstance(event, dict): + continue + message = event.get("message") + if not isinstance(message, dict): + continue + if event.get("direction") == "send" and message.get("method") == "tools/call": + params = message.get("params") + tool_name = params.get("name") if isinstance(params, dict) else None + if isinstance(tool_name, str): + request_id_to_tool[message.get("id")] = tool_name + entry = evidence.setdefault(tool_name, dict(MCP_TOOL_EVIDENCE_DEFAULTS)) + entry["requested"] = True + elif event.get("direction") == "receive" and message.get("id") in request_id_to_tool: + tool_name = request_id_to_tool[message.get("id")] + entry = evidence.setdefault(tool_name, dict(MCP_TOOL_EVIDENCE_DEFAULTS)) + entry["requested"] = True + merge_tool_evidence(entry, message) + + for event in transcript.get("transcriptShape", []): + if not isinstance(event, dict): + continue + method = event.get("method") + if method == "tools/call": + continue + # Shape-only probe fixtures cannot recover the tool name from method alone. + + return evidence, shape_evidence + + +def check_recording(path: pathlib.Path, args: argparse.Namespace) -> dict[str, Any]: + errors: list[str] = [] + warnings: list[str] = [] + try: + recording = resolve_recording(path) + except ValueError as error: + return { + "ok": False, + "path": str(path), + "errors": [str(error)], + "warnings": [], + } + + metadata = recording["metadata"] + metadata_json = recording["metadataJson"] + session_alias = recording["sessionAlias"] + events = recording["events"] + suppressed_events = recording["suppressed"] + manifest = recording["manifest"] + mcp_transcript = recording["mcpTranscript"] + event_types = [kind for event in events if (kind := event_kind(event))] + event_type_counts = Counter(event_types) + suppressed_event_types = [ + event_kind(event) + for event in suppressed_events + if event_kind(event) is not None + ] + suppressed_event_type_counts = Counter(suppressed_event_types) + action_event_count = sum(event_type_counts[event_type] for event_type in ACTION_EVENT_TYPES) + payloads = accessibility_payloads(events) + has_metadata_json = metadata_json is not None + has_session_alias = session_alias is not None + metadata_session_alias_complete = has_metadata_json and has_session_alias + metadata_session_alias_matches = ( + session_alias_compatible(metadata_json, session_alias) + if metadata_session_alias_complete + else None + ) + if metadata_session_alias_matches is False: + errors.append("metadata.json and session.json differ") + if args.require_session_alias: + if not has_metadata_json: + errors.append("missing metadata.json") + if not has_session_alias: + errors.append("missing session.json alias") + metadata_event_count = metadata.get("eventCount") + metadata_suppressed_event_count = metadata.get("suppressedEventCount") + metadata_event_count_matches = ( + metadata_event_count == len(events) + if isinstance(metadata_event_count, int) + else None + ) + metadata_suppressed_event_count_matches = ( + metadata_suppressed_event_count == len(suppressed_events) + if isinstance(metadata_suppressed_event_count, int) + else None + ) + if metadata_event_count_matches is False: + errors.append(f"eventCount={metadata_event_count} does not match events.jsonl lines={len(events)}") + if metadata_suppressed_event_count_matches is False: + errors.append( + "suppressedEventCount=" + f"{metadata_suppressed_event_count} does not match suppressed.jsonl lines={len(suppressed_events)}" + ) + if args.require_metadata_counts: + if not isinstance(metadata_event_count, int): + errors.append("missing metadata eventCount") + if not isinstance(metadata_suppressed_event_count, int): + errors.append("missing metadata suppressedEventCount") + + declared_paths = { + key: handoff_path_evidence(recording["sessionDir"], metadata, key) + for key in HANDOFF_PATH_KEYS + } + for key, evidence in declared_paths.items(): + raw_value = evidence["value"] + if raw_value is None: + continue + if not isinstance(raw_value, str) or not raw_value: + errors.append(f"{key} is not a non-empty string") + elif evidence["exists"] is False: + errors.append(f"{key} does not exist: {raw_value}") + if args.require_handoff_paths: + for key, evidence in declared_paths.items(): + if not isinstance(evidence["value"], str) or not evidence["value"]: + errors.append(f"missing {key}") + if recording["eventsReadError"] is not None: + errors.append(recording["eventsReadError"]) + + state = metadata.get("state") + started_events = [event for event in events if event_kind(event) == "session.started"] + first_event_type = event_kind(events[0]) if events else None + last_event_type = event_kind(events[-1]) if events else None + ended_events = [event for event in events if event_kind(event) == "session.ended"] + end_reasons = [ + event.get("endReason") + for event in ended_events + if isinstance(event.get("endReason"), str) + ] + if isinstance(metadata.get("endReason"), str): + end_reasons.append(metadata["endReason"]) + end_reason_counts = Counter(end_reasons) + if state not in FINAL_STATES: + errors.append(f"metadata state is not final: {state}") + if not started_events: + errors.append("missing session.started event") + elif len(started_events) > 1: + errors.append("multiple session.started events") + if started_events and first_event_type != "session.started": + errors.append("session.started is not the first event") + if not ended_events and not args.allow_no_session_ended: + errors.append("missing session.ended event") + elif len(ended_events) > 1: + errors.append("multiple session.ended events") + if ended_events and last_event_type != "session.ended": + errors.append("session.ended is not the final event") + if not args.allow_no_action and action_event_count == 0: + errors.append("missing action event; pass --allow-no-action only for lifecycle/status fixtures") + if not args.allow_no_ax_payload and not payloads: + errors.append("missing accessibilityInspectorPayload; pass --allow-no-ax-payload only for non-action fixtures") + + for required_type in args.require_event_type: + if event_type_counts[required_type] == 0: + errors.append(f"missing required event type: {required_type}") + if args.require_suppressed_events and not suppressed_events: + errors.append("missing suppressed events") + for required_type in args.require_suppressed_event_type: + if suppressed_event_type_counts[required_type] == 0: + errors.append(f"missing required suppressed event type: {required_type}") + for required_reason in args.require_end_reason: + if end_reason_counts[required_reason] == 0: + errors.append(f"missing required endReason: {required_reason}") + + has_full_tree_payload = any( + is_full_tree_payload(payload) + for payload in payloads + ) + has_full_ax_payload = any( + is_full_ax_payload(payload) + for payload in payloads + ) + has_diff_ax_payload = any(payload.get("kind") == "diff" or payload.get("diffFromPrevious") is True for payload in payloads) + has_cumulative_ax_diff = any( + payload.get("cumulativeDiffFromInitial") is True + or isinstance(payload.get("cumulativeRenderedText"), str) + for payload in payloads + ) + if args.require_full_ax_payload and not has_full_ax_payload: + errors.append("missing full AX payload") + if args.require_diff_payload and not has_diff_ax_payload: + errors.append("missing AX diff payload") + if args.require_cumulative_diff and not has_cumulative_ax_diff: + errors.append("missing cumulative AX diff payload") + + if args.require_fixture_manifest and manifest is None: + errors.append("missing fixture-manifest.json") + if args.require_source: + if manifest is None: + errors.append("cannot verify source without fixture-manifest.json") + elif manifest.get("source") != args.require_source: + errors.append(f"fixture source={manifest.get('source')} does not match required source={args.require_source}") + if args.require_official_plugin_version: + if manifest is None: + errors.append("cannot verify official plugin version without fixture-manifest.json") + elif manifest.get("officialPluginVersion") != args.require_official_plugin_version: + errors.append( + "fixture officialPluginVersion=" + f"{manifest.get('officialPluginVersion')} does not match required " + f"{args.require_official_plugin_version}" + ) + + if manifest is not None: + redaction = manifest.get("redaction") + if not isinstance(redaction, dict): + warnings.append("fixture manifest has no redaction block") + elif redaction.get("screenshotsCopied") is not False: + errors.append("fixture manifest must record screenshotsCopied=false for repository fixtures") + manifest_event_count = manifest.get("eventCount") + if isinstance(manifest_event_count, int) and manifest_event_count != len(events): + errors.append( + f"fixture manifest eventCount={manifest_event_count} does not match events.jsonl lines={len(events)}" + ) + manifest_suppressed_event_count = manifest.get("suppressedEventCount") + if isinstance(manifest_suppressed_event_count, int) and manifest_suppressed_event_count != len(suppressed_events): + errors.append( + "fixture manifest suppressedEventCount=" + f"{manifest_suppressed_event_count} does not match suppressed.jsonl lines={len(suppressed_events)}" + ) + + mcp_evidence, mcp_shape_evidence = mcp_response_evidence(mcp_transcript) + if args.require_mcp_transcript and mcp_transcript is None: + errors.append("missing mcp-transcript.json") + for tool_name in args.require_mcp_tool_response: + tool_evidence = mcp_evidence.get(tool_name) + if not tool_evidence: + errors.append(f"missing MCP response evidence for {tool_name}") + continue + if tool_evidence.get("timedOut"): + errors.append(f"MCP tool timed out: {tool_name}") + elif not tool_evidence.get("responded"): + errors.append(f"MCP tool did not respond: {tool_name}") + elif not (tool_evidence.get("hasResult") or tool_evidence.get("hasError")): + errors.append(f"MCP response has no result or error: {tool_name}") + for response_shape in args.require_mcp_response_shape: + shape_evidence = mcp_shape_evidence.get(response_shape) + if not shape_evidence: + errors.append(f"missing MCP response shape evidence: {response_shape}") + continue + if shape_evidence.get("timedOut"): + errors.append(f"MCP response shape timed out: {response_shape}") + elif not shape_evidence.get("responded"): + errors.append(f"MCP response shape did not respond: {response_shape}") + elif not (shape_evidence.get("hasResult") or shape_evidence.get("hasError")): + errors.append(f"MCP response shape has no result or error: {response_shape}") + + recommendation = "ready for golden comparison" + if errors: + recommendation = "capture or import a richer recording before using this as a golden baseline" + elif warnings: + recommendation = "usable, but review warnings before treating as authoritative" + + return { + "ok": not errors, + "path": str(path), + "sessionDir": str(recording["sessionDir"]), + "metadataPath": str(recording["metadataPath"]), + "metadataJsonPath": str(recording["metadataJsonPath"]) if recording["metadataJsonPath"] else None, + "sessionPath": str(recording["sessionAliasPath"]) if recording["sessionAliasPath"] else None, + "metadataSessionAliasComplete": metadata_session_alias_complete, + "metadataSessionAliasMatches": metadata_session_alias_matches, + "declaredPaths": declared_paths, + "eventsPath": str(recording["eventsPath"]), + "fixtureManifestPath": str(recording["manifestPath"]) if recording["manifestPath"] else None, + "mcpTranscriptPath": str(recording["mcpTranscriptPath"]) if recording["mcpTranscriptPath"] else None, + "source": manifest.get("source") if isinstance(manifest, dict) else None, + "officialPluginVersion": manifest.get("officialPluginVersion") if isinstance(manifest, dict) else None, + "state": state, + "endReason": metadata.get("endReason"), + "endReasons": dict(sorted(end_reason_counts.items())), + "eventCount": len(events), + "metadataEventCount": metadata_event_count, + "metadataEventCountMatches": metadata_event_count_matches, + "firstEventType": first_event_type, + "lastEventType": last_event_type, + "sessionStartedCount": len(started_events), + "sessionStartedIsFirst": first_event_type == "session.started" if first_event_type is not None else None, + "sessionEndedCount": len(ended_events), + "sessionEndedIsFinal": last_event_type == "session.ended" if last_event_type is not None else None, + "eventTypes": dict(sorted(event_type_counts.items())), + "actionEventCount": action_event_count, + "hasAccessibilityPayload": bool(payloads), + "hasFullAXPayload": has_full_ax_payload, + "hasFullTreePayload": has_full_tree_payload, + "hasDiffAXPayload": has_diff_ax_payload, + "hasCumulativeAXDiff": has_cumulative_ax_diff, + "mcpToolResponses": mcp_evidence, + "mcpResponseShapes": mcp_shape_evidence, + "suppressedEventCount": len(recording["suppressed"]), + "metadataSuppressedEventCount": metadata_suppressed_event_count, + "metadataSuppressedEventCountMatches": metadata_suppressed_event_count_matches, + "suppressedEventTypes": dict(sorted(suppressed_event_type_counts.items())), + "warnings": warnings, + "errors": errors, + "recommendation": recommendation, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Check whether Record & Replay recordings are rich enough to use as golden baselines." + ) + parser.add_argument("paths", nargs="+", type=pathlib.Path, help="Recording directories, metadata.json, or session.json files") + parser.add_argument("--require-fixture-manifest", action="store_true", help="Require fixture-manifest.json next to the recording files.") + parser.add_argument("--require-source", choices=["official", "ocu"], help="Require fixture-manifest.json source to match.") + parser.add_argument("--require-official-plugin-version", help="Require an exact fixture-manifest officialPluginVersion value.") + parser.add_argument("--require-event-type", action="append", default=[], help="Require at least one event of this type. May be repeated.") + parser.add_argument( + "--require-session-alias", + action="store_true", + help="Require both metadata.json and session.json, and require them to be compatible.", + ) + parser.add_argument( + "--require-metadata-counts", + action="store_true", + help="Require metadata eventCount and suppressedEventCount fields to be present and match file line counts.", + ) + parser.add_argument( + "--require-handoff-paths", + action="store_true", + help="Require metadataPath, eventsPath, sessionPath, and suppressedEventsPath to resolve to existing files.", + ) + parser.add_argument( + "--require-end-reason", + action="append", + default=[], + help="Require an endReason value in metadata.json/session.json or session.ended. May be repeated.", + ) + parser.add_argument("--require-suppressed-events", action="store_true", help="Require at least one suppressed event in suppressed.jsonl.") + parser.add_argument("--require-suppressed-event-type", action="append", default=[], help="Require at least one suppressed event of this type. May be repeated.") + parser.add_argument("--require-mcp-transcript", action="store_true", help="Require mcp-transcript.json next to the fixture files.") + parser.add_argument( + "--require-mcp-tool-response", + action="append", + default=[], + choices=["event_stream_start", "event_stream_status", "event_stream_stop"], + help="Require result/error response evidence for a Record & Replay MCP tool. May be repeated.", + ) + parser.add_argument( + "--require-mcp-response-shape", + action="append", + default=[], + choices=sorted(MCP_RESPONSE_SHAPES), + help=( + "Require result/error evidence for a specific probe response shape, such as " + "repeatStartResponseShape or finalStatusResponseShape. May be repeated." + ), + ) + parser.add_argument("--allow-no-action", action="store_true", help="Allow lifecycle/status fixtures without user action events.") + parser.add_argument("--allow-no-ax-payload", action="store_true", help="Allow fixtures without AX payloads.") + parser.add_argument("--allow-no-session-ended", action="store_true", help="Allow final-state samples without a session.ended event.") + parser.add_argument("--require-full-ax-payload", action="store_true", help="Require at least one full AX payload.") + parser.add_argument("--require-diff-payload", action="store_true", help="Require at least one diff AX payload.") + parser.add_argument("--require-cumulative-diff", action="store_true", help="Require at least one cumulative AX diff payload.") + args = parser.parse_args() + + recordings = [check_recording(path, args) for path in args.paths] + result = { + "ok": all(recording["ok"] for recording in recordings), + "recordings": recordings, + } + output = json.dumps(result, indent=2, sort_keys=True) + if result["ok"]: + print(output) + return 0 + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-event-stream-official-fixture-coverage.py b/scripts/check-event-stream-official-fixture-coverage.py new file mode 100755 index 00000000..ba1b0ce8 --- /dev/null +++ b/scripts/check-event-stream-official-fixture-coverage.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys + +from record_and_replay_scenarios import ( + DEFAULT_RECOMMENDED_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_recipe, +) + + +DEFAULT_FIXTURE_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) +DEFAULT_RECOMMENDED_SCENARIO_HELP = ", ".join(DEFAULT_RECOMMENDED_SCENARIOS) + + +def load_json(path: pathlib.Path) -> dict: + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def display_path(path: pathlib.Path, repo: pathlib.Path) -> str: + try: + return str(path.resolve().relative_to(repo.resolve())) + except ValueError: + return str(path) + + +def candidate_dirs(root: pathlib.Path) -> list[pathlib.Path]: + if (root / "fixture-manifest.json").exists(): + return [root] + if not root.exists(): + return [] + return sorted(path for path in root.iterdir() if (path / "fixture-manifest.json").exists()) + + +def run_fixture_set_readiness( + repo: pathlib.Path, + fixture_root: pathlib.Path, + required_scenarios: list[str], +) -> tuple[bool, dict]: + command = [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-set.py"), + "--official-root", + str(fixture_root), + ] + for scenario in required_scenarios: + command.extend(["--require-scenario", scenario]) + completed = subprocess.run( + command, + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"fixture set readiness command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def not_ready_official_scenarios(readiness_json: dict | None, required_scenarios: list[str]) -> list[str]: + if not isinstance(readiness_json, dict): + return [] + results = readiness_json.get("scenarioResults") + if not isinstance(results, list): + return [] + + not_ready = [] + for result in results: + if not isinstance(result, dict): + continue + scenario = result.get("scenario") + if not isinstance(scenario, str) or scenario not in required_scenarios: + continue + if not result.get("officialFixture"): + continue + readiness = result.get("officialReadiness") + if not isinstance(readiness, dict) or readiness.get("ok") is not True: + not_ready.append(scenario) + return sorted(set(not_ready)) + + +def inspect_fixture_coverage(args: argparse.Namespace) -> dict: + repo = pathlib.Path(__file__).resolve().parents[1] + root = args.fixture_root + required_scenarios = args.require_scenario or list(DEFAULT_REQUIRED_SCENARIOS) + recommended_scenarios = args.recommended_scenario or list(DEFAULT_RECOMMENDED_SCENARIOS) + official_fixtures = [] + scenarios = {} + errors = [] + + for fixture_dir in candidate_dirs(root): + try: + manifest = load_json(fixture_dir / "fixture-manifest.json") + except ValueError as error: + errors.append(str(error)) + continue + if manifest.get("source") != "official": + continue + scenario = manifest.get("scenario") + fixture_info = { + "path": display_path(fixture_dir, repo), + "name": manifest.get("name"), + "scenario": scenario if isinstance(scenario, str) else None, + "scenarioRecipeMatches": None, + "eventCount": manifest.get("eventCount"), + "officialPluginVersion": manifest.get("officialPluginVersion"), + } + official_fixtures.append(fixture_info) + if not isinstance(scenario, str) or not scenario: + errors.append(f"{fixture_info['path']}: official fixture missing scenario") + continue + actual_recipe = manifest.get("scenarioRecipe") + expected_recipe = scenario_recipe(scenario) + recipe_matches = actual_recipe == expected_recipe + fixture_info["scenarioRecipeMatches"] = recipe_matches + if not recipe_matches: + errors.append( + f"{fixture_info['path']}: fixture-manifest.json scenarioRecipe " + f"does not match scenario {scenario!r}" + ) + scenarios.setdefault(scenario, []).append(fixture_info["path"]) + + duplicate_scenarios = { + scenario: paths for scenario, paths in scenarios.items() if len(paths) > 1 + } + for scenario, paths in sorted(duplicate_scenarios.items()): + errors.append(f"duplicate official scenario {scenario!r}: {', '.join(paths)}") + + available_scenarios = sorted(scenarios) + missing_scenarios = sorted(set(required_scenarios) - set(available_scenarios)) + missing_recommended_scenarios = sorted(set(recommended_scenarios) - set(available_scenarios)) + scenario_coverage_ok = not missing_scenarios and not duplicate_scenarios and not errors + readiness_checked = args.check_readiness or args.require_readiness + readiness_ok: bool | None = None + readiness_json: dict | None = None + readiness_errors: list[str] = [] + not_ready_scenarios: list[str] = [] + + if readiness_checked: + if scenario_coverage_ok: + readiness_ok, readiness_json = run_fixture_set_readiness( + repo, + root, + required_scenarios, + ) + if not readiness_ok: + readiness_errors.append("required official fixture set failed readiness") + not_ready_scenarios = not_ready_official_scenarios( + readiness_json, + required_scenarios, + ) + if not readiness_ok and not not_ready_scenarios: + not_ready_scenarios = sorted(set(required_scenarios)) + else: + readiness_ok = False + readiness_json = { + "ok": False, + "errors": [ + "required official fixture set readiness skipped because scenario coverage is incomplete" + ], + } + + coverage_ok = scenario_coverage_ok and (not readiness_checked or readiness_ok is True) + errors.extend(readiness_errors) + ok = coverage_ok or ( + args.allow_missing + and not errors + and not duplicate_scenarios + and bool(missing_scenarios) + ) + return { + "ok": ok, + "coverageOk": coverage_ok, + "scenarioCoverageOk": scenario_coverage_ok, + "allowMissing": args.allow_missing, + "fixtureRoot": display_path(root, repo), + "requiredOfficialScenarios": required_scenarios, + "recommendedOfficialScenarios": recommended_scenarios, + "availableOfficialScenarios": available_scenarios, + "missingOfficialScenarios": missing_scenarios, + "missingRecommendedOfficialScenarios": missing_recommended_scenarios, + "notReadyOfficialScenarios": not_ready_scenarios, + "hasRequiredOfficialScenarioCoverage": scenario_coverage_ok, + "hasRequiredOfficialSuccessfulFixture": coverage_ok, + "hasRecommendedOfficialScenarioCoverage": not missing_recommended_scenarios, + "requiredOfficialReadinessChecked": readiness_checked, + "requiredOfficialReadinessOk": readiness_ok, + "officialFixtureSetReadiness": readiness_json, + "officialFixtures": official_fixtures, + "errors": errors, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Report whether the repository has the required official Record & Replay " + "successful recording fixture scenarios." + ) + ) + parser.add_argument( + "--fixture-root", + type=pathlib.Path, + default=DEFAULT_FIXTURE_ROOT, + help=f"Directory containing recording fixture subdirectories. Defaults to {DEFAULT_FIXTURE_ROOT}.", + ) + parser.add_argument( + "--require-scenario", + action="append", + default=[], + help=( + "Required official scenario tag. May be repeated. Defaults to " + f"{DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument( + "--recommended-scenario", + action="append", + default=[], + help=( + "Recommended official scenario tag to report. May be repeated. Defaults to " + f"{DEFAULT_RECOMMENDED_SCENARIO_HELP}. " + "Recommended coverage is informational and does not affect the exit code." + ), + ) + parser.add_argument( + "--allow-missing", + action="store_true", + help="Exit successfully when required scenarios are missing, while still reporting coverageOk=false.", + ) + parser.add_argument( + "--check-readiness", + action="store_true", + help=( + "When required scenarios are present, also run the official fixture set readiness gate. " + "Missing scenarios are reported as coverageOk=false." + ), + ) + parser.add_argument( + "--require-readiness", + action="store_true", + help=( + "Require the official fixture set readiness gate for required scenarios. " + "This implies --check-readiness." + ), + ) + args = parser.parse_args() + if args.require_readiness: + args.check_readiness = True + result = inspect_fixture_coverage(args) + output = json.dumps(result, indent=2, sort_keys=True) + if result["ok"]: + print(output) + return 0 + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-event-stream-official-fixture-set.py b/scripts/check-event-stream-official-fixture-set.py new file mode 100755 index 00000000..96e9b958 --- /dev/null +++ b/scripts/check-event-stream-official-fixture-set.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import subprocess +import sys + +from record_and_replay_scenarios import ( + DEFAULT_REQUIRED_SCENARIOS, + scenario_readiness_args, + scenario_recipe, +) + + +DEFAULT_FIXTURE_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) + +BASE_COMPARE_ARGS = ( + "--require-same-event-sequence", + "--require-same-schema", + "--require-same-metadata-keys", + "--require-same-metadata-values", + "--require-handoff-paths", + "--require-final-session-evidence", + "--require-semantic-fields", + "--require-mcp-response-shapes", + "--require-same-mcp-response-schema", + "--require-ax-diff-evidence", + "--require-same-ax-diff-markers", + "--require-same-suppressed-event-sequence", + "--require-same-suppressed-schema", + "--ignore-extra-schema", +) + + +def load_json(path: pathlib.Path) -> dict: + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def discover_fixtures(root: pathlib.Path, expected_source: str) -> tuple[dict[str, pathlib.Path], list[str]]: + if (root / "fixture-manifest.json").exists(): + candidates = [root] + elif root.exists(): + candidates = sorted(path for path in root.iterdir() if (path / "fixture-manifest.json").exists()) + else: + candidates = [] + + by_scenario: dict[str, pathlib.Path] = {} + errors: list[str] = [] + for fixture_dir in candidates: + try: + manifest = load_json(fixture_dir / "fixture-manifest.json") + except ValueError as error: + errors.append(str(error)) + continue + + source = manifest.get("source") + if source != expected_source: + errors.append(f"{fixture_dir}: expected source={expected_source}, got {source!r}") + scenario = manifest.get("scenario") + if not isinstance(scenario, str) or not scenario: + errors.append(f"{fixture_dir}: fixture-manifest.json missing scenario") + continue + actual_recipe = manifest.get("scenarioRecipe") + expected_recipe = scenario_recipe(scenario) + if actual_recipe != expected_recipe: + errors.append(f"{fixture_dir}: fixture-manifest.json scenarioRecipe does not match scenario {scenario!r}") + if scenario in by_scenario: + errors.append(f"duplicate scenario {scenario!r}: {by_scenario[scenario]} and {fixture_dir}") + continue + by_scenario[scenario] = fixture_dir + return by_scenario, errors + + +def run_json(command: list[str], cwd: pathlib.Path) -> tuple[bool, dict]: + completed = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def check_fixture_set(args: argparse.Namespace) -> dict: + repo = pathlib.Path(__file__).resolve().parents[1] + official, official_errors = discover_fixtures(args.official_root, "official") + candidate: dict[str, pathlib.Path] = {} + candidate_errors: list[str] = [] + if args.candidate_root is not None: + candidate, candidate_errors = discover_fixtures(args.candidate_root, "ocu") + + required_scenarios = args.require_scenario or list(DEFAULT_REQUIRED_SCENARIOS) + errors = [*official_errors, *candidate_errors] + scenario_results = [] + + for scenario in required_scenarios: + official_path = official.get(scenario) + scenario_result = { + "scenario": scenario, + "officialFixture": str(official_path) if official_path else None, + "officialReadiness": None, + "candidateFixture": None, + "candidateReadiness": None, + "candidateCompare": None, + } + if official_path is None: + errors.append(f"missing official fixture scenario: {scenario}") + scenario_results.append(scenario_result) + continue + + readiness_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-golden-readiness.py"), + str(official_path), + *scenario_readiness_args( + scenario, + source="official", + require_mcp_transcript_evidence=True, + ), + ] + readiness_ok, readiness_json = run_json(readiness_command, repo) + scenario_result["officialReadiness"] = readiness_json + if not readiness_ok: + errors.append(f"official fixture scenario failed readiness: {scenario}") + + if args.candidate_root is not None: + candidate_path = candidate.get(scenario) + scenario_result["candidateFixture"] = str(candidate_path) if candidate_path else None + if candidate_path is None: + errors.append(f"missing OCU candidate fixture scenario: {scenario}") + else: + candidate_readiness_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-golden-readiness.py"), + str(candidate_path), + *scenario_readiness_args( + scenario, + source="ocu", + require_mcp_transcript_evidence=True, + ), + ] + candidate_readiness_ok, candidate_readiness_json = run_json( + candidate_readiness_command, + repo, + ) + scenario_result["candidateReadiness"] = candidate_readiness_json + if not candidate_readiness_ok: + errors.append(f"OCU candidate scenario failed readiness: {scenario}") + + compare_command = [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(official_path), + str(candidate_path), + *BASE_COMPARE_ARGS, + ] + compare_ok, compare_json = run_json(compare_command, repo) + scenario_result["candidateCompare"] = compare_json + if not compare_ok: + errors.append(f"OCU candidate scenario failed official comparison: {scenario}") + + scenario_results.append(scenario_result) + + return { + "ok": not errors, + "officialRoot": str(args.official_root), + "candidateRoot": str(args.candidate_root) if args.candidate_root is not None else None, + "requiredScenarios": required_scenarios, + "availableOfficialScenarios": sorted(official), + "availableCandidateScenarios": sorted(candidate), + "scenarioResults": scenario_results, + "comparePolicy": { + "requiresAxDiffEvidence": "--require-ax-diff-evidence" in BASE_COMPARE_ARGS, + "requiresSameAxDiffMarkers": "--require-same-ax-diff-markers" in BASE_COMPARE_ARGS, + "requiresSuppressedEventSequence": "--require-same-suppressed-event-sequence" in BASE_COMPARE_ARGS, + "requiresSuppressedSchema": "--require-same-suppressed-schema" in BASE_COMPARE_ARGS, + }, + "errors": errors, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Validate an official Record & Replay golden fixture set and optionally compare " + "same-scenario OCU candidate fixtures against it." + ) + ) + parser.add_argument( + "--official-root", + type=pathlib.Path, + default=DEFAULT_FIXTURE_ROOT, + help=f"Directory containing official fixture subdirectories. Defaults to {DEFAULT_FIXTURE_ROOT}.", + ) + parser.add_argument( + "--candidate-root", + type=pathlib.Path, + help="Optional directory containing OCU candidate fixture subdirectories with matching scenario tags.", + ) + parser.add_argument( + "--require-scenario", + action="append", + default=[], + help=( + "Required scenario tag. May be repeated. Defaults to " + f"{DEFAULT_REQUIRED_SCENARIO_HELP}, " + "the minimum official successful recording baseline." + ), + ) + args = parser.parse_args() + + result = check_fixture_set(args) + output = json.dumps(result, indent=2, sort_keys=True) + if result["ok"]: + print(output) + return 0 + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-record-and-replay-baseline-summary.py b/scripts/check-record-and-replay-baseline-summary.py new file mode 100755 index 00000000..bbefd225 --- /dev/null +++ b/scripts/check-record-and-replay-baseline-summary.py @@ -0,0 +1,890 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import sys +from typing import Any + +from record_and_replay_baseline_contract import ( + NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + REQUIRED_BASELINE_CHECKS, + STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, +) + +REQUIRED_BASELINE_CHECK_SET = set(REQUIRED_BASELINE_CHECKS) + + +def load_json(path: pathlib.Path) -> dict[str, Any]: + value = json.loads(path.read_text()) + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def bool_at(root: dict[str, Any], *path: str) -> bool: + value: Any = root + for key in path: + if not isinstance(value, dict): + return False + value = value.get(key) + return value is True + + +def list_at(root: dict[str, Any], *path: str) -> list[Any]: + value: Any = root + for key in path: + if not isinstance(value, dict): + return [] + value = value.get(key) + return value if isinstance(value, list) else [] + + +def string_set_at(root: dict[str, Any], *path: str) -> set[str]: + return {item for item in list_at(root, *path) if isinstance(item, str)} + + +def duplicate_strings_at(root: dict[str, Any], *path: str) -> list[str]: + seen: set[str] = set() + duplicates: set[str] = set() + for item in list_at(root, *path): + if not isinstance(item, str): + continue + if item in seen: + duplicates.add(item) + seen.add(item) + return sorted(duplicates) + + +def next_action_kinds(summary: dict[str, Any]) -> set[str]: + actions = summary.get("nextActions") + if not isinstance(actions, list): + return set() + return { + action.get("kind") + for action in actions + if isinstance(action, dict) and isinstance(action.get("kind"), str) + } + + +def next_action_commands(summary: dict[str, Any], kind: str) -> list[str]: + actions = summary.get("nextActions") + if not isinstance(actions, list): + return [] + commands: list[str] = [] + for action in actions: + if not isinstance(action, dict) or action.get("kind") != kind: + continue + value = action.get("commands") + if not isinstance(value, list): + continue + commands.extend(command for command in value if isinstance(command, str)) + return commands + + +def has_command_containing(commands: list[str], *needles: str) -> bool: + return any(all(needle in command for needle in needles) for command in commands) + + +def upper_camel(value: str) -> str: + return "".join(part[:1].upper() + part[1:] for part in value.split("_") if part) + + +def evidence_check_name(prefix: str, key: str) -> str: + return f"checked{prefix}{upper_camel(key)}Evidence" + + +def check_summary( + summary: dict[str, Any], + require_official_golden: bool, + allow_strict_official_golden_missing: bool = False, +) -> dict[str, Any]: + status = summary.get("status") + if not isinstance(status, dict): + return { + "ok": False, + "failures": ["missing status object"], + "checks": {}, + } + + declared_checks = string_set_at(summary, "checks") + missing_declared_checks = sorted(REQUIRED_BASELINE_CHECK_SET - declared_checks) + unknown_declared_checks = sorted(declared_checks - REQUIRED_BASELINE_CHECK_SET) + duplicate_declared_checks = duplicate_strings_at(summary, "checks") + actions = next_action_kinds(summary) + required_capture_commands = next_action_commands( + summary, + "capture-official-successful-recording-golden", + ) + recommended_capture_commands = next_action_commands( + summary, + "capture-recommended-official-golden-set", + ) + standalone_commands = next_action_commands( + summary, + "scaffold-standalone-record-and-replay-repo", + ) + missing_golden = not bool_at( + summary, "status", "officialSuccessfulRecordingGoldenComplete" + ) + required_gaps = list_at( + summary, + "status", + "missingRequiredOfficialSuccessfulRecordingScenarios", + ) + not_ready_required = list_at( + summary, + "status", + "notReadyRequiredOfficialSuccessfulRecordingScenarios", + ) + coverage_errors = list_at(summary, "status", "officialFixtureCoverageErrors") + missing_recommended_golden = bool( + list_at(summary, "status", "missingRecommendedOfficialSuccessfulRecordingScenarios") + ) + strict_mode_expected = ( + require_official_golden or allow_strict_official_golden_missing + ) + official_golden_requirement_satisfied = bool_at( + summary, "status", "officialGoldenRequirementSatisfied" + ) + expected_equivalence_ready = ( + bool_at(summary, "status", "usableBaseline") + and bool_at(summary, "status", "officialSuccessfulRecordingGoldenComplete") + ) + standalone_ready = bool_at(summary, "status", "standaloneRepoBaselineReady") + expected_top_level_ok = ( + bool_at(summary, "status", "usableBaseline") + and bool_at(summary, "status", "officialGoldenRequirementSatisfied") + ) + failures: list[str] = [] + checks = { + "checkedBaselineName": summary.get("baseline") == "record-and-replay", + "checkedTopLevelOkMatchesPolicy": summary.get("ok") is expected_top_level_ok, + "checkedRequiredBaselineChecksDeclared": not missing_declared_checks, + "checkedNoUnknownBaselineChecksDeclared": not unknown_declared_checks, + "checkedNoDuplicateBaselineChecksDeclared": not duplicate_declared_checks, + "checkedUsableBaseline": bool_at(summary, "status", "usableBaseline"), + "checkedNoMissingUsableBaselineEvidence": ( + list_at(summary, "status", "missingUsableBaselineEvidence") == [] + ), + "checkedStandaloneRepoBaselineReady": bool_at( + summary, "status", "standaloneRepoBaselineReady" + ), + "checkedOfficialNonRecordingBaselineVerified": bool_at( + summary, "status", "officialNonRecordingBaselineVerified" + ), + "checkedOfficialRawStartTimeoutBoundaryVerified": bool_at( + summary, "status", "officialRawStartTimeoutBoundaryVerified" + ), + "checkedOfficialSurfaceLocalEvidence": bool_at( + summary, "evidence", "officialSurfaceCompare", "local", "ok" + ), + "checkedOfficialSurfaceBundledEvidence": bool_at( + summary, "evidence", "officialSurfaceCompare", "official", "ok" + ), + "checkedOfficialNoActiveResponseEvidence": bool_at( + summary, "evidence", "officialNoActiveResponse", "ok" + ), + "checkedOfficialNoActiveNoSessionFilesEvidence": bool_at( + summary, + "evidence", + "officialNoActiveResponse", + "checkedNoSessionFilesCreated", + ), + "checkedOfficialNoActiveStatusShapeEvidence": bool_at( + summary, "evidence", "officialNoActiveResponse", "checkedStatusShape" + ), + "checkedOfficialNoActiveStopShapeEvidence": bool_at( + summary, "evidence", "officialNoActiveResponse", "checkedStopShape" + ), + "checkedOfficialRawStartTimeoutEvidence": bool_at( + summary, "evidence", "officialRawStartTimeout", "ok" + ), + "checkedOfficialRawStartTimeoutBoundaryEvidence": bool_at( + summary, + "evidence", + "officialRawStartTimeout", + "checkedOfficialRawStartTimeoutBoundary", + ), + "checkedOfficialRawStartStatusStopTimeoutEvidence": bool_at( + summary, + "evidence", + "officialRawStartTimeout", + "checkedOfficialRawStartStatusStopTimeout", + ), + "checkedOfficialRawStartDoesNotReturnRecordingPathsEvidence": bool_at( + summary, + "evidence", + "officialRawStartTimeout", + "checkedOfficialRawStartDoesNotReturnRecordingPaths", + ), + "checkedOfficialRawSurfaceEvidence": bool_at( + summary, + "evidence", + "officialRawStartTimeout", + "checkedOfficialRawSurface", + ), + "checkedOfficialRawProbeFixtureRedactionEvidence": bool_at( + summary, + "evidence", + "officialRawStartTimeout", + "checkedRawProbeFixtureRedaction", + ), + "checkedBaselineContractEvidence": bool_at( + summary, "evidence", "baselineContract", "ok" + ), + "checkedBaselineContractRequiredChecksEvidence": bool_at( + summary, + "evidence", + "baselineContract", + "checkedRequiredBaselineChecks", + ), + "checkedBaselineContractNoDuplicateChecksEvidence": bool_at( + summary, + "evidence", + "baselineContract", + "checkedNoDuplicateRequiredBaselineChecks", + ), + "checkedBaselineContractStandaloneSummaryRenamesEvidence": bool_at( + summary, + "evidence", + "baselineContract", + "checkedStandaloneLifecycleSummaryRenames", + ), + "checkedBaselineAuditMakeTargetsEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditMakeTargets", + ), + "checkedBaselineAuditDefaultSummaryPathEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditDefaultSummaryPath", + ), + "checkedBaselineAuditCustomSummaryPathEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditCustomSummaryPath", + ), + "checkedBaselineAuditIgnoresStrictSummaryVarEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditIgnoresStrictSummaryVar", + ), + "checkedBaselineAuditStrictOfficialGoldenTargetEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditStrictOfficialGoldenTarget", + ), + "checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPathEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPath", + ), + "checkedBaselineAuditStrictOfficialGoldenCustomSummaryPathEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditStrictOfficialGoldenCustomSummaryPath", + ), + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVarEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar", + ), + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPathEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath", + ), + "checkedOfficialCaptureMissingScenarioPreflightEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCaptureMissingScenarioPreflight", + ), + "checkedOfficialCapturePacketEvidence": bool_at( + summary, "evidence", "preflightPipelines", "checkedOfficialCapturePacket" + ), + "checkedOfficialCapturePacketPostCaptureWorkflowEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketPostCaptureWorkflow", + ), + "checkedOfficialCapturePacketWorkflowVerifierEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketWorkflowVerifier", + ), + "checkedOfficialCapturePacketHandoffScriptsEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketHandoffScripts", + ), + "checkedOfficialCapturePacketStrictAuditHandoffEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketStrictAuditHandoff", + ), + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ), + "checkedOfficialCapturePacketOcuCandidateOutputDirEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketOcuCandidateOutputDir", + ), + "checkedOfficialCapturePacketNoTranscriptEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketNoTranscript", + ), + "checkedOfficialCapturePacketSetEvidence": bool_at( + summary, "evidence", "preflightPipelines", "checkedOfficialCapturePacketSet" + ), + "checkedOfficialCapturePacketSetPostCaptureWorkflowEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetPostCaptureWorkflow", + ), + "checkedOfficialCapturePacketSetWorkflowVerifierEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetWorkflowVerifier", + ), + "checkedOfficialCapturePacketSetNoTranscriptEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetNoTranscript", + ), + "checkedOfficialCapturePacketSetOcuCandidateHandoffEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetOcuCandidateHandoff", + ), + "checkedOfficialCapturePacketSetContractManifestEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetContractManifest", + ), + "checkedOfficialCapturePacketTranscriptManifestEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketTranscriptManifest", + ), + "checkedOfficialCapturePacketMakeTargetsEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketMakeTargets", + ), + "checkedOfficialCapturePacketInputSemanticGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketInputSemanticGuard", + ), + "checkedOfficialCapturePacketPlaceholderGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketPlaceholderGuard", + ), + "checkedOfficialCapturePacketVerifyInputsEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketVerifyInputs", + ), + "checkedOfficialCapturePacketImportPlaceholderGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketImportPlaceholderGuard", + ), + "checkedOfficialCapturePacketTranscriptPlaceholderGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketTranscriptPlaceholderGuard", + ), + "checkedOfficialCapturePacketSetRootPlaceholderGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetRootPlaceholderGuard", + ), + "checkedOfficialCapturePacketSetRootPreflightPlaceholderGuardEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard", + ), + "checkedOfficialCapturePacketSetVerifyAllEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCapturePacketSetVerifyAll", + ), + "checkedOfficialCaptureRequireReadyFailureEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCaptureRequireReadyFailure", + ), + "checkedOfficialCaptureMissingPluginFailureEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCaptureMissingPluginFailure", + ), + "checkedOfficialCaptureKeyboardScenarioEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCaptureKeyboardScenario", + ), + "checkedOfficialCaptureCoverageErrorFailureEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOfficialCaptureCoverageErrorFailure", + ), + "checkedOcuPairingMissingOfficialPreflightEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOcuPairingMissingOfficialPreflight", + ), + "checkedOcuPairingNoCandidatePreflightEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOcuPairingNoCandidatePreflight", + ), + "checkedOcuPairingPairedCandidatePreflightEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOcuPairingPairedCandidatePreflight", + ), + "checkedOcuPairingKeyboardRecordingRequiredScenarioEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOcuPairingKeyboardRecordingRequiredScenario", + ), + "checkedOcuPairingCoverageErrorReportEvidence": bool_at( + summary, + "evidence", + "preflightPipelines", + "checkedOcuPairingCoverageErrorReport", + ), + "checkedGoldenRequirementSatisfied": bool_at( + summary, "status", "officialGoldenRequirementSatisfied" + ) + if not allow_strict_official_golden_missing + else not official_golden_requirement_satisfied, + "checkedStrictModeMatchesExpectation": ( + bool_at(summary, "status", "strictOfficialGoldenRequired") + == strict_mode_expected + ), + "checkedGoldenGateConsistency": ( + bool_at(summary, "status", "officialGoldenGatePassed") + == bool_at(summary, "status", "officialSuccessfulRecordingGoldenComplete") + ), + "checkedEquivalenceRequiresGolden": ( + not bool_at(summary, "status", "officialSuccessfulRecordingEquivalenceReady") + or ( + bool_at(summary, "status", "usableBaseline") + and bool_at(summary, "status", "officialSuccessfulRecordingGoldenComplete") + ) + ), + "checkedEquivalenceReadyMatchesPolicy": ( + bool_at(summary, "status", "officialSuccessfulRecordingEquivalenceReady") + == expected_equivalence_ready + ), + "checkedStandaloneReadyRequiresUsableBaseline": ( + not standalone_ready or bool_at(summary, "status", "usableBaseline") + ), + "checkedRequiresOfficialGoldenCaptureConsistency": ( + bool_at(summary, "status", "requiresOfficialGoldenCapture") == missing_golden + ), + "checkedMissingGoldenHasScenarioGapOrCoverageError": ( + not missing_golden + or bool(required_gaps) + or bool(not_ready_required) + or bool(coverage_errors) + ), + "checkedOfficialGoldenCompleteHasNoRequiredGaps": ( + missing_golden + or (not required_gaps and not not_ready_required and not coverage_errors) + ), + "checkedMissingGoldenHasNextAction": ( + not missing_golden + or "capture-official-successful-recording-golden" in actions + ), + "checkedMissingGoldenNextActionPacketMakeCommand": ( + not missing_golden + or has_command_containing( + required_capture_commands, + "make record-and-replay-official-golden-capture-packet", + "RNR_SCENARIO=", + "RNR_PACKET_DIR=", + ) + ), + "checkedMissingGoldenNextActionVerifyStep": ( + not missing_golden + or has_command_containing(required_capture_commands, "./verify-inputs.sh") + ), + "checkedMissingGoldenNextActionInspectStep": ( + not missing_golden + or has_command_containing(required_capture_commands, "./inspect-only.sh") + ), + "checkedMissingGoldenNextActionImportStep": ( + not missing_golden + or has_command_containing(required_capture_commands, "./import-fixture.sh") + ), + "checkedMissingGoldenNextActionStrictGateStep": ( + not missing_golden + or has_command_containing( + required_capture_commands, + "make record-and-replay-official-golden-gate", + ) + ), + "checkedMissingGoldenNextActionStrictAuditStep": ( + not missing_golden + or has_command_containing( + required_capture_commands, + "make record-and-replay-official-golden-gate-audit", + ) + ), + "checkedRecommendedGoldenNextActionPacketSetCommand": ( + not missing_recommended_golden + or has_command_containing( + recommended_capture_commands, + "make record-and-replay-official-golden-capture-packet-set", + "RNR_PACKET_DIR=", + ) + ), + "checkedRecommendedGoldenNextActionVerifyAllStep": ( + not missing_recommended_golden + or has_command_containing(recommended_capture_commands, "./verify-all.sh") + ), + "checkedRecommendedGoldenNextActionInspectAllStep": ( + not missing_recommended_golden + or has_command_containing(recommended_capture_commands, "./inspect-all.sh") + ), + "checkedRecommendedGoldenNextActionImportAllStep": ( + not missing_recommended_golden + or has_command_containing(recommended_capture_commands, "./import-all.sh") + ), + "checkedRecommendedGoldenNextActionIngestOcuCandidatesStep": ( + not missing_recommended_golden + or has_command_containing( + recommended_capture_commands, + "./ingest-ocu-candidates.sh", + ) + ), + "checkedStandaloneNextActionWhenReady": ( + not standalone_ready + or "scaffold-standalone-record-and-replay-repo" in actions + ), + "checkedStandaloneNextActionBaselineAuditCommand": ( + not standalone_ready + or has_command_containing( + standalone_commands, + "make record-and-replay-baseline-audit", + ) + ), + "checkedStandaloneNextActionScaffoldCommand": ( + not standalone_ready + or has_command_containing( + standalone_commands, + "open-computer-use scaffold-record-and-replay-skill-repo", + "--output-dir ", + ) + ), + "checkedStandaloneNextActionCheckCommand": ( + not standalone_ready + or has_command_containing(standalone_commands, "./scripts/check.sh") + ), + "checkedStandaloneNextActionLifecycleSmokeCommand": ( + not standalone_ready + or has_command_containing( + standalone_commands, + "./scripts/recording-lifecycle-smoke.py", + ) + ), + } + + direct_evidence_requirements = { + "baselineContract": ( + "BaselineContract", + [ + "ok", + "checkedRequiredBaselineChecks", + "checkedNoDuplicateRequiredBaselineChecks", + "checkedStandaloneSmokeRequiredKeys", + "checkedStandaloneSummaryEvidenceKeys", + "checkedStandaloneLifecycleSummaryRenames", + "checkedNpmStagedSmokeRequiredKeys", + "checkedNpmStagedSummaryEvidenceKeys", + "checkedNpmStagedSummaryMatchesSmoke", + ], + ), + "eventStreamMatrix": ( + "EventStreamMatrix", + [ + "ok", + "checkedDefaultLifecycleHandoff", + "checkedRequiredModes", + ], + ), + "screenshotContextSmoke": ( + "ScreenshotContextSmoke", + [ + "ok", + "checkedScreenshotPolicyAlways", + "checkedScreenshotNeededForContext", + "checkedScreenshotPathWhenAvailable", + ], + ), + "realInputActionSmoke": ( + "RealInputActionSmoke", + [ + "ok", + "checkedRequiredEventTypes", + "checkedSkillDraftGenerated", + "checkedMcpTranscriptCaptured", + "checkedMcpResponseShapesCaptured", + "checkedSkillReadinessCanCreateDraft", + "checkedSkillCreatorFinalizationHandoff", + "checkedGeneratedSkillPathRedaction", + "checkedSimpleActionStopCandidate", + "checkedDragStopCandidate", + ], + ), + "officialFixtureSetGate": ( + "OfficialFixtureSetGate", + [ + "ok", + "checkedFixtureSetGate", + "checkedRequiredSimpleActionStopScenario", + "checkedCandidatePairing", + "checkedCandidateReadinessFailure", + "checkedAxDiffComparisonPolicy", + "checkedSuppressedStreamComparisonPolicy", + "checkedAxDiffComparisonFailure", + "checkedStopEndReasonPolicy", + "checkedCancelScenarioPolicy", + "checkedKeyboardInputScenarioPolicy", + "checkedDragScenarioPolicy", + "checkedTimeoutScenarioPolicy", + ], + ), + "fixtureIngestPipelines": ( + "FixtureIngestPipelines", + [ + "checkedOfficialFixtureIngest", + "checkedOfficialFixtureInspectOnly", + "checkedOfficialSessionDirectoryPathHandoff", + "checkedPostIngestCoverageReport", + "checkedPostIngestCoverageReadiness", + "checkedPostIngestRequireCoverageFailure", + "checkedOcuCandidateIngest", + "checkedOcuCandidateIngestHandoffCommands", + "checkedOcuSmokeJsonImport", + "checkedOcuOfficialCandidatePairing", + "checkedOcuCandidateRedaction", + "checkedOcuKeyboardInputScenarioReadiness", + "checkedOcuDragScenarioReadiness", + ], + ), + "standaloneSkillRepo": ( + "StandaloneSkillRepo", + STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + ), + "npmStagedSkillRepo": ( + "NpmStagedSkillRepo", + NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + ), + } + for section, (prefix, keys) in direct_evidence_requirements.items(): + for key in keys: + checks[evidence_check_name(prefix, key)] = bool_at( + summary, "evidence", section, key + ) + + if allow_strict_official_golden_missing: + checks["checkedStrictOfficialGoldenMissingFailure"] = ( + bool_at(summary, "status", "strictOfficialGoldenRequired") + and not official_golden_requirement_satisfied + and not bool_at(summary, "status", "officialGoldenGatePassed") + and not bool_at(summary, "status", "officialSuccessfulRecordingGoldenComplete") + and not bool_at( + summary, + "status", + "officialSuccessfulRecordingEquivalenceReady", + ) + ) + checks["checkedStrictMissingGoldenHasRequiredGap"] = bool( + required_gaps or not_ready_required + ) + checks["checkedNoOfficialFixtureCoverageErrors"] = not coverage_errors + elif not require_official_golden: + checks["checkedAllowsMissingOfficialGoldenButNotEquivalence"] = ( + bool_at(summary, "status", "officialGoldenRequirementSatisfied") + and ( + bool_at(summary, "status", "officialSuccessfulRecordingGoldenComplete") + or not bool_at( + summary, + "status", + "officialSuccessfulRecordingEquivalenceReady", + ) + ) + ) + else: + checks["checkedOfficialGoldenGatePassed"] = bool_at( + summary, "status", "officialGoldenGatePassed" + ) + checks["checkedOfficialSuccessfulRecordingEquivalenceReady"] = bool_at( + summary, + "status", + "officialSuccessfulRecordingEquivalenceReady", + ) + checks["checkedNoRequiredOfficialScenarioGaps"] = not list_at( + summary, + "status", + "missingRequiredOfficialSuccessfulRecordingScenarios", + ) and not list_at( + summary, + "status", + "notReadyRequiredOfficialSuccessfulRecordingScenarios", + ) + checks["checkedNoOfficialFixtureCoverageErrors"] = not list_at( + summary, "status", "officialFixtureCoverageErrors" + ) + + for key, passed in checks.items(): + if passed is not True: + failures.append(key) + + return { + "ok": not failures, + "failures": failures, + "checks": checks, + "summaryStatus": { + "usableBaseline": status.get("usableBaseline"), + "standaloneRepoBaselineReady": status.get("standaloneRepoBaselineReady"), + "officialGoldenRequirementSatisfied": status.get( + "officialGoldenRequirementSatisfied" + ), + "officialGoldenGatePassed": status.get("officialGoldenGatePassed"), + "officialSuccessfulRecordingGoldenComplete": status.get( + "officialSuccessfulRecordingGoldenComplete" + ), + "officialSuccessfulRecordingEquivalenceReady": status.get( + "officialSuccessfulRecordingEquivalenceReady" + ), + "missingRequiredOfficialSuccessfulRecordingScenarios": status.get( + "missingRequiredOfficialSuccessfulRecordingScenarios" + ), + "notReadyRequiredOfficialSuccessfulRecordingScenarios": status.get( + "notReadyRequiredOfficialSuccessfulRecordingScenarios" + ), + "officialFixtureCoverageErrors": status.get("officialFixtureCoverageErrors"), + }, + "declaredChecks": { + "missingRequired": missing_declared_checks, + "unknown": unknown_declared_checks, + "duplicates": duplicate_declared_checks, + }, + } + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Audit a saved Record & Replay baseline summary JSON without rerunning the " + "desktop smoke suite." + ) + ) + parser.add_argument("summary_json", type=pathlib.Path) + parser.add_argument( + "--require-official-golden", + action="store_true", + help=( + "Require official successful recording golden readiness and equivalence. " + "Default mode only requires the usable standalone baseline and keeps the " + "golden gap explicit." + ), + ) + parser.add_argument( + "--allow-strict-official-golden-missing", + action="store_true", + help=( + "Audit a strict official golden summary that is expected to fail only " + "because required official successful recording fixtures are still " + "missing or not ready. This still requires the usable baseline, " + "standalone repo baseline, official non-recording evidence, raw " + "timeout boundary, nextActions, and zero fixture coverage errors." + ), + ) + args = parser.parse_args() + if args.require_official_golden and args.allow_strict_official_golden_missing: + parser.error( + "--require-official-golden and " + "--allow-strict-official-golden-missing are mutually exclusive" + ) + + try: + summary = load_json(args.summary_json) + result = check_summary( + summary, + args.require_official_golden, + args.allow_strict_official_golden_missing, + ) + except Exception as exc: + result = { + "ok": False, + "failures": [str(exc)], + "checks": {}, + } + + print(json.dumps(result, indent=2, sort_keys=True)) + if result.get("ok") is not True: + failures = result.get("failures") or [] + if failures: + print("baseline summary audit failed: " + ", ".join(failures), file=sys.stderr) + else: + print("baseline summary audit failed", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci.sh b/scripts/ci.sh index f12dc11a..13f08ebe 100755 --- a/scripts/ci.sh +++ b/scripts/ci.sh @@ -16,6 +16,39 @@ while IFS= read -r file; do node --check "$file" done < <(find "${repo_root}/scripts" -type f -name '*.mjs' | sort) +while IFS= read -r file; do + python3 -m py_compile "$file" +done < <(find "${repo_root}/scripts" -type f -name '*.py' | sort) + +( + cd "${repo_root}" + npm run package:skill + ./scripts/compare-event-stream-surface.py + ./scripts/compare-event-stream-no-active.py + ./scripts/test-event-stream-probe-fixtures.py + ./scripts/test-event-stream-recording-probe.py + ./scripts/test-event-stream-local-probe.py + ./scripts/test-event-stream-golden-readiness.py + ./scripts/test-event-stream-official-fixture-coverage.py + ./scripts/check-event-stream-official-fixture-coverage.py --allow-missing + ./scripts/test-record-and-replay-baseline-contract.py + ./scripts/test-record-and-replay-baseline-summary.py + ./scripts/test-record-and-replay-baseline-summary-audit.py + RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-summary.json RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-ci-outer-golden-summary.json ./scripts/test-record-and-replay-baseline-audit-make-targets.py + ./scripts/test-record-and-replay-baseline-runner-summary-json.py + ./scripts/test-record-and-replay-official-golden-capture-preflight.py + ./scripts/test-record-and-replay-official-capture-finalizer.py + ./scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py + ./scripts/test-event-stream-official-fixture-set.py + ./scripts/test-official-record-and-replay-fixture-ingest.py + ./scripts/test-ocu-record-and-replay-candidate-ingest.py + ./scripts/test-event-stream-skill-scaffold.py + ./scripts/test-record-and-replay-skill-repo-scaffold.py + node ./scripts/test-codex-record-and-replay-installer.mjs + swift test + ./scripts/run-event-stream-smoke-matrix.sh +) + if command -v go >/dev/null 2>&1; then ( cd "${repo_root}/apps/OpenComputerUseWindows" diff --git a/scripts/compare-event-stream-no-active.py b/scripts/compare-event-stream-no-active.py new file mode 100755 index 00000000..6c54abc5 --- /dev/null +++ b/scripts/compare-event-stream-no-active.py @@ -0,0 +1,242 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import select +import subprocess +import sys +import tempfile +from typing import Any + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_FIXTURE = ( + REPO_ROOT + / "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json" +) +EXPECTED_TOOLS = ["event_stream_status", "event_stream_stop"] + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Compare Record & Replay no-active status/stop MCP responses against the official fixture." + ) + parser.add_argument( + "--fixture", + default=str(DEFAULT_FIXTURE), + help="Official Codex-hosted no-active status/stop fixture JSON.", + ) + parser.add_argument( + "--local-command", + default=str(REPO_ROOT / ".build/debug/OpenComputerUse"), + help="Local OpenComputerUse binary to launch with 'event-stream mcp'.", + ) + parser.add_argument( + "--skip-build", + action="store_true", + help="Do not run 'swift build --product OpenComputerUse' before comparison.", + ) + parser.add_argument( + "--timeout", + type=float, + default=10, + help="Seconds to wait for each MCP response.", + ) + return parser.parse_args() + + +def load_json(path: pathlib.Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise AssertionError(f"expected JSON object at {path}") + return value + + +def canonical(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def expected_responses(fixture: dict[str, Any]) -> dict[str, dict[str, Any]]: + tool_responses = fixture.get("toolResponses") + if not isinstance(tool_responses, dict): + raise AssertionError("fixture missing toolResponses object") + + expected: dict[str, dict[str, Any]] = {} + for tool_name in EXPECTED_TOOLS: + response = tool_responses.get(tool_name) + if not isinstance(response, dict): + raise AssertionError(f"fixture missing {tool_name} response") + content = response.get("content") + if not isinstance(content, list) or len(content) != 1: + raise AssertionError(f"fixture {tool_name} response must contain one content item") + item = content[0] + if not isinstance(item, dict) or item.get("type") != "text": + raise AssertionError(f"fixture {tool_name} response must be text content") + text_json = item.get("textJSON") + if not isinstance(text_json, dict): + raise AssertionError(f"fixture {tool_name} response missing textJSON object") + expected[tool_name] = text_json + return expected + + +def text_json_from_tool_result(response: dict[str, Any], tool_name: str) -> dict[str, Any]: + result = response.get("result") + if not isinstance(result, dict): + raise AssertionError(f"{tool_name} response missing result") + content = result.get("content") + if not isinstance(content, list) or len(content) != 1: + raise AssertionError(f"{tool_name} response must contain one content item") + item = content[0] + if not isinstance(item, dict) or item.get("type") != "text": + raise AssertionError(f"{tool_name} response must be text content") + text = item.get("text") + if not isinstance(text, str): + raise AssertionError(f"{tool_name} text content missing text field") + parsed = json.loads(text) + if not isinstance(parsed, dict): + raise AssertionError(f"{tool_name} text JSON must parse to an object") + return parsed + + +def capture_no_active( + command: list[str], + timeout: float, + recordings_dir: pathlib.Path, +) -> dict[str, dict[str, Any]]: + env = { + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings_dir), + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + } + proc = subprocess.Popen( + command, + cwd=str(REPO_ROOT), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + def request(message: dict[str, Any], expect_response: bool = True) -> dict[str, Any] | None: + assert proc.stdin is not None + assert proc.stdout is not None + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + ready, _, _ = select.select([proc.stdout], [], [], timeout) + if not ready: + raise TimeoutError(f"MCP server did not respond within {timeout}s: {command}") + line = proc.stdout.readline() + if not line: + raise AssertionError(f"MCP server exited before responding: {command}") + response = json.loads(line) + if "error" in response: + raise AssertionError(response["error"]) + return response + + try: + request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { + "name": "event-stream-no-active-compare", + "version": "0", + }, + }, + } + ) + request( + {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, + expect_response=False, + ) + actual: dict[str, dict[str, Any]] = {} + for index, tool_name in enumerate(EXPECTED_TOOLS, start=2): + response = request( + { + "jsonrpc": "2.0", + "id": index, + "method": "tools/call", + "params": {"name": tool_name, "arguments": {}}, + } + ) + assert response is not None + actual[tool_name] = text_json_from_tool_result(response, tool_name) + finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + stderr = proc.stderr.read() if proc.stderr else "" + if proc.returncode != 0: + raise AssertionError(stderr) + + forbidden_files = [ + recordings_dir / "latest-session.json", + recordings_dir / "active-session.json", + ] + session_dirs = [path for path in recordings_dir.iterdir() if path.is_dir()] + created_files = [path for path in forbidden_files if path.exists()] + session_dirs + if created_files: + relative = [str(path.relative_to(recordings_dir)) for path in created_files] + raise AssertionError(f"no-active status/stop created session files: {relative}") + + return actual + + +def main() -> int: + args = parse_args() + fixture_path = pathlib.Path(args.fixture) + fixture = load_json(fixture_path) + expected = expected_responses(fixture) + + if not args.skip_build: + subprocess.run( + ["swift", "build", "--product", "OpenComputerUse"], + cwd=str(REPO_ROOT), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + with tempfile.TemporaryDirectory(prefix="event-stream-no-active-") as tmp: + recordings_dir = pathlib.Path(tmp) + actual = capture_no_active( + [args.local_command, "event-stream", "mcp"], + timeout=args.timeout, + recordings_dir=recordings_dir, + ) + ok = canonical(actual) == canonical(expected) + payload: dict[str, Any] = { + "ok": ok, + "fixture": str(fixture_path), + "recordingsDir": str(recordings_dir), + "checkedTools": EXPECTED_TOOLS, + "expected": expected, + "actual": actual, + "createdSessionFiles": False, + } + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 if ok else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compare-event-stream-recordings.py b/scripts/compare-event-stream-recordings.py new file mode 100755 index 00000000..5b12e7f2 --- /dev/null +++ b/scripts/compare-event-stream-recordings.py @@ -0,0 +1,1017 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import sys +from collections import Counter, defaultdict + + +SEMANTIC_SCHEMA_PATHS_BY_EVENT_TYPE = { + "AX.focusedWindowChanged": { + "accessibilityInspectorPayload", + "accessibilityInspectorPayload.kind", + "accessibilityInspectorPayload.renderedText", + "accessibilityInspectorPayload.fullTree", + "accessibilityInspectorPayload.fullTree[]", + "accessibilityInspectorPayload.treeLines", + "accessibilityInspectorPayload.treeLines[]", + "accessibilityInspectorPayload.diffFromPrevious", + "accessibilityInspectorPayload.cumulativeDiffFromInitial", + "accessibilityInspectorPayload.cumulativeRenderedText", + "accessibilityInspectorPayload.cumulativeTreeLines", + "accessibilityInspectorPayload.cumulativeTreeLines[]", + "accessibilityInspectorPayload.screenshotNeededForContext", + "accessibilityInspectorPayload.screenshotPath", + }, + "keyboard.shortcut": { + "focusedAccessibilityElement", + "focusedAccessibilityElement.role", + "focusedAccessibilityElement.title", + "focusedAccessibilityElement.label", + "key", + "modifiers", + }, + "keyboard.submit": { + "focusedAccessibilityElement", + "focusedAccessibilityElement.role", + "focusedAccessibilityElement.title", + "focusedAccessibilityElement.label", + "key", + "modifiers", + }, + "keyboard.text_input": { + "focusedAccessibilityElement", + "focusedAccessibilityElement.role", + "focusedAccessibilityElement.title", + "focusedAccessibilityElement.label", + "secureInput", + "text", + "textLength", + }, + "mouse.click": { + "button", + "location", + "location.x", + "location.y", + "targetAccessibilityElement", + "targetAccessibilityElement.role", + "targetAccessibilityElement.title", + "targetAccessibilityElement.label", + "targetAccessibilityElement.actions", + "targetAccessibilityElement.actions[]", + }, + "mouse.context_menu": { + "button", + "location", + "location.x", + "location.y", + "targetAccessibilityElement", + "targetAccessibilityElement.role", + "targetAccessibilityElement.title", + "targetAccessibilityElement.label", + "targetAccessibilityElement.actions", + "targetAccessibilityElement.actions[]", + }, + "mouse.drag": { + "button", + "distance", + "durationSeconds", + "endLocation", + "endLocation.x", + "endLocation.y", + "startLocation", + "startLocation.x", + "startLocation.y", + "targetAccessibilityElement", + "targetAccessibilityElement.role", + "targetAccessibilityElement.title", + "targetAccessibilityElement.label", + }, + "selection.changed": { + "focusedAccessibilityElement", + "focusedAccessibilityElement.role", + "focusedAccessibilityElement.title", + "focusedAccessibilityElement.label", + "selectedText", + "selectedTextLength", + "selectionCleared", + }, + "terminal.value_changed": { + "focusedAccessibilityElement", + "focusedAccessibilityElement.role", + "focusedAccessibilityElement.title", + "focusedAccessibilityElement.label", + "valueHash", + "valueLength", + }, + "window.changed": { + "app", + "app.bundleIdentifier", + "app.name", + "window", + "window.title", + }, +} + +STABLE_METADATA_VALUE_KEYS = [ + "active", + "endReason", + "eventCount", + "state", + "suppressedEventCount", +] + +MCP_RESPONSE_SHAPES = { + "startResponseShape": "event_stream_start", + "repeatStartResponseShape": "event_stream_start", + "statusResponseShape": "event_stream_status", + "stopResponseShape": "event_stream_stop", + "repeatStopResponseShape": "event_stream_stop", + "finalStatusResponseShape": "event_stream_status", +} + +MCP_RESPONSE_SHAPE_ALIASES = { + "startResponse": "startResponseShape", + "repeatStartResponse": "repeatStartResponseShape", + "statusResponse": "statusResponseShape", + "stopResponse": "stopResponseShape", + "repeatStopResponse": "repeatStopResponseShape", + "finalStatusResponse": "finalStatusResponseShape", +} + +HANDOFF_PATH_KEYS = ( + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", +) + + +def load_json(path): + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def read_jsonl(path): + records = [] + if not path.exists(): + raise ValueError(f"missing JSONL file: {path}") + for index, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSONL in {path}:{index}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}:{index}") + records.append(value) + return records + + +def event_kind(event): + kind = event.get("kind") + if isinstance(kind, str): + return kind + event_type = event.get("type") + if isinstance(event_type, str): + return event_type + return None + + +def resolve_path(base, raw, fallback): + if isinstance(raw, str) and raw: + candidate = pathlib.Path(raw) + if candidate.is_absolute(): + return candidate + return base / candidate + return fallback + + +def handoff_path_evidence(session_dir, metadata): + evidence = {} + for key in HANDOFF_PATH_KEYS: + raw = metadata.get(key) + entry = { + "value": raw, + "resolvedPath": None, + "exists": None, + } + if isinstance(raw, str) and raw: + path = pathlib.Path(raw) + resolved_path = path if path.is_absolute() else session_dir / path + entry["resolvedPath"] = str(resolved_path) + entry["exists"] = resolved_path.exists() + evidence[key] = entry + return evidence + + +def handoff_path_errors(baseline_evidence, candidate_evidence): + errors = [] + for key in HANDOFF_PATH_KEYS: + baseline_entry = baseline_evidence.get(key, {}) + candidate_entry = candidate_evidence.get(key, {}) + baseline_value = baseline_entry.get("value") + candidate_value = candidate_entry.get("value") + if not isinstance(baseline_value, str) or not baseline_value: + errors.append(f"baseline missing declared handoff path: {key}") + elif baseline_entry.get("exists") is False: + errors.append(f"baseline declared handoff path does not exist: {key}") + if not isinstance(candidate_value, str) or not candidate_value: + errors.append(f"candidate missing declared handoff path: {key}") + elif candidate_entry.get("exists") is False: + errors.append(f"candidate declared handoff path does not exist: {key}") + return errors + + +def resolve_recording(path): + if path.is_dir(): + metadata_path = path / "metadata.json" + session_path = path / "session.json" + if metadata_path.exists(): + metadata = load_json(metadata_path) + elif session_path.exists(): + metadata_path = session_path + metadata = load_json(metadata_path) + else: + raise ValueError(f"session directory has no metadata.json or session.json: {path}") + events_path = resolve_path(path, metadata.get("eventsPath"), path / "events.jsonl") + transcript_path = resolve_mcp_transcript_path(path) + suppressed_path = resolve_path(path, metadata.get("suppressedEventsPath"), path / "suppressed.jsonl") + return { + "sessionDir": path, + "metadataPath": metadata_path, + "metadata": metadata, + "eventsPath": events_path, + "events": read_jsonl(events_path), + "suppressedPath": suppressed_path, + "suppressed": read_jsonl(suppressed_path) if suppressed_path.exists() else [], + "mcpTranscriptPath": transcript_path, + "mcpTranscript": load_json(transcript_path) if transcript_path else None, + } + + if not path.exists(): + raise ValueError(f"path does not exist: {path}") + if path.name == "events.jsonl": + return { + "sessionDir": path.parent, + "metadataPath": None, + "metadata": {}, + "eventsPath": path, + "events": read_jsonl(path), + "suppressedPath": path.parent / "suppressed.jsonl", + "suppressed": read_jsonl(path.parent / "suppressed.jsonl") if (path.parent / "suppressed.jsonl").exists() else [], + "mcpTranscriptPath": None, + "mcpTranscript": None, + } + if path.name not in {"metadata.json", "session.json"}: + raise ValueError("input must be a session directory, metadata.json, session.json, or events.jsonl") + metadata = load_json(path) + session_dir = path.parent + events_path = resolve_path(session_dir, metadata.get("eventsPath"), session_dir / "events.jsonl") + suppressed_path = resolve_path(session_dir, metadata.get("suppressedEventsPath"), session_dir / "suppressed.jsonl") + return { + "sessionDir": session_dir, + "metadataPath": path, + "metadata": metadata, + "eventsPath": events_path, + "events": read_jsonl(events_path), + "suppressedPath": suppressed_path, + "suppressed": read_jsonl(suppressed_path) if suppressed_path.exists() else [], + "mcpTranscriptPath": resolve_mcp_transcript_path(session_dir), + "mcpTranscript": ( + load_json(resolve_mcp_transcript_path(session_dir)) + if resolve_mcp_transcript_path(session_dir) + else None + ), + } + + +def resolve_mcp_transcript_path(session_dir): + manifest_path = session_dir / "fixture-manifest.json" + if manifest_path.exists(): + manifest = load_json(manifest_path) + files = manifest.get("files") + if isinstance(files, dict) and isinstance(files.get("mcpTranscript"), str): + candidate = resolve_path(session_dir, files["mcpTranscript"], session_dir / "mcp-transcript.json") + if candidate.exists(): + return candidate + candidate = session_dir / "mcp-transcript.json" + if candidate.exists(): + return candidate + return None + + +def type_name(value): + if value is None: + return "null" + if isinstance(value, bool): + return "bool" + if isinstance(value, int) and not isinstance(value, bool): + return "int" + if isinstance(value, float): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, list): + return "array" + if isinstance(value, dict): + return "object" + return type(value).__name__ + + +def collect_schema(value, prefix=""): + paths = {} + paths[prefix or "$"] = type_name(value) + if isinstance(value, dict): + for key, child in value.items(): + child_prefix = f"{prefix}.{key}" if prefix else key + paths.update(collect_schema(child, child_prefix)) + elif isinstance(value, list): + if value: + item_types = sorted({type_name(item) for item in value}) + paths[f"{prefix}[]"] = "|".join(item_types) + for item in value: + if isinstance(item, (dict, list)): + paths.update(collect_schema(item, f"{prefix}[]")) + else: + paths[f"{prefix}[]"] = "empty" + return paths + + +def schema_by_event_type(events): + by_type = defaultdict(dict) + for event in events: + event_type = event_kind(event) + if not isinstance(event_type, str): + event_type = "" + for path, kind in collect_schema(event).items(): + kinds = by_type[event_type].setdefault(path, set()) + kinds.add(kind) + return { + event_type: { + path: sorted(kinds) + for path, kinds in sorted(paths.items()) + } + for event_type, paths in sorted(by_type.items()) + } + + +def diff_schema(baseline_schema, candidate_schema): + diff = {} + event_types = sorted(set(baseline_schema) | set(candidate_schema)) + for event_type in event_types: + baseline_paths = baseline_schema.get(event_type, {}) + candidate_paths = candidate_schema.get(event_type, {}) + missing_paths = sorted(set(baseline_paths) - set(candidate_paths)) + extra_paths = sorted(set(candidate_paths) - set(baseline_paths)) + changed_types = [] + for path in sorted(set(baseline_paths) & set(candidate_paths)): + if baseline_paths[path] != candidate_paths[path]: + changed_types.append({ + "path": path, + "baseline": baseline_paths[path], + "candidate": candidate_paths[path], + }) + if missing_paths or extra_paths or changed_types: + diff[event_type] = { + "missingPaths": missing_paths, + "extraPaths": extra_paths, + "changedTypes": changed_types, + } + return diff + + +def mcp_response_shape_evidence(transcript): + evidence = {} + if not isinstance(transcript, dict): + return evidence + for raw_key, tool_name in { + **MCP_RESPONSE_SHAPES, + **{alias: MCP_RESPONSE_SHAPES[shape] for alias, shape in MCP_RESPONSE_SHAPE_ALIASES.items()}, + }.items(): + response = transcript.get(raw_key) + if not isinstance(response, dict): + continue + shape_key = MCP_RESPONSE_SHAPE_ALIASES.get(raw_key, raw_key) + evidence[shape_key] = { + "toolName": tool_name, + "hasResult": "result" in response, + "hasError": "error" in response, + "timedOut": "timeout" in response, + "schema": collect_schema(response), + } + return evidence + + +def mcp_response_shape_errors(baseline_evidence, candidate_evidence, require_same_schema): + errors = [] + for shape_key in sorted(baseline_evidence): + baseline_shape = baseline_evidence[shape_key] + candidate_shape = candidate_evidence.get(shape_key) + if candidate_shape is None: + errors.append(f"candidate missing MCP response shape: {shape_key}") + continue + for flag in ["hasResult", "hasError", "timedOut"]: + if baseline_shape.get(flag) != candidate_shape.get(flag): + errors.append( + f"candidate MCP response shape {shape_key} {flag} differs " + f"(baseline={baseline_shape.get(flag)}, candidate={candidate_shape.get(flag)})" + ) + if require_same_schema: + baseline_schema = baseline_shape.get("schema", {}) + candidate_schema = candidate_shape.get("schema", {}) + missing_paths = sorted(set(baseline_schema) - set(candidate_schema)) + changed_types = [ + { + "path": path, + "baseline": baseline_schema[path], + "candidate": candidate_schema[path], + } + for path in sorted(set(baseline_schema) & set(candidate_schema)) + if baseline_schema[path] != candidate_schema[path] + ] + if missing_paths: + errors.append( + f"candidate MCP response shape {shape_key} missing schema paths: {','.join(missing_paths)}" + ) + if changed_types: + changed_paths = ",".join(change["path"] for change in changed_types) + errors.append( + f"candidate MCP response shape {shape_key} changed schema types: {changed_paths}" + ) + return errors + + +def semantic_field_evidence(events): + schemas = schema_by_event_type(events) + evidence = {} + for event_type, required_paths in sorted(SEMANTIC_SCHEMA_PATHS_BY_EVENT_TYPE.items()): + event_schema = schemas.get(event_type, {}) + present_paths = sorted(set(event_schema) & required_paths) + if present_paths or event_type in schemas: + evidence[event_type] = { + "eventCount": sum(1 for event in events if event_kind(event) == event_type), + "presentPaths": present_paths, + } + return evidence + + +def semantic_field_errors(baseline_evidence, candidate_evidence): + errors = [] + for event_type in sorted(baseline_evidence): + baseline_paths = set(baseline_evidence[event_type].get("presentPaths", [])) + if not baseline_paths: + continue + candidate_paths = set(candidate_evidence.get(event_type, {}).get("presentPaths", [])) + missing_paths = sorted(baseline_paths - candidate_paths) + if missing_paths: + errors.append( + f"candidate missing semantic fields for {event_type}: {','.join(missing_paths)}" + ) + return errors + + +def metadata_key_diff(baseline_metadata, candidate_metadata): + baseline_keys = set(baseline_metadata.keys()) + candidate_keys = set(candidate_metadata.keys()) + return { + "missingKeys": sorted(baseline_keys - candidate_keys), + "extraKeys": sorted(candidate_keys - baseline_keys), + } + + +def metadata_value_diff(baseline_metadata, candidate_metadata): + changed_values = [] + missing_keys = [] + checked_keys = [] + for key in STABLE_METADATA_VALUE_KEYS: + if key not in baseline_metadata: + continue + checked_keys.append(key) + if key not in candidate_metadata: + missing_keys.append(key) + continue + if baseline_metadata[key] != candidate_metadata[key]: + changed_values.append({ + "key": key, + "baseline": baseline_metadata[key], + "candidate": candidate_metadata[key], + }) + return { + "checkedKeys": checked_keys, + "missingKeys": missing_keys, + "changedValues": changed_values, + } + + +def lifecycle_session_evidence(events): + started_indexes = [ + index + for index, event in enumerate(events) + if event_kind(event) == "session.started" + ] + ended_indexes = [ + index + for index, event in enumerate(events) + if event_kind(event) == "session.ended" + ] + end_reasons = sorted({ + event.get("endReason") + for event in events + if event_kind(event) == "session.ended" + and isinstance(event.get("endReason"), str) + }) + first_event_type = event_kind(events[0]) if events else None + last_event_type = event_kind(events[-1]) if events else None + return { + "eventCount": len(events), + "sessionStartedCount": len(started_indexes), + "sessionStartedIndexes": started_indexes, + "firstEventType": first_event_type, + "hasInitialSessionStarted": first_event_type == "session.started", + "sessionEndedCount": len(ended_indexes), + "sessionEndedIndexes": ended_indexes, + "lastEventType": last_event_type, + "hasFinalSessionEnded": last_event_type == "session.ended", + "endReasons": end_reasons, + } + + +def lifecycle_session_evidence_errors(baseline_evidence, candidate_evidence): + errors = [] + if baseline_evidence["sessionStartedCount"] != candidate_evidence["sessionStartedCount"]: + errors.append( + "session.started count differs " + f"(baseline={baseline_evidence['sessionStartedCount']}, " + f"candidate={candidate_evidence['sessionStartedCount']})" + ) + if baseline_evidence["hasInitialSessionStarted"] and not candidate_evidence["hasInitialSessionStarted"]: + errors.append("candidate first event is not session.started") + if baseline_evidence["sessionEndedCount"] != candidate_evidence["sessionEndedCount"]: + errors.append( + "session.ended count differs " + f"(baseline={baseline_evidence['sessionEndedCount']}, " + f"candidate={candidate_evidence['sessionEndedCount']})" + ) + if baseline_evidence["hasFinalSessionEnded"] and not candidate_evidence["hasFinalSessionEnded"]: + errors.append("candidate final event is not session.ended") + baseline_reasons = set(baseline_evidence["endReasons"]) + candidate_reasons = set(candidate_evidence["endReasons"]) + missing_reasons = sorted(baseline_reasons - candidate_reasons) + extra_reasons = sorted(candidate_reasons - baseline_reasons) + if missing_reasons: + errors.append(f"candidate missing session.ended endReason: {','.join(missing_reasons)}") + if extra_reasons: + errors.append(f"candidate has extra session.ended endReason: {','.join(extra_reasons)}") + return errors + + +def first_sequence_difference(baseline, candidate): + max_len = max(len(baseline), len(candidate)) + for index in range(max_len): + baseline_value = baseline[index] if index < len(baseline) else None + candidate_value = candidate[index] if index < len(candidate) else None + if baseline_value != candidate_value: + return { + "index": index, + "baseline": baseline_value, + "candidate": candidate_value, + } + return None + + +def event_type_sequence(events): + return [ + event_kind(event) or "" + for event in events + ] + + +def event_type_count_diff(baseline_sequence, candidate_sequence): + baseline_counts = Counter(type_ for type_ in baseline_sequence if isinstance(type_, str)) + candidate_counts = Counter(type_ for type_ in candidate_sequence if isinstance(type_, str)) + return { + event_type: { + "baseline": baseline_counts[event_type], + "candidate": candidate_counts[event_type], + } + for event_type in sorted(set(baseline_counts) | set(candidate_counts)) + if baseline_counts[event_type] != candidate_counts[event_type] + } + + +def prune_extra_schema(schema_diff): + return { + event_type: { + "missingPaths": details["missingPaths"], + "extraPaths": [], + "changedTypes": details["changedTypes"], + } + for event_type, details in schema_diff.items() + if details["missingPaths"] or details["changedTypes"] + } + + +def iter_dicts(value): + if isinstance(value, dict): + yield value + for child in value.values(): + yield from iter_dicts(child) + elif isinstance(value, list): + for child in value: + yield from iter_dicts(child) + + +def accessibility_payloads(events): + payloads = [] + seen_ids = set() + for event in events: + for value in iter_dicts(event): + payload = value.get("accessibilityInspectorPayload") + if isinstance(payload, dict) and id(payload) not in seen_ids: + payloads.append(payload) + seen_ids.add(id(payload)) + official_ax_payload = value.get("ax") + if isinstance(official_ax_payload, dict) and id(official_ax_payload) not in seen_ids: + payloads.append(official_ax_payload) + seen_ids.add(id(official_ax_payload)) + if event_kind(event) == "AX.focusedWindowChanged": + payload = event.get("accessibilityInspectorPayload") + if isinstance(payload, dict) and id(payload) not in seen_ids: + payloads.append(payload) + seen_ids.add(id(payload)) + official_ax_payload = event.get("ax") + if isinstance(official_ax_payload, dict) and id(official_ax_payload) not in seen_ids: + payloads.append(official_ax_payload) + seen_ids.add(id(official_ax_payload)) + return payloads + + +def is_full_ax_payload(payload): + return ( + payload.get("kind") == "full" + or payload.get("mode") in {"full", "fullTree"} + or payload.get("diffFromPrevious") is False + or isinstance(payload.get("fullTree"), (list, str)) + ) + + +def is_full_tree_payload(payload): + return ( + isinstance(payload.get("fullTree"), (list, str)) and bool(payload.get("fullTree")) + ) or payload.get("mode") == "fullTree" + + +def diff_markers(lines): + markers = [] + for line in lines: + if not isinstance(line, str): + continue + stripped = line.lstrip() + if stripped.startswith(("+", "-", "~")): + markers.append(stripped[0]) + return sorted(set(markers)) + + +def ax_diff_evidence(events): + payloads = accessibility_payloads(events) + diff_payloads = [ + payload + for payload in payloads + if payload.get("kind") == "diff" or payload.get("diffFromPrevious") is True + ] + cumulative_payloads = [ + payload + for payload in payloads + if payload.get("cumulativeDiffFromInitial") is True + or isinstance(payload.get("cumulativeRenderedText"), str) + ] + diff_marker_set = set() + cumulative_marker_set = set() + for payload in diff_payloads: + tree_lines = payload.get("treeLines") + if isinstance(tree_lines, list): + diff_marker_set.update(diff_markers(tree_lines)) + for payload in cumulative_payloads: + tree_lines = payload.get("cumulativeTreeLines") + if isinstance(tree_lines, list): + cumulative_marker_set.update(diff_markers(tree_lines)) + + return { + "payloadCount": len(payloads), + "hasFullPayload": any( + is_full_ax_payload(payload) + for payload in payloads + ), + "hasFullTree": any( + is_full_tree_payload(payload) + for payload in payloads + ), + "hasDiffPayload": bool(diff_payloads), + "hasCumulativeDiff": bool(cumulative_payloads), + "diffMarkers": sorted(diff_marker_set), + "cumulativeDiffMarkers": sorted(cumulative_marker_set), + } + + +def ax_diff_evidence_errors(baseline_evidence, candidate_evidence, require_same_markers): + errors = [] + if baseline_evidence["hasDiffPayload"] and not candidate_evidence["hasDiffPayload"]: + errors.append("candidate missing AX diff payload") + if baseline_evidence["hasCumulativeDiff"] and not candidate_evidence["hasCumulativeDiff"]: + errors.append("candidate missing cumulative AX diff payload") + + baseline_diff_markers = set(baseline_evidence["diffMarkers"]) + candidate_diff_markers = set(candidate_evidence["diffMarkers"]) + missing_diff_markers = sorted(baseline_diff_markers - candidate_diff_markers) + extra_diff_markers = sorted(candidate_diff_markers - baseline_diff_markers) + if missing_diff_markers: + errors.append(f"candidate missing AX diff markers: {','.join(missing_diff_markers)}") + if require_same_markers and extra_diff_markers: + errors.append(f"candidate has extra AX diff markers: {','.join(extra_diff_markers)}") + + baseline_cumulative_markers = set(baseline_evidence["cumulativeDiffMarkers"]) + candidate_cumulative_markers = set(candidate_evidence["cumulativeDiffMarkers"]) + missing_cumulative_markers = sorted(baseline_cumulative_markers - candidate_cumulative_markers) + extra_cumulative_markers = sorted(candidate_cumulative_markers - baseline_cumulative_markers) + if missing_cumulative_markers: + errors.append(f"candidate missing cumulative AX diff markers: {','.join(missing_cumulative_markers)}") + if require_same_markers and extra_cumulative_markers: + errors.append(f"candidate has extra cumulative AX diff markers: {','.join(extra_cumulative_markers)}") + + return errors + + +def compare(baseline_path, candidate_path, ignore_extra_schema): + baseline = resolve_recording(baseline_path) + candidate = resolve_recording(candidate_path) + + baseline_sequence = event_type_sequence(baseline["events"]) + candidate_sequence = event_type_sequence(candidate["events"]) + baseline_counts = Counter(type_ for type_ in baseline_sequence if isinstance(type_, str)) + candidate_counts = Counter(type_ for type_ in candidate_sequence if isinstance(type_, str)) + baseline_suppressed_sequence = event_type_sequence(baseline["suppressed"]) + candidate_suppressed_sequence = event_type_sequence(candidate["suppressed"]) + + baseline_schema = schema_by_event_type(baseline["events"]) + candidate_schema = schema_by_event_type(candidate["events"]) + baseline_suppressed_schema = schema_by_event_type(baseline["suppressed"]) + candidate_suppressed_schema = schema_by_event_type(candidate["suppressed"]) + baseline_semantic_evidence = semantic_field_evidence(baseline["events"]) + candidate_semantic_evidence = semantic_field_evidence(candidate["events"]) + schema_diff = diff_schema(baseline_schema, candidate_schema) + suppressed_schema_diff = diff_schema(baseline_suppressed_schema, candidate_suppressed_schema) + if ignore_extra_schema: + schema_diff = prune_extra_schema(schema_diff) + suppressed_schema_diff = prune_extra_schema(suppressed_schema_diff) + + missing_event_types = sorted(set(baseline_counts) - set(candidate_counts)) + extra_event_types = sorted(set(candidate_counts) - set(baseline_counts)) + count_differences = { + event_type: { + "baseline": baseline_counts[event_type], + "candidate": candidate_counts[event_type], + } + for event_type in sorted(set(baseline_counts) | set(candidate_counts)) + if baseline_counts[event_type] != candidate_counts[event_type] + } + suppressed_count_differences = event_type_count_diff( + baseline_suppressed_sequence, + candidate_suppressed_sequence, + ) + baseline_ax_diff_evidence = ax_diff_evidence(baseline["events"]) + candidate_ax_diff_evidence = ax_diff_evidence(candidate["events"]) + baseline_final_session_evidence = lifecycle_session_evidence(baseline["events"]) + candidate_final_session_evidence = lifecycle_session_evidence(candidate["events"]) + baseline_mcp_response_shape_evidence = mcp_response_shape_evidence(baseline["mcpTranscript"]) + candidate_mcp_response_shape_evidence = mcp_response_shape_evidence(candidate["mcpTranscript"]) + metadata_diff = metadata_key_diff(baseline["metadata"], candidate["metadata"]) + metadata_value_drift = metadata_value_diff(baseline["metadata"], candidate["metadata"]) + baseline_handoff_path_evidence = handoff_path_evidence(baseline["sessionDir"], baseline["metadata"]) + candidate_handoff_path_evidence = handoff_path_evidence(candidate["sessionDir"], candidate["metadata"]) + + return { + "baseline": { + "sessionDir": str(baseline["sessionDir"]), + "metadataPath": str(baseline["metadataPath"]) if baseline["metadataPath"] else None, + "eventsPath": str(baseline["eventsPath"]), + "suppressedPath": str(baseline["suppressedPath"]), + "mcpTranscriptPath": str(baseline["mcpTranscriptPath"]) if baseline["mcpTranscriptPath"] else None, + "eventCount": len(baseline["events"]), + "suppressedEventCount": len(baseline["suppressed"]), + "metadataKeys": sorted(baseline["metadata"].keys()), + }, + "candidate": { + "sessionDir": str(candidate["sessionDir"]), + "metadataPath": str(candidate["metadataPath"]) if candidate["metadataPath"] else None, + "eventsPath": str(candidate["eventsPath"]), + "suppressedPath": str(candidate["suppressedPath"]), + "mcpTranscriptPath": str(candidate["mcpTranscriptPath"]) if candidate["mcpTranscriptPath"] else None, + "eventCount": len(candidate["events"]), + "suppressedEventCount": len(candidate["suppressed"]), + "metadataKeys": sorted(candidate["metadata"].keys()), + }, + "eventSequenceEqual": baseline_sequence == candidate_sequence, + "firstEventSequenceDifference": first_sequence_difference(baseline_sequence, candidate_sequence), + "missingEventTypes": missing_event_types, + "extraEventTypes": extra_event_types, + "eventCountDifferences": count_differences, + "suppressedEventSequenceEqual": baseline_suppressed_sequence == candidate_suppressed_sequence, + "firstSuppressedEventSequenceDifference": first_sequence_difference( + baseline_suppressed_sequence, + candidate_suppressed_sequence, + ), + "suppressedEventCountDifferences": suppressed_count_differences, + "metadataKeysEqual": not metadata_diff["missingKeys"] and not metadata_diff["extraKeys"], + "metadataKeyDiff": metadata_diff, + "metadataStableValuesEqual": ( + not metadata_value_drift["missingKeys"] + and not metadata_value_drift["changedValues"] + ), + "metadataValueDiff": metadata_value_drift, + "handoffPathEvidence": { + "baseline": baseline_handoff_path_evidence, + "candidate": candidate_handoff_path_evidence, + }, + "schemaEqual": not schema_diff, + "schemaDiff": schema_diff, + "suppressedSchemaEqual": not suppressed_schema_diff, + "suppressedSchemaDiff": suppressed_schema_diff, + "semanticFieldEvidence": { + "baseline": baseline_semantic_evidence, + "candidate": candidate_semantic_evidence, + }, + "axDiffEvidence": { + "baseline": baseline_ax_diff_evidence, + "candidate": candidate_ax_diff_evidence, + }, + "finalSessionEvidence": { + "baseline": baseline_final_session_evidence, + "candidate": candidate_final_session_evidence, + }, + "mcpResponseShapeEvidence": { + "baseline": baseline_mcp_response_shape_evidence, + "candidate": candidate_mcp_response_shape_evidence, + }, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Compare two Record & Replay recording event streams by sequence, JSON schema, and AX diff evidence." + ) + parser.add_argument("baseline", type=pathlib.Path, help="Official/golden recording directory or metadata/events file.") + parser.add_argument("candidate", type=pathlib.Path, help="Candidate OCU recording directory or metadata/events file.") + parser.add_argument( + "--require-same-event-sequence", + action="store_true", + help="Exit non-zero when event type sequences differ.", + ) + parser.add_argument( + "--require-same-schema", + action="store_true", + help="Exit non-zero when per-event JSON schema paths or value types differ.", + ) + parser.add_argument( + "--require-same-suppressed-event-sequence", + action="store_true", + help="Exit non-zero when suppressed.jsonl event type sequences differ.", + ) + parser.add_argument( + "--require-same-suppressed-schema", + action="store_true", + help="Exit non-zero when suppressed.jsonl JSON schema paths or value types differ.", + ) + parser.add_argument( + "--require-same-metadata-keys", + action="store_true", + help="Exit non-zero when metadata key sets differ.", + ) + parser.add_argument( + "--require-same-metadata-values", + action="store_true", + help="Exit non-zero when stable metadata values differ.", + ) + parser.add_argument( + "--ignore-extra-schema", + action="store_true", + help="Only fail/report schema paths missing from candidate or changed in type; ignore candidate-only fields.", + ) + parser.add_argument( + "--require-ax-diff-evidence", + action="store_true", + help="Fail when the baseline has AX diff/cumulative evidence that the candidate is missing.", + ) + parser.add_argument( + "--require-semantic-fields", + action="store_true", + help="Fail when the candidate is missing replay-critical semantic fields present in the baseline.", + ) + parser.add_argument( + "--require-final-session-evidence", + action="store_true", + help="Fail when candidate final session.ended evidence differs from the baseline.", + ) + parser.add_argument( + "--require-same-ax-diff-markers", + action="store_true", + help="With --require-ax-diff-evidence, also fail when candidate has extra AX diff marker types.", + ) + parser.add_argument( + "--require-mcp-response-shapes", + action="store_true", + help=( + "Fail when baseline mcp-transcript.json has start/status/stop response shapes " + "that the candidate transcript is missing or changes from result/error/timeout." + ), + ) + parser.add_argument( + "--require-same-mcp-response-schema", + action="store_true", + help="With --require-mcp-response-shapes, also require candidate response shape schemas to keep baseline paths and types.", + ) + parser.add_argument( + "--require-handoff-paths", + action="store_true", + help="Fail unless both recordings declare metadataPath, sessionPath, eventsPath, and suppressedEventsPath that resolve to existing files.", + ) + args = parser.parse_args() + + try: + result = compare(args.baseline, args.candidate, args.ignore_extra_schema) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + + errors = [] + if args.require_same_event_sequence and not result["eventSequenceEqual"]: + errors.append("event type sequence differs") + if args.require_same_schema and not result["schemaEqual"]: + errors.append("event schema differs") + if args.require_same_suppressed_event_sequence and not result["suppressedEventSequenceEqual"]: + errors.append("suppressed event type sequence differs") + if args.require_same_suppressed_schema and not result["suppressedSchemaEqual"]: + errors.append("suppressed event schema differs") + if args.require_same_metadata_keys and not result["metadataKeysEqual"]: + errors.append("metadata keys differ") + if args.require_same_metadata_values and not result["metadataStableValuesEqual"]: + errors.append("stable metadata values differ") + if args.require_ax_diff_evidence: + errors.extend( + ax_diff_evidence_errors( + result["axDiffEvidence"]["baseline"], + result["axDiffEvidence"]["candidate"], + args.require_same_ax_diff_markers, + ) + ) + if args.require_semantic_fields: + errors.extend( + semantic_field_errors( + result["semanticFieldEvidence"]["baseline"], + result["semanticFieldEvidence"]["candidate"], + ) + ) + if args.require_final_session_evidence: + errors.extend( + lifecycle_session_evidence_errors( + result["finalSessionEvidence"]["baseline"], + result["finalSessionEvidence"]["candidate"], + ) + ) + if args.require_mcp_response_shapes: + errors.extend( + mcp_response_shape_errors( + result["mcpResponseShapeEvidence"]["baseline"], + result["mcpResponseShapeEvidence"]["candidate"], + args.require_same_mcp_response_schema, + ) + ) + if args.require_handoff_paths: + errors.extend( + handoff_path_errors( + result["handoffPathEvidence"]["baseline"], + result["handoffPathEvidence"]["candidate"], + ) + ) + result["ok"] = not errors + result["errors"] = errors + + output = json.dumps(result, indent=2, sort_keys=True) + if errors: + print(output, file=sys.stderr) + return 1 + print(output) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/compare-event-stream-surface.py b/scripts/compare-event-stream-surface.py new file mode 100755 index 00000000..cf059d7e --- /dev/null +++ b/scripts/compare-event-stream-surface.py @@ -0,0 +1,282 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import select +import subprocess +import sys +from typing import Any + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_FIXTURE = ( + REPO_ROOT + / "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-event-stream-surface-1.0.857.json" +) +DEFAULT_OFFICIAL_PLUGIN_DIR = ( + pathlib.Path.home() + / ".codex/plugins/cache/openai-bundled/record-and-replay/1.0.857" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Compare Record & Replay event-stream MCP initialize/tools-list surface." + ) + parser.add_argument( + "--fixture", + default=str(DEFAULT_FIXTURE), + help="Normalized official surface fixture JSON.", + ) + parser.add_argument( + "--local-command", + default=str(REPO_ROOT / ".build/debug/OpenComputerUse"), + help="Local OpenComputerUse binary to launch with 'event-stream mcp'.", + ) + parser.add_argument( + "--skip-build", + action="store_true", + help="Do not run 'swift build --product OpenComputerUse' before local comparison.", + ) + parser.add_argument( + "--official-plugin-dir", + default=os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_PLUGIN_DIR"), + help="Official record-and-replay plugin directory. If omitted, official comparison is skipped unless --use-default-official is set.", + ) + parser.add_argument( + "--use-default-official", + action="store_true", + help=f"Probe the default official plugin path: {DEFAULT_OFFICIAL_PLUGIN_DIR}", + ) + parser.add_argument( + "--require-official", + action="store_true", + help="Fail if the official plugin/client is unavailable.", + ) + parser.add_argument( + "--timeout", + type=float, + default=float(os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT", "10")), + help="Seconds to wait for each MCP response.", + ) + return parser.parse_args() + + +def load_json(path: pathlib.Path) -> dict[str, Any]: + with path.open("r", encoding="utf-8") as handle: + value = json.load(handle) + if not isinstance(value, dict): + raise AssertionError(f"expected JSON object at {path}") + return value + + +def canonical(value: Any) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def client_for_official_plugin(plugin_dir: pathlib.Path) -> pathlib.Path: + return ( + plugin_dir + / "Codex Computer Use.app/Contents/SharedSupport/SkyComputerUseClient.app/Contents/MacOS/SkyComputerUseClient" + ) + + +def capture_surface( + command: list[str], + cwd: pathlib.Path | None, + timeout: float, + env: dict[str, str] | None = None, +) -> dict[str, Any]: + proc = subprocess.Popen( + command, + cwd=str(cwd) if cwd else None, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + def request(message: dict[str, Any], expect_response: bool = True) -> dict[str, Any] | None: + assert proc.stdin is not None + assert proc.stdout is not None + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + ready, _, _ = select.select([proc.stdout], [], [], timeout) + if not ready: + raise TimeoutError(f"MCP server did not respond within {timeout}s: {command}") + line = proc.stdout.readline() + if not line: + raise AssertionError(f"MCP server exited before responding: {command}") + response = json.loads(line) + if "error" in response: + raise AssertionError(response["error"]) + return response + + try: + initialize_response = request( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { + "name": "event-stream-surface-compare", + "version": "0", + }, + }, + } + ) + request( + {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, + expect_response=False, + ) + tools_response = request( + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}} + ) + finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + stderr = proc.stderr.read() if proc.stderr else "" + if proc.returncode != 0: + raise AssertionError(stderr) + + assert initialize_response is not None + assert tools_response is not None + initialize = initialize_response["result"] + tools = tools_response["result"]["tools"] + return {"initialize": initialize, "tools": tools} + + +def comparable_surface(surface: dict[str, Any], ignore_server_version: bool) -> dict[str, Any]: + initialize = dict(surface["initialize"]) + server_info = dict(initialize["serverInfo"]) + if ignore_server_version: + server_info.pop("version", None) + initialize["serverInfo"] = server_info + return { + "initialize": initialize, + "tools": surface["tools"], + } + + +def compare_surface( + label: str, + actual: dict[str, Any], + fixture: dict[str, Any], + ignore_server_version: bool, +) -> dict[str, Any]: + comparable_actual = comparable_surface(actual, ignore_server_version=ignore_server_version) + comparable_fixture = comparable_surface(fixture, ignore_server_version=ignore_server_version) + ok = canonical(comparable_actual) == canonical(comparable_fixture) + result: dict[str, Any] = { + "label": label, + "ok": ok, + "ignoreServerVersion": ignore_server_version, + "protocolVersion": actual["initialize"].get("protocolVersion"), + "serverName": actual["initialize"].get("serverInfo", {}).get("name"), + "toolNames": [tool.get("name") for tool in actual.get("tools", [])], + } + if not ok: + result["expected"] = comparable_fixture + result["actual"] = comparable_actual + return result + + +def main() -> int: + args = parse_args() + fixture_path = pathlib.Path(args.fixture) + fixture = load_json(fixture_path) + + if not args.skip_build: + subprocess.run( + ["swift", "build", "--product", "OpenComputerUse"], + cwd=str(REPO_ROOT), + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + results: list[dict[str, Any]] = [] + local_surface = capture_surface( + [args.local_command, "event-stream", "mcp"], + cwd=REPO_ROOT, + timeout=args.timeout, + env={**os.environ, "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1"}, + ) + results.append( + compare_surface( + "local-open-computer-use", + local_surface, + fixture, + ignore_server_version=True, + ) + ) + + official_plugin_dir = args.official_plugin_dir + if args.use_default_official and not official_plugin_dir: + official_plugin_dir = str(DEFAULT_OFFICIAL_PLUGIN_DIR) + + official_result: dict[str, Any] | None = None + if official_plugin_dir: + plugin_dir = pathlib.Path(official_plugin_dir) + official_client = client_for_official_plugin(plugin_dir) + if official_client.exists(): + official_surface = capture_surface( + [str(official_client), "event-stream", "mcp"], + cwd=plugin_dir, + timeout=args.timeout, + ) + official_result = compare_surface( + "official-record-and-replay", + official_surface, + fixture, + ignore_server_version=False, + ) + official_result["pluginDir"] = str(plugin_dir) + official_result["client"] = str(official_client) + results.append(official_result) + else: + official_result = { + "label": "official-record-and-replay", + "ok": not args.require_official, + "skipped": True, + "reason": "officialClientNotFound", + "pluginDir": str(plugin_dir), + "client": str(official_client), + } + results.append(official_result) + elif args.require_official: + official_result = { + "label": "official-record-and-replay", + "ok": False, + "skipped": True, + "reason": "officialPluginDirNotProvided", + } + results.append(official_result) + + payload = { + "ok": all(result.get("ok") is True for result in results), + "fixture": str(fixture_path), + "results": results, + } + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 if payload["ok"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/finalize-record-and-replay-official-capture-packet.py b/scripts/finalize-record-and-replay-official-capture-packet.py new file mode 100755 index 00000000..e2b0dda8 --- /dev/null +++ b/scripts/finalize-record-and-replay-official-capture-packet.py @@ -0,0 +1,331 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys +from typing import Any + + +RESPONSE_SHAPE_KEYS = { + "startResponseShape": ("start", "startResponse", "startResponseShape"), + "statusResponseShape": ("status", "statusResponse", "statusResponseShape"), + "stopResponseShape": ("stop", "stopResponse", "stopResponseShape"), + "finalStatusResponseShape": ( + "finalStatus", + "finalStatusResponse", + "finalStatusResponseShape", + ), + "repeatStartResponseShape": ( + "repeatStart", + "repeatStartResponse", + "repeatStartResponseShape", + ), + "repeatStopResponseShape": ( + "repeatStop", + "repeatStopResponse", + "repeatStopResponseShape", + ), +} + +REQUIRED_HOSTED_SHAPES = ( + "startResponseShape", + "statusResponseShape", + "stopResponseShape", + "finalStatusResponseShape", +) + +HANDOFF_PATH_KEYS = ( + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + "sessionDirectoryPath", +) + + +def load_json(path: pathlib.Path, label: str) -> Any: + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing {label} JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid {label} JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + + +def write_json(path: pathlib.Path, value: Any) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False) + "\n") + + +def decode_json_string(value: str) -> Any | None: + try: + return json.loads(value) + except json.JSONDecodeError: + return None + + +def iter_json_objects(value: Any): + if isinstance(value, dict): + yield value + for child in value.values(): + if isinstance(child, str): + decoded = decode_json_string(child) + if decoded is not None: + yield from iter_json_objects(decoded) + continue + yield from iter_json_objects(child) + elif isinstance(value, list): + for child in value: + yield from iter_json_objects(child) + + +def handoff_path_keys(value: Any) -> list[str]: + return sorted( + { + key + for obj in iter_json_objects(value) + for key in HANDOFF_PATH_KEYS + if isinstance(obj.get(key), str) and obj.get(key) + } + ) + + +def unwrap_shape(value: Any) -> Any: + if isinstance(value, dict): + for key in ("startResponseShape", "statusResponseShape", "stopResponseShape", "finalStatusResponseShape"): + nested = value.get(key) + if isinstance(nested, dict): + return nested + return value + + +def as_tool_response_shape(value: Any) -> dict[str, Any]: + value = unwrap_shape(value) + if isinstance(value, dict) and any(key in value for key in ("result", "error", "timeout", "eof")): + return value + return { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps(value, sort_keys=True, ensure_ascii=False), + } + ] + } + } + + +def shape_is_usable(shape: Any) -> bool: + return isinstance(shape, dict) and ("result" in shape or "error" in shape) and "timeout" not in shape + + +def extract_capture_value(capture: Any, keys: tuple[str, ...]) -> Any | None: + if not isinstance(capture, dict): + return None + for key in keys: + if key in capture: + return capture[key] + return None + + +def load_inputs(args: argparse.Namespace) -> dict[str, Any]: + capture = load_json(args.capture_json, "capture") if args.capture_json else {} + values: dict[str, Any] = {} + individual_paths = { + "startResponseShape": args.start_json, + "statusResponseShape": args.status_json, + "stopResponseShape": args.stop_json, + "finalStatusResponseShape": args.final_status_json, + "repeatStartResponseShape": args.repeat_start_json, + "repeatStopResponseShape": args.repeat_stop_json, + } + for shape_key, aliases in RESPONSE_SHAPE_KEYS.items(): + if individual_paths[shape_key] is not None: + values[shape_key] = load_json(individual_paths[shape_key], shape_key) + continue + capture_value = extract_capture_value(capture, aliases) + if capture_value is not None: + values[shape_key] = capture_value + if "finalStatusResponseShape" not in values and args.use_stop_as_final_status: + if "stopResponseShape" in values: + values["finalStatusResponseShape"] = values["stopResponseShape"] + return values + + +def synthetic_transcript(shapes: dict[str, Any]) -> list[dict[str, Any]]: + calls = [ + ("event_stream_start", "startResponseShape"), + ("event_stream_status", "statusResponseShape"), + ("event_stream_stop", "stopResponseShape"), + ("event_stream_status", "finalStatusResponseShape"), + ] + if "repeatStartResponseShape" in shapes: + calls.insert(1, ("event_stream_start", "repeatStartResponseShape")) + if "repeatStopResponseShape" in shapes: + calls.insert(-1, ("event_stream_stop", "repeatStopResponseShape")) + transcript: list[dict[str, Any]] = [] + for index, (tool_name, shape_key) in enumerate(calls, start=1): + transcript.append( + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": index, + "method": "tools/call", + "params": {"name": tool_name, "arguments": {}}, + }, + } + ) + transcript.append( + { + "direction": "receive", + "message": { + **shapes[shape_key], + "id": index, + "jsonrpc": "2.0", + }, + } + ) + return transcript + + +def run_packet_check(packet_dir: pathlib.Path, script_name: str) -> dict[str, Any]: + script = packet_dir / script_name + completed = subprocess.run( + [str(script)], + cwd=str(packet_dir.parent), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "raw": raw.strip(), + } + return { + "ok": completed.returncode == 0, + "returncode": completed.returncode, + "result": parsed, + } + + +def finalize(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + packet_dir = args.packet_dir.resolve() + contract = load_json(packet_dir / "capture-contract.json", "capture contract") + inputs_dir = packet_dir / "inputs" + values = load_inputs(args) + missing_shapes = [key for key in REQUIRED_HOSTED_SHAPES if key not in values] + if missing_shapes: + raise ValueError( + "missing hosted response JSON for required shapes: " + ", ".join(missing_shapes) + ) + + shapes = {key: as_tool_response_shape(value) for key, value in values.items()} + unusable_shapes = [ + key + for key in REQUIRED_HOSTED_SHAPES + if not shape_is_usable(shapes.get(key)) + ] + if unusable_shapes: + raise ValueError( + "required hosted response shape has no result/error or timed out: " + + ", ".join(unusable_shapes) + ) + + status_source_key = "stopResponseShape" if args.status_input == "stop" else "finalStatusResponseShape" + status_input = values[status_source_key] + found_handoff_keys = handoff_path_keys(status_input) + if not found_handoff_keys: + raise ValueError( + "selected status input does not contain Record & Replay handoff paths: " + + ", ".join(HANDOFF_PATH_KEYS) + ) + + transcript_required = bool(contract.get("requiresMcpTranscriptInput")) + transcript = { + "fixtureFormatVersion": 1, + "kind": "record-and-replay-hosted-capture-transcript", + "source": "codex-hosted-record-and-replay", + "scenario": contract.get("scenario"), + "requiredHostedResponseShapes": list(REQUIRED_HOSTED_SHAPES), + **shapes, + "transcript": synthetic_transcript(shapes), + } + + status_path = inputs_dir / "event_stream_stop-response.json" + write_json(status_path, status_input) + transcript_path = inputs_dir / "mcp-transcript.json" + if transcript_required: + write_json(transcript_path, transcript) + + verify_workflow = run_packet_check(packet_dir, "verify-workflow.sh") + verify_inputs = run_packet_check(packet_dir, "verify-inputs.sh") + ok = verify_workflow["ok"] and verify_inputs["ok"] + return ( + 0 if ok else 1, + { + "ok": ok, + "stage": "finalize-hosted-capture-packet", + "packetDir": str(packet_dir), + "scenario": contract.get("scenario"), + "statusInputPath": str(status_path), + "mcpTranscriptInputPath": str(transcript_path) if transcript_required else None, + "statusInputSource": status_source_key, + "foundHandoffPathKeys": found_handoff_keys, + "checkedHostedResponseShapes": list(REQUIRED_HOSTED_SHAPES), + "verifyWorkflow": verify_workflow, + "verifyInputs": verify_inputs, + }, + ) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Finalize an official Record & Replay capture packet from Codex-hosted " + "event_stream_start/status/stop/final-status JSON responses." + ) + ) + parser.add_argument("--packet-dir", type=pathlib.Path, required=True) + parser.add_argument("--capture-json", type=pathlib.Path) + parser.add_argument("--start-json", type=pathlib.Path) + parser.add_argument("--status-json", type=pathlib.Path) + parser.add_argument("--stop-json", type=pathlib.Path) + parser.add_argument("--final-status-json", type=pathlib.Path) + parser.add_argument("--repeat-start-json", type=pathlib.Path) + parser.add_argument("--repeat-stop-json", type=pathlib.Path) + parser.add_argument( + "--status-input", + choices=["stop", "final-status"], + default="stop", + help="Which hosted response to write as inputs/event_stream_stop-response.json.", + ) + parser.add_argument( + "--use-stop-as-final-status", + action="store_true", + help="Use the stop response as finalStatusResponseShape when no final status JSON is available.", + ) + args = parser.parse_args() + try: + exit_code, result = finalize(args) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + output = json.dumps(result, indent=2, sort_keys=True) + if exit_code == 0: + print(output) + else: + print(output, file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/import-event-stream-fixture.py b/scripts/import-event-stream-fixture.py new file mode 100755 index 00000000..909a652e --- /dev/null +++ b/scripts/import-event-stream-fixture.py @@ -0,0 +1,501 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import json +import pathlib +import shutil +import sys +from collections import Counter + +from record_and_replay_scenarios import scenario_recipe + + +DEFAULT_OUTPUT_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) + +TEXT_KEYS = { + "AXDescription", + "AXSelectedText", + "AXTitle", + "AXValue", + "characters", + "description", + "label", + "rawCharacters", + "renderedText", + "selectedText", + "text", + "treeLines", + "title", + "value", + "windowTitle", + "cumulativeRenderedText", + "cumulativeTreeLines", + "diffRenderedText", + "fullTree", + "fullRenderedText", +} + +APP_ATTRIBUTION_KEYS = { + "appBundleIdentifier", + "appName", + "applicationName", + "bundleId", + "bundleIdentifier", + "localizedName", +} + +TIMESTAMP_KEYS = { + "capturedAt", + "createdAt", + "endedAt", + "startTime", + "startedAt", + "timestamp", + "updatedAt", +} + +SESSION_ID_KEYS = { + "id", + "sessionID", + "sessionId", +} + +RELATIVE_PATHS = { + "currentSegmentEventsPath": "events.jsonl", + "currentSegmentMetadataPath": "metadata.json", + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "suppressedEventsPath": "suppressed.jsonl", +} + +MCP_TRANSCRIPT_FILENAME = "mcp-transcript.json" + + +def should_redact_path_key(key): + return key == "path" or key.endswith("Path") + + +def load_json(path): + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def read_jsonl(path): + records = [] + if not path.exists(): + raise ValueError(f"missing JSONL file: {path}") + for index, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSONL in {path}:{index}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}:{index}") + records.append(value) + return records + + +def event_kind(event): + kind = event.get("kind") + if isinstance(kind, str): + return kind + event_type = event.get("type") + if isinstance(event_type, str): + return event_type + return None + + +def first_string(value, keys): + for key in keys: + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + return None + + +def recording_session_id(metadata, events): + session_id = first_string(metadata, ["sessionId", "sessionID", "id"]) + if session_id is not None: + return session_id + for event in events: + session_id = first_string(event, ["sessionId", "sessionID", "id"]) + if session_id is not None: + return session_id + return "fixture-session" + + +def recording_end_reason(metadata, events): + end_reason = metadata.get("endReason") + if isinstance(end_reason, str) and end_reason: + return end_reason + ended_reasons = [ + event.get("endReason") + for event in events + if event_kind(event) == "session.ended" and isinstance(event.get("endReason"), str) + ] + return ended_reasons[-1] if ended_reasons else None + + +def normalized_fixture_metadata(metadata, events, suppressed): + end_reason = recording_end_reason(metadata, events) + ended_at = first_string(metadata, ["endedAt"]) + state = metadata.get("state") if isinstance(metadata.get("state"), str) else None + if state is None: + state = "cancelled" if end_reason == "recording_controls_cancelled" else "stopped" + if ended_at is None and end_reason is None and not any(event_kind(event) == "session.ended" for event in events): + state = "recording" + + normalized = { + "sessionId": recording_session_id(metadata, events), + "state": state, + "active": state == "recording", + "eventCount": len(events), + "suppressedEventCount": len(suppressed), + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "suppressedEventsPath": "suppressed.jsonl", + } + started_at = first_string(metadata, ["startedAt"]) + if started_at is not None: + normalized["startedAt"] = started_at + if ended_at is not None: + normalized["endedAt"] = ended_at + if end_reason is not None: + normalized["endReason"] = end_reason + return normalized + + +def session_handoff_from_metadata(metadata): + handoff = { + "id": metadata["sessionId"], + "eventsPath": metadata["eventsPath"], + } + for key in ["startedAt", "endedAt", "endReason"]: + if key in metadata: + handoff[key] = metadata[key] + return handoff + + +def resolve_path(base, raw, fallback): + if isinstance(raw, str) and raw: + candidate = pathlib.Path(raw) + if candidate.is_absolute(): + return candidate + return base / candidate + return fallback + + +def resolve_input(path): + if path.is_dir(): + metadata_path = path / "metadata.json" + session_path = path / "session.json" + if metadata_path.exists(): + metadata = load_json(metadata_path) + elif session_path.exists(): + metadata_path = session_path + metadata = load_json(metadata_path) + else: + raise ValueError(f"session directory has no metadata.json or session.json: {path}") + events_path = resolve_path(path, metadata.get("eventsPath"), path / "events.jsonl") + suppressed_path = resolve_path(path, metadata.get("suppressedEventsPath"), path / "suppressed.jsonl") + return { + "session_dir": path, + "metadata_path": metadata_path, + "session_path": session_path if session_path.exists() else None, + "metadata": metadata, + "events_path": events_path, + "suppressed_path": suppressed_path, + } + + if not path.exists(): + raise ValueError(f"path does not exist: {path}") + if path.name not in {"metadata.json", "session.json"}: + raise ValueError("input must be a session directory, metadata.json, or session.json") + session_dir = path.parent + metadata = load_json(path) + events_path = resolve_path(session_dir, metadata.get("eventsPath"), session_dir / "events.jsonl") + suppressed_path = resolve_path( + session_dir, + metadata.get("suppressedEventsPath"), + session_dir / "suppressed.jsonl", + ) + return { + "session_dir": session_dir, + "metadata_path": path, + "session_path": session_dir / "session.json" if (session_dir / "session.json").exists() else None, + "metadata": metadata, + "events_path": events_path, + "suppressed_path": suppressed_path, + } + + +def redacted_string(key, value): + if key in SESSION_ID_KEYS: + return "fixture-session" + if key in TIMESTAMP_KEYS: + return "" + if key in APP_ATTRIBUTION_KEYS: + return f"" + return f"" + + +def redacted_tree_line(key, value): + stripped = value.lstrip() + prefix = value[: len(value) - len(stripped)] + if stripped.startswith(("+", "-", "~")): + return f"{prefix}{stripped[0]} " + return redacted_string(key, value) + + +def sanitize(value, parent_key=None, preserve_app_attribution=False): + if isinstance(value, dict): + result = {} + for key, child in value.items(): + if key in {"client", "command", "cwd", "recordingsDir"}: + result[f"{key}Redacted"] = True + continue + if key in RELATIVE_PATHS: + result[key] = RELATIVE_PATHS[key] + continue + if key == "screenshotPath": + result["screenshotPathRedacted"] = True + continue + if isinstance(child, str) and should_redact_path_key(key): + result[f"{key}Redacted"] = True + result[f"{key}Length"] = len(child) + continue + if key == "text" and isinstance(child, str): + decoded = decode_json_text(child) + result["textRedacted"] = True + if decoded is None: + result["textLength"] = len(child) + else: + result["decodedText"] = sanitize( + decoded, + key, + preserve_app_attribution, + ) + continue + if isinstance(child, str) and should_redact_key(key, preserve_app_attribution): + result[key] = redacted_string(key, child) + continue + result[key] = sanitize(child, key, preserve_app_attribution) + return result + if isinstance(value, list): + return [sanitize(child, parent_key, preserve_app_attribution) for child in value] + if isinstance(value, str) and parent_key in {"treeLines", "cumulativeTreeLines", "fullTree"}: + return redacted_tree_line(parent_key, value) + if isinstance(value, str) and parent_key and should_redact_key(parent_key, preserve_app_attribution): + return redacted_string(parent_key, value) + return value + + +def decode_json_text(value): + try: + decoded = json.loads(value) + except json.JSONDecodeError: + return None + if isinstance(decoded, (dict, list)): + return decoded + return None + + +def should_redact_key(key, preserve_app_attribution): + if key in TEXT_KEYS or key in TIMESTAMP_KEYS or key in SESSION_ID_KEYS: + return True + if not preserve_app_attribution and key in APP_ATTRIBUTION_KEYS: + return True + return False + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + lines = [json.dumps(record, separators=(",", ":"), ensure_ascii=False) for record in records] + path.write_text("\n".join(lines) + ("\n" if lines else "")) + + +def fixture_readme(name, source, official_version): + version_line = f"- Official plugin version: `{official_version}`\n" if official_version else "" + return f"""# {name} + +This is a sanitized Record & Replay event-stream fixture. + +- Source: `{source}` +{version_line}- Screenshots are not copied. +- User text, window titles, timestamps, session ids, local paths, and app attribution are redacted by default. + +Use `open-computer-use event-stream validate --json ` or +`scripts/validate-event-stream-recording.py ` to check structure. +""" + + +def import_fixture(args): + resolved = resolve_input(args.input) + events = read_jsonl(resolved["events_path"]) + suppressed = [] + suppressed_exists = resolved["suppressed_path"].exists() + if suppressed_exists: + suppressed = read_jsonl(resolved["suppressed_path"]) + + target = args.output_dir / args.name + if target.exists(): + if not args.force: + raise ValueError(f"fixture already exists: {target}; pass --force to replace it") + shutil.rmtree(target) + target.mkdir(parents=True) + + fixture_metadata = normalized_fixture_metadata(resolved["metadata"], events, suppressed) + sanitized_metadata = sanitize( + fixture_metadata, + preserve_app_attribution=args.preserve_app_attribution, + ) + sanitized_events = [ + sanitize(event, preserve_app_attribution=args.preserve_app_attribution) + for event in events + ] + sanitized_suppressed = [ + sanitize(event, preserve_app_attribution=args.preserve_app_attribution) + for event in suppressed + ] + + write_json(target / "metadata.json", sanitized_metadata) + if resolved["session_path"] is not None: + session_json = load_json(resolved["session_path"]) + write_json( + target / "session.json", + sanitize(session_json, preserve_app_attribution=args.preserve_app_attribution), + ) + else: + write_json( + target / "session.json", + sanitize( + session_handoff_from_metadata(fixture_metadata), + preserve_app_attribution=args.preserve_app_attribution, + ), + ) + write_jsonl(target / "events.jsonl", sanitized_events) + write_jsonl(target / "suppressed.jsonl", sanitized_suppressed) + + sanitized_mcp_transcript = None + if args.mcp_transcript: + mcp_transcript = load_json(args.mcp_transcript) + sanitized_mcp_transcript = sanitize( + mcp_transcript, + preserve_app_attribution=args.preserve_app_attribution, + ) + write_json(target / MCP_TRANSCRIPT_FILENAME, sanitized_mcp_transcript) + + event_types = Counter(kind for event in events if (kind := event_kind(event))) + manifest = { + "fixtureFormatVersion": 1, + "name": args.name, + "scenario": args.scenario, + "scenarioRecipe": scenario_recipe(args.scenario) if args.scenario else None, + "source": args.source, + "officialPluginVersion": args.official_plugin_version, + "capturedAt": args.captured_at, + "importedAt": datetime.date.today().isoformat(), + "eventCount": len(events), + "suppressedEventCount": len(suppressed), + "eventTypes": dict(sorted(event_types.items())), + "files": { + "metadata": "metadata.json", + "session": "session.json", + "events": "events.jsonl", + "suppressed": "suppressed.jsonl", + "mcpTranscript": MCP_TRANSCRIPT_FILENAME if sanitized_mcp_transcript is not None else None, + }, + "redaction": { + "screenshotsCopied": False, + "mcpTranscriptSanitized": sanitized_mcp_transcript is not None, + "preserveAppAttribution": args.preserve_app_attribution, + "textKeys": sorted(TEXT_KEYS), + "timestampKeys": sorted(TIMESTAMP_KEYS), + "sessionIdKeys": sorted(SESSION_ID_KEYS), + "pathKeysRewrittenRelative": sorted(RELATIVE_PATHS), + "pathKeysRedacted": ["path", "*Path"], + }, + } + write_json(target / "fixture-manifest.json", manifest) + (target / "README.md").write_text( + fixture_readme(args.name, args.source, args.official_plugin_version) + ) + return { + "ok": True, + "fixtureDir": str(target), + "eventCount": len(events), + "eventTypes": dict(sorted(event_types.items())), + "mcpTranscript": MCP_TRANSCRIPT_FILENAME if sanitized_mcp_transcript is not None else None, + } + + +def main(): + parser = argparse.ArgumentParser( + description="Import and sanitize a Record & Replay recording as a repository fixture." + ) + parser.add_argument("input", type=pathlib.Path, help="Session directory, metadata.json, or session.json") + parser.add_argument("--name", required=True, help="Fixture directory name to create.") + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=DEFAULT_OUTPUT_ROOT, + help=f"Fixture root directory. Defaults to {DEFAULT_OUTPUT_ROOT}.", + ) + parser.add_argument("--source", choices=["official", "ocu"], default="official") + parser.add_argument( + "--scenario", + help=( + "Machine-readable scenario tag, for example simple-action-stop. " + "Official fixture set gates use this to pair baseline and candidate recordings." + ), + ) + parser.add_argument("--official-plugin-version", help="Official Record & Replay plugin version, when known.") + parser.add_argument( + "--captured-at", + default=datetime.date.today().isoformat(), + help="Capture date to record in fixture-manifest.json.", + ) + parser.add_argument( + "--preserve-app-attribution", + action="store_true", + help="Keep app names and bundle identifiers when the fixture is already safe to publish.", + ) + parser.add_argument( + "--mcp-transcript", + type=pathlib.Path, + help=( + "Optional JSON probe output or MCP transcript to sanitize into mcp-transcript.json. " + "Use this for official golden fixtures that need event_stream_start/status/stop response evidence." + ), + ) + parser.add_argument("--force", action="store_true", help="Replace an existing fixture directory.") + args = parser.parse_args() + + try: + result = import_fixture(args) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ingest-ocu-record-and-replay-candidate.py b/scripts/ingest-ocu-record-and-replay-candidate.py new file mode 100755 index 00000000..48858a59 --- /dev/null +++ b/scripts/ingest-ocu-record-and-replay-candidate.py @@ -0,0 +1,489 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import datetime +import json +import pathlib +import subprocess +import sys +import tempfile +from typing import Any + +from record_and_replay_scenarios import ( + ACTION_SMOKE_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_readiness_args, +) + + +DEFAULT_OUTPUT_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates" +) +DEFAULT_OFFICIAL_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) +DEFAULT_SCENARIO = DEFAULT_REQUIRED_SCENARIOS[0] +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) + + +def load_json(path: pathlib.Path) -> Any: + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + + +def read_json_records(path: pathlib.Path) -> list[dict[str, Any]]: + records: list[dict[str, Any]] = [] + try: + lines = path.read_text().splitlines() + except FileNotFoundError: + raise ValueError(f"missing smoke JSONL file: {path}") + for line in lines: + stripped = line.strip() + if not stripped.startswith("{"): + continue + try: + value = json.loads(stripped) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + records.append(value) + return records + + +def select_smoke_record(records: list[dict[str, Any]], mode: str) -> dict[str, Any]: + matches = [record for record in records if record.get("mode") == mode] + if not matches: + raise ValueError(f"smoke JSONL has no record with mode={mode!r}") + return matches[-1] + + +def path_from_smoke_record(record: dict[str, Any], base_dir: pathlib.Path | None = None) -> pathlib.Path: + for key in ("metadataPath", "sessionPath"): + raw = record.get(key) + if isinstance(raw, str) and raw: + path = pathlib.Path(raw) + return path if path.is_absolute() or base_dir is None else base_dir / path + + recordings_root = record.get("recordingsRoot") + session_id = record.get("sessionId") + if isinstance(recordings_root, str) and recordings_root and isinstance(session_id, str) and session_id: + root = pathlib.Path(recordings_root) + if not root.is_absolute() and base_dir is not None: + root = base_dir / root + return root / session_id + + raise ValueError( + "smoke record must include metadataPath/sessionPath or recordingsRoot plus sessionId" + ) + + +def optional_path_from_smoke_record( + record: dict[str, Any], + key: str, + base_dir: pathlib.Path | None = None, +) -> pathlib.Path | None: + raw = record.get(key) + if not isinstance(raw, str) or not raw: + return None + path = pathlib.Path(raw) + return path if path.is_absolute() or base_dir is None else base_dir / path + + +def action_smoke_scenario_for_fixture_scenario(scenario: str) -> str: + if scenario in ACTION_SMOKE_SCENARIOS: + return scenario + if scenario == "keyboard-input-stop": + raise ValueError( + "keyboard-input-stop does not support --run-action-smoke yet; " + "import an existing recording with --recording or a preserved smoke JSONL with --smoke-json" + ) + return "mixed-action-stop" + + +def run_action_smoke( + repo: pathlib.Path, + output_dir: pathlib.Path, + scenario: str, +) -> tuple[list[dict[str, Any]], dict[str, Any]]: + action_scenario = action_smoke_scenario_for_fixture_scenario(scenario) + completed = subprocess.run( + [ + "env", + "OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1", + f"OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO={action_scenario}", + f"OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR={output_dir}", + "./scripts/run-event-stream-smoke-tests.sh", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + records: list[dict[str, Any]] = [] + for line in completed.stdout.splitlines(): + stripped = line.strip() + if not stripped.startswith("{"): + continue + try: + value = json.loads(stripped) + except json.JSONDecodeError: + continue + if isinstance(value, dict): + records.append(value) + evidence = { + "exitCode": completed.returncode, + "stdoutLineCount": len(completed.stdout.splitlines()), + "stderrLineCount": len(completed.stderr.splitlines()), + "jsonRecordCount": len(records), + "temporaryDirectory": str(output_dir), + "actionScenario": action_scenario, + } + if completed.returncode != 0: + raise ValueError( + "action smoke failed while capturing OCU candidate " + f"(exit={completed.returncode}): {completed.stderr.strip() or completed.stdout.strip()}" + ) + return records, evidence + + +def validation_args_for_scenario(scenario: str) -> list[str]: + args = ["--strict-ocu", "--require-event-type", "session.ended"] + if scenario == "simple-action-stop": + args.extend( + [ + "--require-event-type", + "mouse.click", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-skill-draft", + ] + ) + elif scenario == "keyboard-input-stop": + args.extend( + [ + "--require-event-type", + "keyboard.text_input", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-skill-draft", + ] + ) + elif scenario == "drag-stop": + args.extend( + [ + "--require-event-type", + "mouse.drag", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-skill-draft", + ] + ) + return args + + +def run_json(command: list[str], cwd: pathlib.Path) -> tuple[bool, dict[str, Any]]: + completed = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def shell_join(argv: list[str]) -> str: + import shlex + + return " ".join(shlex.quote(part) for part in argv) + + +def display_path(path: pathlib.Path, repo: pathlib.Path) -> str: + try: + return str(path.resolve().relative_to(repo.resolve())) + except (FileNotFoundError, ValueError): + return str(path) + + +def handoff_commands(repo: pathlib.Path, args: argparse.Namespace) -> dict[str, Any]: + official_root = args.official_root or DEFAULT_OFFICIAL_ROOT + candidate_root = args.output_dir + pairing_preflight = [ + "python3", + "scripts/prepare-record-and-replay-ocu-candidate-pairing.py", + "--scenario", + args.scenario, + "--official-root", + display_path(official_root, repo), + "--candidate-root", + display_path(candidate_root, repo), + "--require-candidate-ready", + ] + fixture_set_gate = [ + "python3", + "scripts/check-event-stream-official-fixture-set.py", + "--official-root", + display_path(official_root, repo), + "--candidate-root", + display_path(candidate_root, repo), + "--require-scenario", + args.scenario, + ] + return { + "pairingPreflight": pairing_preflight, + "pairingPreflightShell": shell_join(pairing_preflight), + "fixtureSetGate": fixture_set_gate, + "fixtureSetGateShell": shell_join(fixture_set_gate), + } + + +def ingest(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + repo = pathlib.Path(__file__).resolve().parents[1] + capture_evidence: dict[str, Any] | None = None + smoke_record: dict[str, Any] | None = None + capture_tmp: tempfile.TemporaryDirectory[str] | None = None + + if args.recording is not None: + recording_input = args.recording + mcp_transcript = args.mcp_transcript + elif args.smoke_json is not None: + records = read_json_records(args.smoke_json) + smoke_record = select_smoke_record(records, args.smoke_mode) + recording_input = path_from_smoke_record(smoke_record, args.smoke_json.parent) + mcp_transcript = args.mcp_transcript or optional_path_from_smoke_record( + smoke_record, + "mcpTranscriptPath", + args.smoke_json.parent, + ) + elif args.run_action_smoke: + capture_tmp = tempfile.TemporaryDirectory(prefix="ocu-rnr-", dir="/tmp") + records, capture_evidence = run_action_smoke(repo, pathlib.Path(capture_tmp.name), args.scenario) + smoke_record = select_smoke_record(records, args.smoke_mode) + recording_input = path_from_smoke_record(smoke_record) + mcp_transcript = args.mcp_transcript or optional_path_from_smoke_record( + smoke_record, + "mcpTranscriptPath", + ) + else: + raise ValueError("pass --recording, --smoke-json, or --run-action-smoke") + + validate_ok, validate_json = run_json( + [ + sys.executable, + str(repo / "scripts/validate-event-stream-recording.py"), + str(recording_input), + *validation_args_for_scenario(args.scenario), + ], + repo, + ) + if not validate_ok: + return 1, { + "ok": False, + "stage": "validate", + "recordingInput": str(recording_input), + "smokeRecord": smoke_record, + "capture": capture_evidence, + "validation": validate_json, + } + + import_command = [ + sys.executable, + str(repo / "scripts/import-event-stream-fixture.py"), + str(recording_input), + "--name", + args.name, + "--output-dir", + str(args.output_dir), + "--source", + "ocu", + "--scenario", + args.scenario, + "--captured-at", + args.captured_at, + ] + if mcp_transcript is not None: + import_command.extend(["--mcp-transcript", str(mcp_transcript)]) + if args.force: + import_command.append("--force") + + import_ok, import_json = run_json(import_command, repo) + if not import_ok: + return 1, { + "ok": False, + "stage": "import", + "recordingInput": str(recording_input), + "smokeRecord": smoke_record, + "capture": capture_evidence, + "validation": validate_json, + "importResult": import_json, + } + + fixture_dir = pathlib.Path(import_json["fixtureDir"]) + readiness_ok, readiness_json = run_json( + [ + sys.executable, + str(repo / "scripts/check-event-stream-golden-readiness.py"), + str(fixture_dir), + *scenario_readiness_args( + args.scenario, + source="ocu", + require_mcp_transcript_evidence=args.require_mcp_transcript_evidence, + ), + ], + repo, + ) + + fixture_set_json = None + fixture_set_ok = None + if args.check_fixture_set: + if args.official_root is None: + raise ValueError("--check-fixture-set requires --official-root") + fixture_set_ok, fixture_set_json = run_json( + [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-set.py"), + "--official-root", + str(args.official_root), + "--candidate-root", + str(args.output_dir), + "--require-scenario", + args.scenario, + ], + repo, + ) + + ok = readiness_ok and (fixture_set_ok is not False) + result: dict[str, Any] = { + "ok": ok, + "fixtureDir": str(fixture_dir), + "scenario": args.scenario, + "commands": handoff_commands(repo, args), + "recordingInput": str(recording_input), + "smokeRecord": smoke_record, + "capture": capture_evidence, + "validation": validate_json, + "importResult": import_json, + "readiness": readiness_json, + "requiredMcpTranscriptEvidence": args.require_mcp_transcript_evidence, + "usedMcpTranscript": str(mcp_transcript) if mcp_transcript is not None else None, + } + if fixture_set_json is not None: + result["fixtureSetGate"] = fixture_set_json + + if ok or args.allow_readiness_failure: + return 0, result + return 1, result + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Import an Open Computer Use Record & Replay recording as an OCU candidate fixture " + "and optionally compare it with official same-scenario fixtures." + ) + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--recording", + type=pathlib.Path, + help="OCU recording directory, metadata.json, or session.json to import directly.", + ) + source.add_argument( + "--smoke-json", + type=pathlib.Path, + help="JSONL/stdout captured from OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1.", + ) + source.add_argument( + "--run-action-smoke", + action="store_true", + help="Run the opt-in real input action smoke and import its resulting recording.", + ) + parser.add_argument("--name", required=True, help="Candidate fixture directory name to create.") + parser.add_argument( + "--scenario", + default=DEFAULT_SCENARIO, + help=( + "Scenario tag to write into fixture-manifest.json. Defaults to " + f"{DEFAULT_SCENARIO}. Required scenarios: {DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument( + "--smoke-mode", + default="actions", + help="Smoke JSON record mode to import when using --smoke-json or --run-action-smoke.", + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=DEFAULT_OUTPUT_ROOT, + help=f"Candidate fixture root directory. Defaults to {DEFAULT_OUTPUT_ROOT}.", + ) + parser.add_argument( + "--captured-at", + default=datetime.date.today().isoformat(), + help="Capture date to record in fixture-manifest.json.", + ) + parser.add_argument( + "--mcp-transcript", + type=pathlib.Path, + help=( + "Optional local probe output or MCP transcript to sanitize into mcp-transcript.json. " + "Provide this when comparing against official fixtures that require response-shape evidence." + ), + ) + parser.add_argument( + "--require-mcp-transcript-evidence", + action="store_true", + help="Require mcp-transcript.json and scenario-specific response-shape evidence during readiness.", + ) + parser.add_argument( + "--official-root", + type=pathlib.Path, + help="Official fixture root used with --check-fixture-set.", + ) + parser.add_argument( + "--check-fixture-set", + action="store_true", + help="After import, run the collection-level official-vs-OCU candidate gate for this scenario.", + ) + parser.add_argument( + "--allow-readiness-failure", + action="store_true", + help="Exit 0 after import even when readiness or fixture-set checks fail; the JSON still reports failures.", + ) + parser.add_argument("--force", action="store_true", help="Replace an existing fixture directory.") + args = parser.parse_args() + + try: + exit_code, result = ingest(args) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + + output = json.dumps(result, indent=2, sort_keys=True) + if exit_code == 0: + print(output) + else: + print(output, file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/ingest-official-record-and-replay-fixture.py b/scripts/ingest-official-record-and-replay-fixture.py new file mode 100755 index 00000000..bc3dd89d --- /dev/null +++ b/scripts/ingest-official-record-and-replay-fixture.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import datetime +import json +import pathlib +import subprocess +import sys +import tempfile +from typing import Any + +from record_and_replay_scenarios import DEFAULT_REQUIRED_SCENARIOS, scenario_readiness_args + + +DEFAULT_OUTPUT_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) + +DEFAULT_OFFICIAL_PLUGIN_VERSION = "record-and-replay 1.0.857" +DEFAULT_SCENARIO = DEFAULT_REQUIRED_SCENARIOS[0] +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) + +PATH_KEYS = ( + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + "sessionDirectoryPath", +) + +TRANSCRIPT_KEYS = { + "startResponseShape", + "repeatStartResponseShape", + "statusResponseShape", + "stopResponseShape", + "repeatStopResponseShape", + "finalStatusResponseShape", + "transcript", +} + + +def load_json(path: pathlib.Path) -> Any: + try: + return json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + + +def load_json_text(raw: str, label: str) -> Any: + try: + return json.loads(raw) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {label}:{error.lineno}:{error.colno}: {error.msg}") + + +def decode_json_string(value: str) -> Any | None: + try: + return json.loads(value) + except json.JSONDecodeError: + return None + + +def iter_json_objects(value: Any): + if isinstance(value, dict): + yield value + for child in value.values(): + if isinstance(child, str): + decoded = decode_json_string(child) + if decoded is not None: + yield from iter_json_objects(decoded) + else: + yield from iter_json_objects(child) + elif isinstance(value, list): + for child in value: + yield from iter_json_objects(child) + + +def extract_handoff_paths(value: Any) -> dict[str, str]: + paths: dict[str, str] = {} + for obj in iter_json_objects(value): + for key in PATH_KEYS: + raw = obj.get(key) + if isinstance(raw, str) and raw and key not in paths: + paths[key] = raw + return paths + + +def looks_like_mcp_transcript(value: Any) -> bool: + return any(any(key in obj for key in TRANSCRIPT_KEYS) for obj in iter_json_objects(value)) + + +def resolve_path(raw: str, base_dir: pathlib.Path | None) -> pathlib.Path: + path = pathlib.Path(raw) + if path.is_absolute() or base_dir is None: + return path + return base_dir / path + + +def recording_input_from_paths(paths: dict[str, str], base_dir: pathlib.Path | None) -> pathlib.Path: + for key in ("metadataPath", "sessionPath"): + raw = paths.get(key) + if raw: + return resolve_path(raw, base_dir) + events_path = paths.get("eventsPath") + if events_path: + return resolve_path(events_path, base_dir).parent + session_directory_path = paths.get("sessionDirectoryPath") + if session_directory_path: + return resolve_path(session_directory_path, base_dir) + raise ValueError( + "could not find metadataPath, sessionPath, eventsPath, or sessionDirectoryPath in official status JSON; " + "pass --recording instead" + ) + + +def describe_path(path: pathlib.Path) -> dict[str, Any]: + resolved = path + exists = resolved.exists() + return { + "path": str(path), + "exists": exists, + "isDirectory": exists and resolved.is_dir(), + "isFile": exists and resolved.is_file(), + "name": resolved.name, + } + + +def inspect_recording_input(recording_input: pathlib.Path) -> dict[str, Any]: + info = describe_path(recording_input) + if recording_input.is_dir(): + metadata_path = recording_input / "metadata.json" + session_path = recording_input / "session.json" + events_path = recording_input / "events.jsonl" + suppressed_path = recording_input / "suppressed.jsonl" + else: + session_dir = recording_input.parent + metadata_path = recording_input if recording_input.name == "metadata.json" else session_dir / "metadata.json" + session_path = recording_input if recording_input.name == "session.json" else session_dir / "session.json" + events_path = session_dir / "events.jsonl" + suppressed_path = session_dir / "suppressed.jsonl" + info.update( + { + "metadataPath": describe_path(metadata_path), + "sessionPath": describe_path(session_path), + "eventsPath": describe_path(events_path), + "suppressedEventsPath": describe_path(suppressed_path), + } + ) + return info + + +def inspect_mcp_transcript(path: pathlib.Path | None, status_json: Any | None) -> dict[str, Any]: + status_json_looks_like_transcript = ( + looks_like_mcp_transcript(status_json) if status_json is not None else False + ) + if path is None: + return { + "path": None, + "exists": False, + "looksLikeMcpTranscript": False, + "statusJsonLooksLikeMcpTranscript": status_json_looks_like_transcript, + } + info = describe_path(path) + parsed: Any | None = None + if path.exists() and path.is_file(): + parsed = load_json(path) + info["looksLikeMcpTranscript"] = looks_like_mcp_transcript(parsed) + info["statusJsonLooksLikeMcpTranscript"] = status_json_looks_like_transcript + return info + + +def run_json(command: list[str], cwd: pathlib.Path) -> tuple[bool, dict[str, Any]]: + completed = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def ingest(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + repo = pathlib.Path(__file__).resolve().parents[1] + if args.status_json_base_dir is not None and args.status_json is None: + raise ValueError("--status-json-base-dir requires --status-json") + + status_json: Any | None = None + status_json_base: pathlib.Path | None = args.status_json_base_dir + status_json_transcript_tmp: tempfile.TemporaryDirectory[str] | None = None + status_json_stdin_transcript: pathlib.Path | None = None + mcp_transcript_tmp: tempfile.TemporaryDirectory[str] | None = None + mcp_transcript_stdin_path: pathlib.Path | None = None + + def cleanup_tempdirs() -> None: + if mcp_transcript_tmp is not None: + mcp_transcript_tmp.cleanup() + if status_json_transcript_tmp is not None: + status_json_transcript_tmp.cleanup() + + if args.status_json is not None: + if str(args.status_json) == "-": + if args.mcp_transcript is not None and str(args.mcp_transcript) == "-": + raise ValueError( + "cannot read both --status-json and --mcp-transcript from stdin; " + "use --use-status-json-as-transcript when one JSON contains both" + ) + raw_status_json = sys.stdin.read() + status_json = load_json_text(raw_status_json, "") + if args.use_status_json_as_transcript: + status_json_transcript_tmp = tempfile.TemporaryDirectory() + status_json_stdin_transcript = ( + pathlib.Path(status_json_transcript_tmp.name) / "status-json-transcript.json" + ) + status_json_stdin_transcript.write_text(raw_status_json) + else: + status_json = load_json(args.status_json) + if status_json_base is None: + status_json_base = args.status_json.parent + + if args.recording is not None: + recording_input = args.recording + extracted_paths: dict[str, str] = {} + elif status_json is not None: + extracted_paths = extract_handoff_paths(status_json) + recording_input = recording_input_from_paths(extracted_paths, status_json_base) + else: + raise ValueError("pass --recording or --status-json") + + mcp_transcript = args.mcp_transcript + if mcp_transcript is not None and str(mcp_transcript) == "-": + raw_mcp_transcript = sys.stdin.read() + load_json_text(raw_mcp_transcript, "") + mcp_transcript_tmp = tempfile.TemporaryDirectory() + mcp_transcript_stdin_path = pathlib.Path(mcp_transcript_tmp.name) / "mcp-transcript.json" + mcp_transcript_stdin_path.write_text(raw_mcp_transcript) + mcp_transcript = mcp_transcript_stdin_path + + if mcp_transcript is None and args.use_status_json_as_transcript: + if status_json is None: + raise ValueError("--use-status-json-as-transcript requires --status-json") + if not looks_like_mcp_transcript(status_json): + raise ValueError("--status-json does not look like an MCP transcript") + mcp_transcript = status_json_stdin_transcript if status_json_stdin_transcript is not None else args.status_json + + used_mcp_transcript = ( + "" + if ( + (status_json_stdin_transcript is not None and mcp_transcript == status_json_stdin_transcript) + or (mcp_transcript_stdin_path is not None and mcp_transcript == mcp_transcript_stdin_path) + ) + else str(mcp_transcript) + if mcp_transcript is not None + else None + ) + + if args.inspect_only: + recording_inspection = inspect_recording_input(recording_input) + transcript_inspection = inspect_mcp_transcript(mcp_transcript, status_json) + ok = recording_inspection["exists"] and ( + not args.require_mcp_transcript_evidence + or transcript_inspection["looksLikeMcpTranscript"] + ) + cleanup_tempdirs() + return ( + 0 if ok else 1, + { + "ok": ok, + "stage": "inspect", + "wouldImport": False, + "scenario": args.scenario, + "recordingInput": str(recording_input), + "recordingInputInspection": recording_inspection, + "extractedPaths": extracted_paths, + "statusJsonBaseDir": str(status_json_base) if status_json_base is not None else None, + "requiredMcpTranscriptEvidence": args.require_mcp_transcript_evidence, + "usedMcpTranscript": used_mcp_transcript, + "mcpTranscriptInspection": transcript_inspection, + }, + ) + + import_command = [ + sys.executable, + str(repo / "scripts/import-event-stream-fixture.py"), + str(recording_input), + "--name", + args.name, + "--output-dir", + str(args.output_dir), + "--source", + "official", + "--scenario", + args.scenario, + "--official-plugin-version", + args.official_plugin_version, + "--captured-at", + args.captured_at, + ] + if mcp_transcript is not None: + import_command.extend(["--mcp-transcript", str(mcp_transcript)]) + if args.force: + import_command.append("--force") + + import_ok, import_json = run_json(import_command, repo) + if not import_ok: + cleanup_tempdirs() + return 1, { + "ok": False, + "stage": "import", + "recordingInput": str(recording_input), + "extractedPaths": extracted_paths, + "importResult": import_json, + } + + fixture_dir = pathlib.Path(import_json["fixtureDir"]) + readiness_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-golden-readiness.py"), + str(fixture_dir), + *scenario_readiness_args( + args.scenario, + source="official", + require_mcp_transcript_evidence=args.require_mcp_transcript_evidence, + ), + ] + readiness_ok, readiness_json = run_json(readiness_command, repo) + + fixture_set_json = None + fixture_set_ok = None + if args.check_fixture_set: + fixture_set_ok, fixture_set_json = run_json( + [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-set.py"), + "--official-root", + str(args.output_dir), + "--require-scenario", + args.scenario, + ], + repo, + ) + + coverage_json = None + coverage_ok = None + if args.check_coverage or args.require_coverage: + coverage_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-coverage.py"), + "--fixture-root", + str(args.output_dir), + ] + if args.require_coverage: + coverage_command.append("--require-readiness") + else: + coverage_command.extend(["--allow-missing", "--check-readiness"]) + coverage_ok, coverage_json = run_json(coverage_command, repo) + + ok = ( + readiness_ok + and (fixture_set_ok is not False) + and (coverage_ok is not False) + ) + result: dict[str, Any] = { + "ok": ok, + "fixtureDir": str(fixture_dir), + "scenario": args.scenario, + "recordingInput": str(recording_input), + "extractedPaths": extracted_paths, + "statusJsonBaseDir": str(status_json_base) if status_json_base is not None else None, + "importResult": import_json, + "readiness": readiness_json, + "requiredMcpTranscriptEvidence": args.require_mcp_transcript_evidence, + "usedMcpTranscript": used_mcp_transcript, + } + if fixture_set_json is not None: + result["fixtureSetGate"] = fixture_set_json + if coverage_json is not None: + result["coverage"] = coverage_json + + if ok or args.allow_readiness_failure: + cleanup_tempdirs() + return 0, result + cleanup_tempdirs() + return 1, result + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Import an official Codex Record & Replay recording from hosted tool paths, " + "sanitize it as a fixture, and run golden readiness checks." + ) + ) + source = parser.add_mutually_exclusive_group(required=True) + source.add_argument( + "--status-json", + type=pathlib.Path, + help=( + "JSON captured from event_stream_stop/status or a JSON-RPC response. " + "The script extracts metadataPath/sessionPath/eventsPath recursively. " + "Use '-' to read the JSON from stdin." + ), + ) + source.add_argument( + "--recording", + type=pathlib.Path, + help="Official recording directory, metadata.json, or session.json to import directly.", + ) + parser.add_argument( + "--status-json-base-dir", + type=pathlib.Path, + help=( + "Resolve relative metadataPath/sessionPath/eventsPath values from --status-json against this directory. " + "Defaults to the JSON file's parent; required for relative paths when --status-json is '-'." + ), + ) + parser.add_argument("--name", required=True, help="Fixture directory name to create.") + parser.add_argument( + "--scenario", + default=DEFAULT_SCENARIO, + help=( + "Scenario tag to write into fixture-manifest.json. Defaults to " + f"{DEFAULT_SCENARIO}. Required scenarios: {DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + default=DEFAULT_OUTPUT_ROOT, + help=f"Fixture root directory. Defaults to {DEFAULT_OUTPUT_ROOT}.", + ) + parser.add_argument( + "--official-plugin-version", + default=DEFAULT_OFFICIAL_PLUGIN_VERSION, + help=f"Official plugin version string. Defaults to {DEFAULT_OFFICIAL_PLUGIN_VERSION!r}.", + ) + parser.add_argument( + "--captured-at", + default=datetime.date.today().isoformat(), + help="Capture date to record in fixture-manifest.json.", + ) + parser.add_argument( + "--mcp-transcript", + type=pathlib.Path, + help=( + "Optional raw probe output or MCP transcript to sanitize into mcp-transcript.json. " + "Use '-' to read the transcript JSON from stdin when --status-json is a file." + ), + ) + parser.add_argument( + "--use-status-json-as-transcript", + action="store_true", + help="Treat --status-json as the MCP transcript too, when it contains transcript/response-shape keys.", + ) + parser.add_argument( + "--require-mcp-transcript-evidence", + action="store_true", + help=( + "Require mcp-transcript.json and scenario-specific MCP response-shape evidence during readiness. " + "Use this for official golden fixtures intended to pass the collection-level gate." + ), + ) + parser.add_argument( + "--check-fixture-set", + action="store_true", + help="After import, run the collection-level official fixture set gate for this scenario.", + ) + parser.add_argument( + "--check-coverage", + action="store_true", + help=( + "After import, report official successful recording scenario coverage for the output root. " + "Missing required scenarios are reported but do not fail the command." + ), + ) + parser.add_argument( + "--require-coverage", + action="store_true", + help=( + "After import, require the output root to cover required official successful recording scenarios. " + "This implies --check-coverage and returns non-zero when coverage is incomplete." + ), + ) + parser.add_argument( + "--allow-readiness-failure", + action="store_true", + help="Exit 0 after import even when readiness or fixture-set checks fail; the JSON still reports failures.", + ) + parser.add_argument( + "--inspect-only", + action="store_true", + help=( + "Parse inputs and report recording/transcript readiness without creating a fixture. " + "Use this immediately after copying hosted Codex Record & Replay JSON." + ), + ) + parser.add_argument("--force", action="store_true", help="Replace an existing fixture directory.") + args = parser.parse_args() + + try: + exit_code, result = ingest(args) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + + output = json.dumps(result, indent=2, sort_keys=True) + if exit_code == 0: + print(output) + else: + print(output, file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/install-codex-record-and-replay-mcp.sh b/scripts/install-codex-record-and-replay-mcp.sh new file mode 100755 index 00000000..faee8ea7 --- /dev/null +++ b/scripts/install-codex-record-and-replay-mcp.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +config_helper="${script_dir}/install-config-helper.mjs" +codex_home="${CODEX_HOME:-${HOME}/.codex}" +config_path="${codex_home}/config.toml" +server_name="record_and_replay" +command_name="open-computer-use" +command_args='["event-stream","mcp"]' + +usage() { + cat <<'EOF' +Usage: ./scripts/install-codex-record-and-replay-mcp.sh + +Install the Open Computer Use Record & Replay stdio MCP entry into ~/.codex/config.toml. +The script is idempotent: if the same MCP server entry already exists, it leaves the file unchanged. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 1 + ;; + esac +done + +mkdir -p "${codex_home}" + +node "${config_helper}" codex-mcp "${config_path}" "${server_name}" "${command_name}" "${command_args}" diff --git a/scripts/install-config-helper.mjs b/scripts/install-config-helper.mjs index b5b8d95d..df49e946 100644 --- a/scripts/install-config-helper.mjs +++ b/scripts/install-config-helper.mjs @@ -11,7 +11,7 @@ function fail(message) { function usage() { process.stdout.write(`Usage: node ./scripts/install-config-helper.mjs claude-mcp - node ./scripts/install-config-helper.mjs codex-mcp + node ./scripts/install-config-helper.mjs codex-mcp [] node ./scripts/install-config-helper.mjs gemini-mcp node ./scripts/install-config-helper.mjs opencode-mcp node ./scripts/install-config-helper.mjs codex-plugin-version @@ -374,8 +374,24 @@ function installOpencodeMcp(primaryConfigPath, secondaryConfigPath, serverName, process.stdout.write(`Installed opencode MCP server "${serverName}" into ${primaryConfigPath}.\n`); } -function installCodexMcp(configPath, serverName, commandName) { - const desiredBody = `command = ${JSON.stringify(commandName)}\nargs = ["mcp"]`; +function parseCommandArgs(argsJSON, label) { + let commandArgs; + try { + commandArgs = JSON.parse(argsJSON); + } catch (error) { + fail(`${label} is not valid JSON: ${error.message}`); + } + + if (!Array.isArray(commandArgs) || !commandArgs.every((entry) => typeof entry === "string")) { + fail(`${label} must be a JSON array of strings.`); + } + + return commandArgs; +} + +function installCodexMcp(configPath, serverName, commandName, argsJSON = '["mcp"]') { + const commandArgs = parseCommandArgs(argsJSON, "Codex MCP args"); + const desiredBody = `command = ${JSON.stringify(commandName)}\nargs = ${JSON.stringify(commandArgs)}`; const targetHeader = `mcp_servers."${serverName}"`; const legacyServerName = "open-codex-computer-use"; const legacyHeader = `mcp_servers."${legacyServerName}"`; @@ -486,7 +502,7 @@ function main(argv) { installClaudeMcp(...args); return; case "codex-mcp": - if (args.length !== 3) { + if (args.length !== 3 && args.length !== 4) { usage(); process.exit(1); } diff --git a/scripts/npm/build-packages.mjs b/scripts/npm/build-packages.mjs index 64ff43b1..91cb779e 100644 --- a/scripts/npm/build-packages.mjs +++ b/scripts/npm/build-packages.mjs @@ -21,6 +21,7 @@ const repoRoot = path.resolve(__dirname, "..", ".."); const defaultOutDir = path.join(repoRoot, "dist", "npm"); const appBundleName = "Open Computer Use.app"; const appExecutableName = "OpenComputerUse"; +const recordAndReplayBaselineSummaryPath = path.join(repoRoot, "dist", "record-and-replay-baseline-summary.json"); const metaPackageNames = [ "open-computer-use", "open-computer-use-mcp", @@ -213,7 +214,7 @@ function platformLaunchTable() { function renderLauncher() { return `#!/usr/bin/env node -const { spawn } = require("node:child_process"); +const { spawn, spawnSync } = require("node:child_process"); const fs = require("node:fs"); const path = require("node:path"); @@ -226,9 +227,13 @@ const installCommands = new Map([ ["install-clauce-mcp", "install-claude-mcp.sh"], ["install-gemini-mcp", "install-gemini-mcp.sh"], ["install-codex-mcp", "install-codex-mcp.sh"], + ["install-codex-record-and-replay-mcp", "install-codex-record-and-replay-mcp.sh"], ["install-opencode-mcp", "install-opencode-mcp.sh"], ["install-codex-plugin", "install-codex-plugin.sh"], ]); +const bundledScriptCommands = new Map([ + ["scaffold-record-and-replay-skill-repo", "scaffold-record-and-replay-skill-repo.py"], +]); function printLauncherHelp() { console.log(\`Open Computer Use @@ -248,6 +253,10 @@ Commands: install-claude-mcp Install the MCP server into ~/.claude.json for this project. install-gemini-mcp Install the MCP server into Gemini CLI config. install-codex-mcp Install the MCP server into ~/.codex/config.toml. + install-codex-record-and-replay-mcp + Install the Record & Replay MCP server into ~/.codex/config.toml. + scaffold-record-and-replay-skill-repo + Generate a standalone Record & Replay thin skill repo. install-opencode-mcp Install the MCP server into ~/.config/opencode. install-codex-plugin Install this npm package into the local Codex plugin cache. help [command] Show general or command-specific help. @@ -262,29 +271,49 @@ Notes: Use 'open-computer-use help ' for command-specific help.\`); } -function printInstallHelp(scriptName, usage) { +function printInstallHelp(scriptName, usage, targetCommand = "open-computer-use mcp") { console.log(\`Usage: \${usage} This helper updates a local MCP or plugin config to run: - open-computer-use mcp + \${targetCommand} Script: \${scriptName}\`); } +function printScaffoldRecordAndReplaySkillRepoHelp() { + console.log(\`Usage: + open-computer-use scaffold-record-and-replay-skill-repo --output-dir + +Generates a standalone Record & Replay thin skill repo. The generated repo does not copy the Open Computer Use runtime; it expects the target environment to have the open-computer-use CLI installed. + +Runtime contract: + open-computer-use event-stream mcp + +Requirements: + Python 3 must be available as python3, python, or through PYTHON=/path/to/python. + +Generated repo checks: + ./scripts/check.sh + ./scripts/recording-lifecycle-smoke.py # optional; starts a real local recording\`); +} + function fail(message) { console.error(message); process.exit(1); } -function spawnAndExit(executable, executableArgs) { +function spawnAndExit(executable, executableArgs, options = {}) { const child = spawn(executable, executableArgs, { stdio: "inherit", windowsHide: false, }); child.on("error", (error) => { + if (options.onErrorMessage) { + fail(options.onErrorMessage(error, executable)); + } fail(\`Failed to start \${executable}: \${error.message}\`); }); @@ -315,6 +344,50 @@ function runInstallCommand(scriptName, scriptArgs) { spawnAndExit(scriptPath, scriptArgs); } +function resolvePythonExecutableForBundledScript() { + const candidates = process.env.PYTHON + ? [process.env.PYTHON] + : (process.platform === "win32" ? ["python"] : ["python3", "python"]); + + for (const candidate of candidates) { + const result = spawnSync(candidate, ["--version"], { + encoding: "utf8", + windowsHide: true, + }); + if (result.error || result.status !== 0) { + continue; + } + + const versionText = [result.stdout, result.stderr].filter(Boolean).join(" ").trim(); + const match = /\\bPython\\s+(\\d+)\\.(\\d+)/.exec(versionText); + if (match && Number(match[1]) >= 3) { + return candidate; + } + } + + fail(\`Python 3 is required to run "open-computer-use \${command}". + +No Python 3 executable was found. Install Python 3 or set PYTHON=/path/to/python3, then retry: + open-computer-use \${command} --output-dir \`); +} + +function runBundledScriptCommand(scriptName, scriptArgs) { + const scriptPath = path.join(packageRoot, "scripts", scriptName); + if (!fs.existsSync(scriptPath)) { + fail(\`Missing bundled helper at \${scriptPath}.\`); + } + + const python = resolvePythonExecutableForBundledScript(); + spawnAndExit(python, [scriptPath, ...scriptArgs], { + onErrorMessage: (error, executable) => \`Failed to start Python for "open-computer-use \${command}": \${error.message} + +Tried executable: + \${executable} + +Set PYTHON=/path/to/python3 or install Python 3, then retry.\`, + }); +} + function resolveNativeExecutable() { const platformKey = \`\${process.platform}-\${process.arch}\`; const target = platformPackages[platformKey]; @@ -349,6 +422,20 @@ if (command === "help" && args[1] === "install-codex-mcp") { process.exit(0); } +if (command === "help" && args[1] === "install-codex-record-and-replay-mcp") { + printInstallHelp( + "install-codex-record-and-replay-mcp.sh", + "open-computer-use install-codex-record-and-replay-mcp", + "open-computer-use event-stream mcp", + ); + process.exit(0); +} + +if (command === "help" && args[1] === "scaffold-record-and-replay-skill-repo") { + printScaffoldRecordAndReplaySkillRepoHelp(); + process.exit(0); +} + if (command === "help" && args[1] === "install-gemini-mcp") { printInstallHelp("install-gemini-mcp.sh", "open-computer-use install-gemini-mcp [--scope project|user]"); process.exit(0); @@ -367,6 +454,9 @@ if (command === "help" && (args[1] === "install-claude-mcp" || args[1] === "inst if (installCommands.has(command)) { const scriptName = installCommands.get(command); runInstallCommand(scriptName, args.slice(1)); +} else if (bundledScriptCommands.has(command)) { + const scriptName = bundledScriptCommands.get(command); + runBundledScriptCommand(scriptName, args.slice(1)); } else { spawnAndExit(resolveNativeExecutable(), args); } @@ -464,6 +554,8 @@ open-computer-use install-claude-mcp open-computer-use install-gemini-mcp open-computer-use install-gemini-mcp --scope user open-computer-use install-codex-mcp +open-computer-use install-codex-record-and-replay-mcp +open-computer-use scaffold-record-and-replay-skill-repo --output-dir ./open-computer-use-rnr-skill-repo open-computer-use install-opencode-mcp open-computer-use install-codex-plugin \`\`\` @@ -531,10 +623,15 @@ function renderMetaPackageJson(packageName, version) { "plugins/open-computer-use/.mcp.json", "plugins/open-computer-use/assets/", "plugins/open-computer-use/scripts/", + "skills/open-computer-use-record-and-replay/", "scripts/install-claude-mcp.sh", "scripts/install-gemini-mcp.sh", "scripts/install-config-helper.mjs", "scripts/install-codex-mcp.sh", + "scripts/install-codex-record-and-replay-mcp.sh", + "scripts/scaffold-record-and-replay-skill-repo.py", + "scripts/record_and_replay_scenarios.py", + "dist/record-and-replay-baseline-summary.json", "scripts/install-opencode-mcp.sh", "scripts/install-codex-plugin.sh", "scripts/postinstall.mjs", @@ -544,11 +641,33 @@ function renderMetaPackageJson(packageName, version) { }; } +function copyRecordAndReplayBaselineSummary(packageRoot) { + if (!existsSync(recordAndReplayBaselineSummaryPath)) { + throw new Error( + [ + `Missing Record & Replay baseline summary artifact: ${recordAndReplayBaselineSummaryPath}.`, + "Run `make record-and-replay-baseline-audit` before building the npm package.", + "The standalone Record & Replay skill repo scaffold copies a redacted projection of this artifact into evidence/source-baseline-summary.json.", + ].join(" "), + ); + } + + mkdirSync(path.join(packageRoot, "dist"), { recursive: true }); + cpSync( + recordAndReplayBaselineSummaryPath, + path.join(packageRoot, "dist", "record-and-replay-baseline-summary.json"), + ); +} + function copyInstallerScripts(packageRoot) { cpSync(path.join(repoRoot, "scripts", "install-claude-mcp.sh"), path.join(packageRoot, "scripts", "install-claude-mcp.sh")); cpSync(path.join(repoRoot, "scripts", "install-gemini-mcp.sh"), path.join(packageRoot, "scripts", "install-gemini-mcp.sh")); cpSync(path.join(repoRoot, "scripts", "install-config-helper.mjs"), path.join(packageRoot, "scripts", "install-config-helper.mjs")); cpSync(path.join(repoRoot, "scripts", "install-codex-mcp.sh"), path.join(packageRoot, "scripts", "install-codex-mcp.sh")); + cpSync(path.join(repoRoot, "scripts", "install-codex-record-and-replay-mcp.sh"), path.join(packageRoot, "scripts", "install-codex-record-and-replay-mcp.sh")); + cpSync(path.join(repoRoot, "scripts", "scaffold-record-and-replay-skill-repo.py"), path.join(packageRoot, "scripts", "scaffold-record-and-replay-skill-repo.py")); + cpSync(path.join(repoRoot, "scripts", "record_and_replay_scenarios.py"), path.join(packageRoot, "scripts", "record_and_replay_scenarios.py")); + copyRecordAndReplayBaselineSummary(packageRoot); cpSync(path.join(repoRoot, "scripts", "install-opencode-mcp.sh"), path.join(packageRoot, "scripts", "install-opencode-mcp.sh")); cpSync(path.join(repoRoot, "scripts", "install-codex-plugin.sh"), path.join(packageRoot, "scripts", "install-codex-plugin.sh")); @@ -556,6 +675,8 @@ function copyInstallerScripts(packageRoot) { "install-claude-mcp.sh", "install-gemini-mcp.sh", "install-codex-mcp.sh", + "install-codex-record-and-replay-mcp.sh", + "scaffold-record-and-replay-skill-repo.py", "install-opencode-mcp.sh", "install-codex-plugin.sh", ]) { @@ -603,11 +724,17 @@ function stageMetaPackage(packageName, version, outDir) { mkdirSync(path.join(packageRoot, "dist"), { recursive: true }); mkdirSync(path.join(packageRoot, "plugins"), { recursive: true }); mkdirSync(path.join(packageRoot, "scripts"), { recursive: true }); + mkdirSync(path.join(packageRoot, "skills"), { recursive: true }); cpSync(path.join(repoRoot, ".agents", "plugins", "marketplace.json"), path.join(packageRoot, ".agents", "plugins", "marketplace.json")); cpSync(path.join(repoRoot, "plugins", "open-computer-use"), path.join(packageRoot, "plugins", "open-computer-use"), { recursive: true, }); + cpSync( + path.join(repoRoot, "skills", "open-computer-use-record-and-replay"), + path.join(packageRoot, "skills", "open-computer-use-record-and-replay"), + { recursive: true }, + ); cpSync(path.join(repoRoot, "LICENSE"), path.join(packageRoot, "LICENSE")); copyBundledRuntimes(packageRoot, packageName); copyInstallerScripts(packageRoot); diff --git a/scripts/package-skill.sh b/scripts/package-skill.sh index ec35b19a..1e24b5eb 100755 --- a/scripts/package-skill.sh +++ b/scripts/package-skill.sh @@ -3,11 +3,8 @@ set -euo pipefail repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -skill_name="open-computer-use" -skill_dir="${repo_root}/skills/${skill_name}" +skills_root="${repo_root}/skills" dist_dir="${repo_root}/dist/skills" -zip_path="${dist_dir}/${skill_name}-skill.zip" -skill_path="${dist_dir}/${skill_name}.skill" manifest_path="${dist_dir}/package-manifest.json" if ! command -v node >/dev/null 2>&1; then @@ -25,23 +22,39 @@ if ! command -v unzip >/dev/null 2>&1; then exit 1 fi -if [[ ! -f "${skill_dir}/SKILL.md" ]]; then - echo "missing skill entrypoint: ${skill_dir}/SKILL.md" >&2 +skill_dirs=() +while IFS= read -r skill_entrypoint; do + skill_dirs+=("$(dirname "${skill_entrypoint}")") +done < <(find "${skills_root}" -mindepth 2 -maxdepth 2 -name SKILL.md -print | sort) + +if [[ "${#skill_dirs[@]}" -eq 0 ]]; then + echo "missing skill entrypoints under ${skills_root}" >&2 exit 1 fi -node - "${skill_dir}/SKILL.md" <<'NODE' +rm -rf "${dist_dir}" +mkdir -p "${dist_dir}" + +manifest_entries=() + +for skill_dir in "${skill_dirs[@]}"; do + skill_name="$(basename "${skill_dir}")" + zip_path="${dist_dir}/${skill_name}-skill.zip" + skill_path="${dist_dir}/${skill_name}.skill" + + node - "${skill_dir}/SKILL.md" "${skill_name}" <<'NODE' const fs = require("fs"); const skillPath = process.argv[2]; +const expectedName = process.argv[3]; const content = fs.readFileSync(skillPath, "utf8"); const errors = []; if (!content.startsWith("---\n")) { errors.push("SKILL.md must start with YAML frontmatter"); } -if (!/^name:\s*open-computer-use\s*$/m.test(content)) { - errors.push("SKILL.md frontmatter must include name: open-computer-use"); +if (!new RegExp(`^name:\\s*${expectedName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\s*$`, "m").test(content)) { + errors.push(`SKILL.md frontmatter must include name: ${expectedName}`); } if (!/^description:\s*\S/m.test(content)) { errors.push("SKILL.md frontmatter must include a non-empty description"); @@ -55,73 +68,92 @@ if (errors.length > 0) { } NODE -rm -rf "${dist_dir}" -mkdir -p "${dist_dir}" + ( + cd "${skills_root}" + zip -q -r "${zip_path}" "${skill_name}" + ) + cp "${zip_path}" "${skill_path}" + + if ! cmp -s "${zip_path}" "${skill_path}"; then + echo "${skill_name}-skill.zip and ${skill_name}.skill differ" >&2 + exit 1 + fi -( - cd "${repo_root}/skills" - zip -q -r "${zip_path}" "${skill_name}" -) -cp "${zip_path}" "${skill_path}" + has_skill_entrypoint=0 + entry_count=0 + while IFS= read -r entry; do + entry_count=$((entry_count + 1)) + if [[ "${entry}" != "${skill_name}/"* ]]; then + echo "skill zip entry must be under ${skill_name}/: ${entry}" >&2 + exit 1 + fi + if [[ "${entry}" == "${skill_name}/SKILL.md" ]]; then + has_skill_entrypoint=1 + fi + done < <(unzip -Z1 "${zip_path}") + + if [[ "${entry_count}" -eq 0 ]]; then + echo "${skill_name} skill zip is empty" >&2 + exit 1 + fi -node - "${zip_path}" "${skill_path}" "${manifest_path}" <<'NODE' + if [[ "${has_skill_entrypoint}" -ne 1 ]]; then + echo "skill zip is missing ${skill_name}/SKILL.md" >&2 + exit 1 + fi + + manifest_entries+=("${skill_name}|${zip_path}|${skill_path}") +done + +node - "${repo_root}" "${manifest_path}" "${manifest_entries[@]}" <<'NODE' const crypto = require("crypto"); const fs = require("fs"); +const path = require("path"); + +const repoRoot = process.argv[2]; +const manifestPath = process.argv[3]; +const entries = process.argv.slice(4); -const zipPath = process.argv[2]; -const skillPath = process.argv[3]; -const manifestPath = process.argv[4]; -const zip = fs.readFileSync(zipPath); -const skill = fs.readFileSync(skillPath); +const skills = entries.map((entry) => { + const [name, zipPath, skillPath] = entry.split("|"); + const zip = fs.readFileSync(zipPath); + const skill = fs.readFileSync(skillPath); + const zipSha = crypto.createHash("sha256").update(zip).digest("hex"); + const skillSha = crypto.createHash("sha256").update(skill).digest("hex"); + if (zipSha !== skillSha) { + throw new Error(`${name}-skill.zip and ${name}.skill must contain identical bytes`); + } + + return { + name, + rootDirectory: name, + artifacts: { + zip: path.relative(repoRoot, zipPath), + skill: path.relative(repoRoot, skillPath) + }, + sha256: { + zip: zipSha, + skill: skillSha + } + }; +}); + +const primarySkill = skills.find((skill) => skill.name === "open-computer-use") ?? skills[0]; const payload = { - name: "open-computer-use", - rootDirectory: "open-computer-use", - artifacts: { - zip: zipPath, - skill: skillPath - }, - sha256: { - zip: crypto.createHash("sha256").update(zip).digest("hex"), - skill: crypto.createHash("sha256").update(skill).digest("hex") - }, - generatedAtUtc: new Date().toISOString() + name: primarySkill.name, + rootDirectory: primarySkill.rootDirectory, + artifacts: primarySkill.artifacts, + sha256: primarySkill.sha256, + generatedAtUtc: new Date().toISOString(), + skills }; -if (payload.sha256.zip !== payload.sha256.skill) { - throw new Error("open-computer-use-skill.zip and open-computer-use.skill must contain identical bytes"); -} - fs.writeFileSync(manifestPath, `${JSON.stringify(payload, null, 2)}\n`); NODE -if ! cmp -s "${zip_path}" "${skill_path}"; then - echo "open-computer-use-skill.zip and open-computer-use.skill differ" >&2 - exit 1 -fi - -has_skill_entrypoint=0 -entry_count=0 -while IFS= read -r entry; do - entry_count=$((entry_count + 1)) - if [[ "${entry}" != "${skill_name}/"* ]]; then - echo "skill zip entry must be under ${skill_name}/: ${entry}" >&2 - exit 1 - fi - if [[ "${entry}" == "${skill_name}/SKILL.md" ]]; then - has_skill_entrypoint=1 - fi -done < <(unzip -Z1 "${zip_path}") - -if [[ "${entry_count}" -eq 0 ]]; then - echo "skill zip is empty" >&2 - exit 1 -fi - -if [[ "${has_skill_entrypoint}" -ne 1 ]]; then - echo "skill zip is missing ${skill_name}/SKILL.md" >&2 - exit 1 -fi - -echo "${zip_path}" -echo "${skill_path}" +for entry in "${manifest_entries[@]}"; do + IFS="|" read -r _ zip_path skill_path <<<"${entry}" + echo "${zip_path}" + echo "${skill_path}" +done diff --git a/scripts/prepare-record-and-replay-ocu-candidate-pairing.py b/scripts/prepare-record-and-replay-ocu-candidate-pairing.py new file mode 100755 index 00000000..1cce4e52 --- /dev/null +++ b/scripts/prepare-record-and-replay-ocu-candidate-pairing.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import subprocess +import sys +from typing import Any + +from record_and_replay_scenarios import ( + ACTION_SMOKE_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_recipe, +) + + +DEFAULT_OFFICIAL_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) +DEFAULT_CANDIDATE_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings/ocu-candidates" +) +DEFAULT_SCENARIO = DEFAULT_REQUIRED_SCENARIOS[0] +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) + + +def display_path(path: pathlib.Path, repo: pathlib.Path) -> str: + try: + return str(path.resolve().relative_to(repo.resolve())) + except (FileNotFoundError, ValueError): + try: + return "~/" + str(path.expanduser().resolve().relative_to(pathlib.Path.home())) + except (FileNotFoundError, ValueError): + return str(path) + + +def run_json(command: list[str], cwd: pathlib.Path) -> tuple[bool, dict[str, Any]]: + completed = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def shell_join(argv: list[str]) -> str: + import shlex + + return " ".join(shlex.quote(part) for part in argv) + + +def candidate_name(scenario: str) -> str: + return f"ocu-{scenario}" + + +def candidate_source_command(repo: pathlib.Path, args: argparse.Namespace) -> tuple[list[str], str]: + base = [ + "python3", + "scripts/ingest-ocu-record-and-replay-candidate.py", + ] + if args.recording_placeholder: + source = ["--recording", args.recording_placeholder] + source_kind = "recording" + elif args.smoke_json_placeholder: + source = ["--smoke-json", args.smoke_json_placeholder] + source_kind = "smoke-json" + elif args.scenario in ACTION_SMOKE_SCENARIOS: + source = ["--run-action-smoke"] + source_kind = "run-action-smoke" + else: + source = ["--recording", ""] + source_kind = "recording-required" + return [*base, *source], source_kind + + +def build_commands(repo: pathlib.Path, args: argparse.Namespace) -> dict[str, Any]: + candidate_root = display_path(args.candidate_root, repo) + official_root = display_path(args.official_root, repo) + ingest_prefix, source_kind = candidate_source_command(repo, args) + ingest = [ + *ingest_prefix, + "--name", + args.name or candidate_name(args.scenario), + "--scenario", + args.scenario, + "--output-dir", + candidate_root, + "--official-root", + official_root, + "--check-fixture-set", + "--require-mcp-transcript-evidence", + ] + if args.mcp_transcript_placeholder: + ingest.extend(["--mcp-transcript", args.mcp_transcript_placeholder]) + fixture_set = [ + "python3", + "scripts/check-event-stream-official-fixture-set.py", + "--official-root", + official_root, + "--candidate-root", + candidate_root, + "--require-scenario", + args.scenario, + ] + official_gate = [ + "python3", + "scripts/check-event-stream-official-fixture-coverage.py", + "--fixture-root", + official_root, + "--require-readiness", + ] + return { + "sourceKind": source_kind, + "officialFixtureGate": official_gate, + "officialFixtureGateShell": shell_join(official_gate), + "ingestCandidate": ingest, + "ingestCandidateShell": shell_join(ingest), + "fixtureSetGate": fixture_set, + "fixtureSetGateShell": shell_join(fixture_set), + } + + +def scenario_status(fixture_set: dict[str, Any], scenario: str) -> dict[str, Any]: + scenario_results = fixture_set.get("scenarioResults") + result = None + if isinstance(scenario_results, list): + for item in scenario_results: + if isinstance(item, dict) and item.get("scenario") == scenario: + result = item + break + official_fixture = result.get("officialFixture") if isinstance(result, dict) else None + official_readiness = result.get("officialReadiness") if isinstance(result, dict) else None + candidate_fixture = result.get("candidateFixture") if isinstance(result, dict) else None + candidate_readiness = result.get("candidateReadiness") if isinstance(result, dict) else None + candidate_compare = result.get("candidateCompare") if isinstance(result, dict) else None + return { + "scenario": scenario, + "officialFixtureReady": bool( + official_fixture + and isinstance(official_readiness, dict) + and official_readiness.get("ok") is True + ), + "candidateFixturePresent": bool(candidate_fixture), + "candidateFixtureReady": bool( + candidate_fixture + and isinstance(candidate_readiness, dict) + and candidate_readiness.get("ok") is True + ), + "candidateComparePassed": bool( + candidate_compare + and isinstance(candidate_compare, dict) + and candidate_compare.get("ok") is True + ), + "officialFixture": official_fixture, + "candidateFixture": candidate_fixture, + } + + +def preflight(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + repo = pathlib.Path(__file__).resolve().parents[1] + commands = build_commands(repo, args) + coverage_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-coverage.py"), + "--fixture-root", + str(args.official_root), + "--allow-missing", + "--check-readiness", + "--require-scenario", + args.scenario, + ] + coverage_ok, coverage = run_json(coverage_command, repo) + coverage_errors = coverage.get("errors") + if not isinstance(coverage_errors, list): + coverage_errors = [] + fixture_set_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-set.py"), + "--official-root", + str(args.official_root), + "--require-scenario", + args.scenario, + ] + if args.candidate_root.exists(): + fixture_set_command.extend(["--candidate-root", str(args.candidate_root)]) + fixture_set_ok, fixture_set = run_json(fixture_set_command, repo) + status = scenario_status(fixture_set, args.scenario) + next_actions: list[str] = [] + if not status["officialFixtureReady"]: + next_actions.append( + f"Import a ready official fixture for scenario {args.scenario!r} before pairing an OCU candidate." + ) + elif not status["candidateFixturePresent"]: + next_actions.append("Run commands.ingestCandidate to create the same-scenario OCU candidate fixture.") + elif not status["candidateFixtureReady"]: + next_actions.append("Repair or re-import the OCU candidate; it does not pass scenario readiness.") + elif not status["candidateComparePassed"]: + next_actions.append("Use commands.fixtureSetGate output to inspect official-vs-OCU drift.") + else: + next_actions.append("Official fixture and OCU candidate are paired and pass the fixture set gate.") + + if commands["sourceKind"] == "recording-required": + next_actions.append( + "This scenario cannot use synthetic --run-action-smoke; provide --recording-placeholder or --smoke-json-placeholder." + ) + if coverage_errors: + next_actions.insert( + 0, + "Fix official fixture coverage errors before pairing an OCU candidate: " + + " | ".join(str(error) for error in coverage_errors), + ) + + ok = coverage_ok and fixture_set_ok and status["officialFixtureReady"] and ( + not args.require_candidate_ready + or (status["candidateFixtureReady"] and status["candidateComparePassed"]) + ) + result: dict[str, Any] = { + "ok": ok, + "stage": "candidate-pairing-preflight", + "scenario": args.scenario, + "officialRoot": display_path(args.official_root, repo), + "candidateRoot": display_path(args.candidate_root, repo), + "scenarioRecipe": scenario_recipe(args.scenario), + "scenarioStatus": status, + "officialCoverage": coverage, + "officialCoverageErrors": coverage_errors, + "fixtureSetGate": fixture_set, + "commands": commands, + "nextActions": next_actions, + "requireCandidateReady": args.require_candidate_ready, + } + exit_code = 1 if args.require_candidate_ready and not ok else 0 + return exit_code, result + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Prepare same-scenario OCU candidate capture/import commands after an official " + "Record & Replay golden fixture is available. This command is read-only." + ) + ) + parser.add_argument( + "--scenario", + default=DEFAULT_SCENARIO, + help=( + f"Scenario to pair. Defaults to {DEFAULT_SCENARIO}. " + f"Required scenarios: {DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument("--name", help="Candidate fixture name for generated ingest command.") + parser.add_argument( + "--official-root", + type=pathlib.Path, + default=DEFAULT_OFFICIAL_ROOT, + help=f"Official fixture root. Defaults to {DEFAULT_OFFICIAL_ROOT}.", + ) + parser.add_argument( + "--candidate-root", + type=pathlib.Path, + default=DEFAULT_CANDIDATE_ROOT, + help=f"OCU candidate fixture root. Defaults to {DEFAULT_CANDIDATE_ROOT}.", + ) + parser.add_argument( + "--recording-placeholder", + default=None, + help="Use this placeholder/path as --recording in generated candidate ingest command.", + ) + parser.add_argument( + "--smoke-json-placeholder", + default=None, + help="Use this placeholder/path as --smoke-json in generated candidate ingest command.", + ) + parser.add_argument( + "--mcp-transcript-placeholder", + default=None, + help="Optional placeholder/path for --mcp-transcript in generated candidate ingest command.", + ) + parser.add_argument( + "--require-candidate-ready", + action="store_true", + help="Fail unless a same-scenario OCU candidate exists, passes readiness, and compares successfully.", + ) + args = parser.parse_args() + exit_code, result = preflight(args) + output = json.dumps(result, indent=2, sort_keys=True) + if exit_code == 0: + print(output) + else: + print(output, file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/prepare-record-and-replay-official-golden-capture.py b/scripts/prepare-record-and-replay-official-golden-capture.py new file mode 100755 index 00000000..126216c4 --- /dev/null +++ b/scripts/prepare-record-and-replay-official-golden-capture.py @@ -0,0 +1,1571 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import pathlib +import subprocess +import sys +import textwrap +from typing import Any + +from record_and_replay_scenarios import ( + DEFAULT_RECOMMENDED_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_recipe, +) + + +DEFAULT_FIXTURE_ROOT = pathlib.Path( + "docs/references/codex-computer-use-reverse-engineering/fixtures/recordings" +) +DEFAULT_OFFICIAL_PLUGIN_ROOT = pathlib.Path( + "~/.codex/plugins/cache/openai-bundled/record-and-replay" +).expanduser() +DEFAULT_OFFICIAL_PLUGIN_VERSION = "record-and-replay 1.0.857" +DEFAULT_SCENARIO = DEFAULT_REQUIRED_SCENARIOS[0] +DEFAULT_REQUIRED_SCENARIO_HELP = ", ".join(DEFAULT_REQUIRED_SCENARIOS) + + +def display_path(path: pathlib.Path, repo: pathlib.Path) -> str: + try: + return str(path.resolve().relative_to(repo.resolve())) + except (FileNotFoundError, ValueError): + try: + return "~/" + str(path.expanduser().resolve().relative_to(pathlib.Path.home())) + except (FileNotFoundError, ValueError): + return str(path) + + +def run_json(command: list[str], cwd: pathlib.Path) -> tuple[bool, dict[str, Any]]: + completed = subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + raw = completed.stdout if completed.returncode == 0 else completed.stderr + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + parsed = { + "ok": False, + "errors": [ + f"command exited {completed.returncode} and did not emit JSON", + completed.stderr.strip() or completed.stdout.strip(), + ], + } + return completed.returncode == 0, parsed + + +def shell_join(argv: list[str]) -> str: + import shlex + + return " ".join(shlex.quote(part) for part in argv) + + +def discover_official_plugin_versions(plugin_root: pathlib.Path, repo: pathlib.Path) -> list[dict[str, Any]]: + if not plugin_root.exists(): + return [] + versions: list[dict[str, Any]] = [] + for path in sorted(plugin_root.iterdir()): + if not path.is_dir(): + continue + skill_path = path / "skills/record-and-replay/SKILL.md" + versions.append( + { + "version": path.name, + "path": display_path(path, repo), + "hasSkill": skill_path.exists(), + } + ) + return versions + + +def default_fixture_name(scenario: str, official_plugin_version: str) -> str: + version = official_plugin_version.replace("record-and-replay", "").strip() or "unknown" + safe_version = version.replace(" ", "-") + return f"official-{scenario}-{safe_version}" + + +def scenario_candidate_command( + repo: pathlib.Path, + scenario: str, + name: str, + fixture_root: pathlib.Path, +) -> list[str] | None: + if scenario not in {"simple-action-stop", "drag-stop"}: + return None + candidate_root = fixture_root / "ocu-candidates" + return [ + "python3", + "scripts/ingest-ocu-record-and-replay-candidate.py", + "--run-action-smoke", + "--scenario", + scenario, + "--name", + f"ocu-{scenario}", + "--output-dir", + display_path(candidate_root, repo), + "--official-root", + display_path(fixture_root, repo), + "--require-mcp-transcript-evidence", + ] + + +def build_commands( + repo: pathlib.Path, + args: argparse.Namespace, + fixture_name: str, +) -> dict[str, Any]: + fixture_root = args.fixture_root + status_json = args.status_json_placeholder + transcript = args.mcp_transcript_placeholder + base = [ + "python3", + "scripts/ingest-official-record-and-replay-fixture.py", + "--status-json", + status_json, + "--name", + fixture_name, + "--scenario", + args.scenario, + "--output-dir", + display_path(fixture_root, repo), + "--official-plugin-version", + args.official_plugin_version, + ] + if args.status_json_base_dir_placeholder: + base.extend(["--status-json-base-dir", args.status_json_base_dir_placeholder]) + + inspect = [*base, "--inspect-only"] + if args.include_transcript: + inspect.extend(["--mcp-transcript", transcript, "--require-mcp-transcript-evidence"]) + + import_cmd = [ + *base, + "--check-fixture-set", + "--check-coverage", + ] + if args.include_transcript: + import_cmd.extend(["--mcp-transcript", transcript, "--require-mcp-transcript-evidence"]) + if args.require_coverage_on_import: + import_cmd.append("--require-coverage") + + candidate = scenario_candidate_command(repo, args.scenario, fixture_name, fixture_root) + commands: dict[str, Any] = { + "inspectOnly": inspect, + "inspectOnlyShell": shell_join(inspect), + "importFixture": import_cmd, + "importFixtureShell": shell_join(import_cmd), + "coverageReadiness": [ + "python3", + "scripts/check-event-stream-official-fixture-coverage.py", + "--fixture-root", + display_path(fixture_root, repo), + "--require-readiness", + ], + "fixtureSet": [ + "python3", + "scripts/check-event-stream-official-fixture-set.py", + "--official-root", + display_path(fixture_root, repo), + "--require-scenario", + args.scenario, + ], + "strictOfficialGoldenGate": ["make", "record-and-replay-official-golden-gate-audit"], + "strictOfficialGoldenExpectedFailureAudit": [ + "python3", + "scripts/check-record-and-replay-baseline-summary.py", + "dist/record-and-replay-official-golden-gate-summary.json", + "--allow-strict-official-golden-missing", + ], + } + commands["coverageReadinessShell"] = shell_join(commands["coverageReadiness"]) + commands["fixtureSetShell"] = shell_join(commands["fixtureSet"]) + commands["strictOfficialGoldenGateShell"] = shell_join(commands["strictOfficialGoldenGate"]) + commands["strictOfficialGoldenExpectedFailureAuditShell"] = shell_join( + commands["strictOfficialGoldenExpectedFailureAudit"] + ) + if candidate is not None: + commands["ocuCandidate"] = candidate + commands["ocuCandidateShell"] = shell_join(candidate) + else: + commands["ocuCandidate"] = None + commands["ocuCandidateNote"] = ( + "Import this scenario from an existing OCU recording or preserved smoke JSON; " + "synthetic --run-action-smoke is only supported for simple-action-stop and drag-stop." + ) + return commands + + +def write_text(path: pathlib.Path, text: str, *, executable: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + if executable: + path.chmod(0o755) + + +def bash_array( + command: list[str] | None, + args: argparse.Namespace, + *, + status_json_path: str = '"${packet_dir}/inputs/event_stream_stop-response.json"', + transcript_path: str = '"${packet_dir}/inputs/mcp-transcript.json"', +) -> str: + if command is None: + return "" + + import shlex + + parts: list[str] = [] + for item in command: + if item == args.status_json_placeholder: + parts.append(status_json_path) + elif item == args.mcp_transcript_placeholder: + parts.append(transcript_path) + else: + parts.append(shlex.quote(item)) + return "\n ".join(parts) + + +def render_packet_script(command: list[str] | None, args: argparse.Namespace, repo: pathlib.Path) -> str: + if command is None: + return "#!/usr/bin/env bash\nset -euo pipefail\necho 'No command is available for this scenario.' >&2\nexit 1\n" + import shlex + + repo_root = shlex.quote(str(repo)) + needs_status_json = args.status_json_placeholder in command + needs_transcript = args.mcp_transcript_placeholder in command + guard_lines: list[str] = [] + if needs_status_json or needs_transcript: + guard_lines.append( + textwrap.dedent( + """\ + check_replaced_input() { + local path="$1" + local label="$2" + if [[ ! -f "${path}" ]]; then + echo "Missing packet input: ${label} (${path})" >&2 + exit 2 + fi + if python3 - "${path}" <<'PY' + import json + import sys + + try: + with open(sys.argv[1], "r", encoding="utf-8") as handle: + value = json.load(handle) + except Exception: + sys.exit(1) + sys.exit(0 if isinstance(value, dict) and value.get("_placeholder") is True else 1) + PY + then + echo "Packet input still contains placeholder: ${label} (${path}). Replace it with hosted official JSON before running ${BASH_SOURCE[0]##*/}." >&2 + exit 2 + fi + } + """ + ) + ) + if needs_status_json: + guard_lines.append( + 'check_replaced_input "${packet_dir}/inputs/event_stream_stop-response.json" "event_stream_stop response"\n' + ) + if needs_transcript: + guard_lines.append( + 'check_replaced_input "${packet_dir}/inputs/mcp-transcript.json" "MCP transcript"\n' + ) + guard = "\n".join(guard_lines) + if guard: + guard += "\n" + semantic_verify = "" + if needs_status_json: + semantic_verify = '"${packet_dir}/verify-inputs.sh" >/dev/null\n\n' + return ( + "#!/usr/bin/env bash\n\n" + "set -euo pipefail\n\n" + 'packet_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + f"default_repo_root={repo_root}\n" + 'repo_root="${REPO_ROOT:-${default_repo_root}}"\n\n' + f"{guard}" + f"{semantic_verify}" + "cd \"${repo_root}\"\n" + "cmd=(\n" + f" {bash_array(command, args)}\n" + ")\n" + '"${cmd[@]}"\n' + ) + + +def render_repo_script(command: list[str], repo: pathlib.Path) -> str: + import shlex + + repo_root = shlex.quote(str(repo)) + return ( + "#!/usr/bin/env bash\n\n" + "set -euo pipefail\n\n" + f"default_repo_root={repo_root}\n" + 'repo_root="${REPO_ROOT:-${default_repo_root}}"\n\n' + "cd \"${repo_root}\"\n" + "cmd=(\n" + f" {shell_join(command)}\n" + ")\n" + '"${cmd[@]}"\n' + ) + + +def render_packet_inputs_script(args: argparse.Namespace) -> str: + transcript_check = "" + if args.include_transcript: + transcript_check = ( + 'check_json_input "${packet_dir}/inputs/mcp-transcript.json" "MCP transcript"\n' + ) + return ( + "#!/usr/bin/env bash\n\n" + "set -euo pipefail\n\n" + 'packet_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n\n' + "check_json_input() {\n" + ' local path="$1"\n' + ' local label="$2"\n' + ' if [[ ! -f "${path}" ]]; then\n' + ' echo "Missing packet input: ${label} (${path})" >&2\n' + " exit 2\n" + " fi\n" + ' python3 - "${path}" "${label}" <<\'PY\'\n' + "import json\n" + "import sys\n\n" + "path, label = sys.argv[1:3]\n" + "try:\n" + " with open(path, \"r\", encoding=\"utf-8\") as handle:\n" + " value = json.load(handle)\n" + "except Exception as error:\n" + " print(f\"Invalid packet JSON: {label} ({path}): {error}\", file=sys.stderr)\n" + " sys.exit(2)\n" + "if isinstance(value, dict) and value.get(\"_placeholder\") is True:\n" + " print(\n" + " f\"Packet input still contains placeholder: {label} ({path}). \"\n" + " \"Replace it with hosted official JSON before running packet import.\",\n" + " file=sys.stderr,\n" + " )\n" + " sys.exit(2)\n" + "PY\n" + "}\n\n" + 'check_json_input "${packet_dir}/inputs/event_stream_stop-response.json" "event_stream_stop response"\n' + f"{transcript_check}" + 'python3 - "${packet_dir}" <<\'PY\'\n' + "import json\n" + "import sys\n\n" + "from pathlib import Path\n\n" + "packet_dir = Path(sys.argv[1])\n" + "path_keys = (\"metadataPath\", \"sessionPath\", \"eventsPath\", \"suppressedEventsPath\", \"sessionDirectoryPath\")\n" + "transcript_keys = {\n" + " \"startResponseShape\",\n" + " \"repeatStartResponseShape\",\n" + " \"statusResponseShape\",\n" + " \"stopResponseShape\",\n" + " \"repeatStopResponseShape\",\n" + " \"finalStatusResponseShape\",\n" + " \"transcript\",\n" + "}\n\n" + "def load_packet_json(relative_path):\n" + " path = packet_dir / relative_path\n" + " try:\n" + " with path.open(\"r\", encoding=\"utf-8\") as handle:\n" + " return json.load(handle)\n" + " except Exception as error:\n" + " print(f\"Invalid packet JSON: {relative_path}: {error}\", file=sys.stderr)\n" + " sys.exit(2)\n\n" + "def decode_json_string(value):\n" + " if not isinstance(value, str):\n" + " return None\n" + " try:\n" + " return json.loads(value)\n" + " except json.JSONDecodeError:\n" + " return None\n\n" + "def iter_objects(value):\n" + " if isinstance(value, dict):\n" + " yield value\n" + " for child in value.values():\n" + " decoded = decode_json_string(child)\n" + " if decoded is not None:\n" + " yield from iter_objects(decoded)\n" + " else:\n" + " yield from iter_objects(child)\n" + " elif isinstance(value, list):\n" + " for child in value:\n" + " yield from iter_objects(child)\n\n" + "contract = load_packet_json(\"capture-contract.json\")\n" + "recipe = load_packet_json(\"scenario-recipe.json\")\n" + "if contract.get(\"scenario\") != recipe.get(\"scenario\"):\n" + " print(\"Packet contract scenario does not match scenario recipe\", file=sys.stderr)\n" + " sys.exit(2)\n" + "status_json = load_packet_json(\"inputs/event_stream_stop-response.json\")\n" + "found_handoff_keys = sorted({\n" + " key\n" + " for obj in iter_objects(status_json)\n" + " for key in path_keys\n" + " if isinstance(obj.get(key), str) and obj.get(key)\n" + "})\n" + "if not found_handoff_keys:\n" + " print(\n" + " \"Packet status JSON does not contain Record & Replay handoff path evidence: \"\n" + " + \", \".join(path_keys),\n" + " file=sys.stderr,\n" + " )\n" + " sys.exit(2)\n" + "required_handoff_keys = tuple(contract.get(\"requiredStatusHandoffPathKeys\") or ())\n" + "missing_required_handoff_keys = [\n" + " key for key in required_handoff_keys if key not in found_handoff_keys\n" + "]\n" + "required_any_of_groups = contract.get(\"requiredStatusHandoffAnyOf\") or []\n" + "missing_any_of_groups = [\n" + " group\n" + " for group in required_any_of_groups\n" + " if isinstance(group, list) and not any(key in found_handoff_keys for key in group)\n" + "]\n" + "if missing_required_handoff_keys or missing_any_of_groups:\n" + " print(\n" + " \"Packet status JSON is missing required Record & Replay handoff evidence: \"\n" + " + json.dumps({\n" + " \"missingRequiredStatusHandoffPathKeys\": missing_required_handoff_keys,\n" + " \"missingRequiredStatusHandoffAnyOf\": missing_any_of_groups,\n" + " \"foundHandoffPathKeys\": found_handoff_keys,\n" + " }, sort_keys=True),\n" + " file=sys.stderr,\n" + " )\n" + " sys.exit(2)\n" + "found_transcript_keys = []\n" + "requires_transcript = bool(contract.get(\"requiresMcpTranscriptInput\"))\n" + "if requires_transcript:\n" + " transcript_json = load_packet_json(\"inputs/mcp-transcript.json\")\n" + " found_transcript_keys = sorted({\n" + " key\n" + " for obj in iter_objects(transcript_json)\n" + " for key in transcript_keys\n" + " if key in obj\n" + " })\n" + " if not found_transcript_keys:\n" + " print(\n" + " \"Packet MCP transcript does not contain MCP transcript evidence: \"\n" + " + \", \".join(sorted(transcript_keys)),\n" + " file=sys.stderr,\n" + " )\n" + " sys.exit(2)\n" + "print(json.dumps({\n" + " \"ok\": True,\n" + " \"stage\": \"verify-inputs\",\n" + " \"packetDir\": str(packet_dir),\n" + " \"scenario\": contract.get(\"scenario\"),\n" + " \"foundHandoffPathKeys\": found_handoff_keys,\n" + " \"requiredStatusHandoffPathKeys\": list(required_handoff_keys),\n" + " \"requiredStatusHandoffAnyOf\": required_any_of_groups,\n" + " \"requiresMcpTranscriptInput\": requires_transcript,\n" + " \"foundTranscriptEvidenceKeys\": found_transcript_keys,\n" + "}, sort_keys=True))\n" + "PY\n" + ) + + +def render_packet_set_script(scenarios: list[str], child_script: str, *, include_transcript: bool) -> str: + import shlex + + scenario_lines = "\n".join(f" {shlex.quote(scenario)}" for scenario in scenarios) + preflight = "" + if child_script in {"inspect-only.sh", "import-fixture.sh"}: + preflight = textwrap.dedent( + """\ + check_replaced_input() { + local path="$1" + local label="$2" + if [[ ! -f "${path}" ]]; then + echo "Missing packet input: ${label} (${path})" >&2 + exit 2 + fi + if python3 - "${path}" <<'PY' + import json + import sys + + try: + with open(sys.argv[1], "r", encoding="utf-8") as handle: + value = json.load(handle) + except Exception: + sys.exit(1) + sys.exit(0 if isinstance(value, dict) and value.get("_placeholder") is True else 1) + PY + then + echo "Packet input still contains placeholder: ${label} (${path}). Replace it with hosted official JSON before running ${BASH_SOURCE[0]##*/}." >&2 + exit 2 + fi + } + + for scenario in "${scenarios[@]}"; do + check_replaced_input "${packet_set_dir}/${scenario}/inputs/event_stream_stop-response.json" "event_stream_stop response" + """ + ) + if include_transcript: + preflight += textwrap.dedent( + """\ + check_replaced_input "${packet_set_dir}/${scenario}/inputs/mcp-transcript.json" "MCP transcript" + """ + ) + preflight += textwrap.dedent( + """\ + done + + """ + ) + return ( + "#!/usr/bin/env bash\n\n" + "set -euo pipefail\n\n" + 'packet_set_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + "scenarios=(\n" + f"{scenario_lines}\n" + ")\n\n" + f"{preflight}" + 'for scenario in "${scenarios[@]}"; do\n' + f' script="${{packet_set_dir}}/${{scenario}}/{child_script}"\n' + ' if [[ ! -x "${script}" ]]; then\n' + f' echo "Missing executable packet script: ${{scenario}}/{child_script}" >&2\n' + " exit 1\n" + " fi\n" + ' echo "==> ${scenario}"\n' + ' "${script}"\n' + "done\n" + ) + + +def render_packet_set_optional_script(scenarios: list[str], child_script: str) -> str: + import shlex + + scenario_lines = "\n".join(f" {shlex.quote(scenario)}" for scenario in scenarios) + return ( + "#!/usr/bin/env bash\n\n" + "set -euo pipefail\n\n" + 'packet_set_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"\n' + "scenarios=(\n" + f"{scenario_lines}\n" + ")\n\n" + 'ran=0\n' + 'for scenario in "${scenarios[@]}"; do\n' + f' script="${{packet_set_dir}}/${{scenario}}/{child_script}"\n' + ' if [[ ! -x "${script}" ]]; then\n' + f' echo "==> ${{scenario}}: skipping; no generated {child_script}"\n' + " continue\n" + " fi\n" + ' echo "==> ${scenario}"\n' + ' "${script}"\n' + " ran=$((ran + 1))\n" + "done\n\n" + 'if [[ "${ran}" -eq 0 ]]; then\n' + f' echo "No generated {child_script} scripts were available." >&2\n' + "fi\n" + ) + + +def render_workflow_verifier_script(*, packet_set: bool) -> str: + mode = "packet-set" if packet_set else "packet" + return textwrap.dedent( + f"""\ + #!/usr/bin/env bash + + set -euo pipefail + + packet_dir="$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)" + + python3 - "${{packet_dir}}" "{mode}" <<'PY' + import json + import pathlib + import sys + + packet_dir = pathlib.Path(sys.argv[1]) + mode = sys.argv[2] + + def fail(message): + print(message, file=sys.stderr) + sys.exit(2) + + def load_json(path): + try: + return json.loads(path.read_text()) + except Exception as exc: + fail(f"Invalid workflow JSON {{path}}: {{exc}}") + + def require_file(path, label): + if not path.is_file(): + fail(f"Workflow {{label}} is missing: {{path}}") + + def require_executable(path, label): + require_file(path, label) + if not path.stat().st_mode & 0o111: + fail(f"Workflow command is not executable: {{path}}") + + def verify_workflow(root, contract, *, scenario): + workflow = contract.get("postCaptureWorkflow") + if not isinstance(workflow, list) or not workflow: + fail(f"Workflow missing for {{scenario}}") + requires_transcript = bool(contract.get("requiresMcpTranscriptInput")) + ids = [step.get("id") for step in workflow if isinstance(step, dict)] + if "replace-status-json" not in ids: + fail(f"Workflow missing replace-status-json for {{scenario}}") + if requires_transcript and "replace-mcp-transcript" not in ids: + fail(f"Workflow missing replace-mcp-transcript for {{scenario}}") + if not requires_transcript and "replace-mcp-transcript" in ids: + fail(f"Workflow unexpectedly requires MCP transcript for {{scenario}}") + required_commands = {{ + "verify-inputs": "verify-inputs.sh", + "inspect-only": "inspect-only.sh", + "import-fixture": "import-fixture.sh", + "check-coverage": "check-coverage.sh", + "check-fixture-set": "check-fixture-set.sh", + "strict-golden-gate": "strict-golden-gate.sh", + "strict-expected-failure-audit": "strict-expected-failure-audit.sh", + }} + optional_commands = {{"ingest-ocu-candidate": "ingest-ocu-candidate.sh"}} + for step in workflow: + if not isinstance(step, dict): + fail(f"Workflow step is not an object for {{scenario}}") + step_id = step.get("id") + input_path = step.get("input") + command = step.get("command") + if input_path is not None: + require_file(root / input_path, f"input {{step_id}}") + if command is not None: + if not isinstance(command, str) or not command.startswith("./"): + fail(f"Workflow command must be a local ./ script for {{scenario}}: {{command!r}}") + require_executable(root / command[2:], f"command {{step_id}}") + expected = required_commands.get(step_id) + if expected is not None and command != f"./{{expected}}": + fail(f"Workflow command drift for {{scenario}} {{step_id}}: {{command!r}}") + optional = optional_commands.get(step_id) + if optional is not None and command != f"./{{optional}}": + fail(f"Workflow optional command drift for {{scenario}} {{step_id}}: {{command!r}}") + for step_id, script_name in required_commands.items(): + if step_id not in ids: + fail(f"Workflow missing required step {{step_id}} for {{scenario}}") + require_executable(root / script_name, f"script {{step_id}}") + if "ingest-ocu-candidate" in ids: + require_executable(root / "ingest-ocu-candidate.sh", f"script ingest-ocu-candidate for {{scenario}}") + else: + if (root / "ingest-ocu-candidate.sh").exists(): + fail(f"Workflow omits ingest-ocu-candidate but script exists for {{scenario}}") + require_file(root / "inputs/event_stream_stop-response.json", f"status input for {{scenario}}") + if requires_transcript: + require_file(root / "inputs/mcp-transcript.json", f"MCP transcript input for {{scenario}}") + elif (root / "inputs/mcp-transcript.json").exists(): + fail(f"Workflow says MCP transcript is disabled but input exists for {{scenario}}") + + if mode == "packet": + contract = load_json(packet_dir / "capture-contract.json") + verify_workflow(packet_dir, contract, scenario=contract.get("scenario", "")) + print(json.dumps({{ + "ok": True, + "stage": "verify-workflow", + "checkedWorkflow": True, + "checkedWorkflowCommands": True, + "checkedWorkflowInputs": True, + "scenario": contract.get("scenario"), + }}, sort_keys=True)) + elif mode == "packet-set": + manifest = load_json(packet_dir / "capture-packets.json") + contracts = manifest.get("captureContracts") + workflows = manifest.get("postCaptureWorkflow") + if not isinstance(contracts, dict) or not contracts: + fail("Packet set workflow manifest is missing captureContracts") + if not isinstance(workflows, dict) or not workflows: + fail("Packet set workflow manifest is missing postCaptureWorkflow") + scenarios = manifest.get("scenarios") + if not isinstance(scenarios, list) or not scenarios: + fail("Packet set workflow manifest is missing scenarios") + for scenario in scenarios: + contract = contracts.get(scenario) + if not isinstance(contract, dict): + fail(f"Packet set missing capture contract for {{scenario}}") + if workflows.get(scenario) != contract.get("postCaptureWorkflow"): + fail(f"Packet set workflow does not match capture contract for {{scenario}}") + verify_workflow(packet_dir / scenario, contract, scenario=scenario) + scripts = manifest.get("scripts") + required_root_scripts = {{ + "verifyAll": "verify-all.sh", + "inspectAll": "inspect-all.sh", + "importAll": "import-all.sh", + "checkAll": "check-all.sh", + "ingestOcuCandidates": "ingest-ocu-candidates.sh", + "strictExpectedFailureAudit": "strict-expected-failure-audit.sh", + "verifyWorkflow": "verify-workflow.sh", + }} + if not isinstance(scripts, dict): + fail("Packet set workflow manifest is missing scripts") + for key, script_name in required_root_scripts.items(): + if not str(scripts.get(key, "")).endswith(script_name): + fail(f"Packet set scripts manifest drift for {{key}}") + require_executable(packet_dir / script_name, f"root script {{key}}") + print(json.dumps({{ + "ok": True, + "stage": "verify-workflow", + "checkedWorkflow": True, + "checkedWorkflowCommands": True, + "checkedWorkflowInputs": True, + "checkedPacketSetWorkflow": True, + "scenarios": scenarios, + }}, sort_keys=True)) + else: + fail(f"unknown workflow verifier mode: {{mode}}") + PY + """ + ) + + +def placeholder_json(label: str) -> str: + return json.dumps( + { + "_placeholder": True, + "replaceWith": label, + "note": "Replace this file with the JSON captured from the official Codex Record & Replay workflow before running the packet scripts.", + }, + indent=2, + sort_keys=True, + ) + "\n" + + +def capture_contract(result: dict[str, Any], *, include_transcript: bool) -> dict[str, Any]: + recipe = result["scenarioRecipe"] + workflow = post_capture_workflow(result, include_transcript=include_transcript) + return { + "formatVersion": 1, + "scenario": result["scenario"], + "fixtureName": result["fixtureName"], + "captureGoal": recipe["captureGoal"], + "userAction": recipe["userAction"], + "expectedActionEvents": recipe.get("expectedActionEvents", []), + "expectedEndReason": recipe.get("expectedEndReason"), + "requiresMcpTranscriptInput": include_transcript, + "statusHandoffPathKeys": [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + "sessionDirectoryPath", + ], + "requiredStatusHandoffPathKeys": [ + "eventsPath", + ], + "requiredStatusHandoffAnyOf": [ + [ + "metadataPath", + "sessionPath", + ], + ], + "optionalStatusHandoffPathKeys": [ + "suppressedEventsPath", + "sessionDirectoryPath", + ], + "requiredTranscriptEvidenceKeys": [ + "startResponseShape", + "statusResponseShape", + "stopResponseShape", + "finalStatusResponseShape", + "transcript", + ] + if include_transcript + else [], + "postCaptureWorkflow": workflow, + "postCaptureChecks": [ + "./verify-inputs.sh", + "./verify-workflow.sh", + "./inspect-only.sh", + "./import-fixture.sh", + "./check-fixture-set.sh", + "./strict-golden-gate.sh", + "./strict-expected-failure-audit.sh", + ], + } + + +def post_capture_workflow( + result: dict[str, Any], + *, + include_transcript: bool, +) -> list[dict[str, Any]]: + workflow: list[dict[str, Any]] = [ + { + "id": "replace-status-json", + "description": "Replace hosted status/stop JSON placeholder with official Record & Replay handoff JSON.", + "input": "inputs/event_stream_stop-response.json", + "required": True, + } + ] + if include_transcript: + workflow.append( + { + "id": "replace-mcp-transcript", + "description": "Replace MCP transcript placeholder with same-session official transcript evidence.", + "input": "inputs/mcp-transcript.json", + "required": True, + } + ) + workflow.extend( + [ + { + "id": "verify-inputs", + "command": "./verify-inputs.sh", + "required": True, + }, + { + "id": "inspect-only", + "command": "./inspect-only.sh", + "required": True, + }, + { + "id": "import-fixture", + "command": "./import-fixture.sh", + "required": True, + }, + { + "id": "check-coverage", + "command": "./check-coverage.sh", + "required": True, + }, + { + "id": "check-fixture-set", + "command": "./check-fixture-set.sh", + "required": True, + }, + ] + ) + if (result.get("commands") or {}).get("ocuCandidate") is not None: + workflow.append( + { + "id": "ingest-ocu-candidate", + "command": "./ingest-ocu-candidate.sh", + "required": False, + "condition": "same-scenario OCU candidate wrapper is generated for this scenario", + } + ) + workflow.extend( + [ + { + "id": "strict-golden-gate", + "command": "./strict-golden-gate.sh", + "required": True, + "condition": "after required official successful recording fixtures are imported", + }, + { + "id": "strict-expected-failure-audit", + "command": "./strict-expected-failure-audit.sh", + "required": True, + "condition": "while required official successful recording fixtures are still missing", + }, + ] + ) + return workflow + + +def render_capture_readme(result: dict[str, Any], *, include_transcript: bool) -> str: + recipe = result["scenarioRecipe"] + commands = result["commands"] + expected_events = ", ".join(recipe.get("expectedActionEvents") or [""]) + expected_end_reason = recipe.get("expectedEndReason") + if expected_end_reason is None: + expected_end_reason = "" + notes = "\n".join(f"- {note}" for note in recipe.get("notes", [])) + transcript_requirement = "yes" if include_transcript else "no" + if include_transcript: + transcript_input = ( + "- `inputs/mcp-transcript.json`: required same-session MCP transcript evidence." + ) + else: + transcript_input = ( + "This packet was generated with `--no-include-transcript`; it does not create " + "`inputs/mcp-transcript.json`, and generated inspect/import commands do not " + "require MCP transcript evidence." + ) + if commands.get("ocuCandidateShell"): + ocu_candidate_step = ( + "\n After the official fixture imports cleanly, run:\n\n" + " ```sh\n" + " ./ingest-ocu-candidate.sh\n" + " ```\n" + ) + else: + ocu_candidate_step = ( + "\n This scenario does not have a generated OCU candidate ingest wrapper; " + "use the `commands.ocuCandidateNote` guidance in `preflight.json`.\n" + ) + return textwrap.dedent( + f"""\ + # Record & Replay Official Capture Packet + + Scenario: `{result["scenario"]}` + Fixture name: `{result["fixtureName"]}` + MCP transcript input required: `{transcript_requirement}` + + ## Capture Goal + + {recipe["captureGoal"]} + + User action: + + {recipe["userAction"]} + + Expected action events: `{expected_events}` + Expected end reason: `{expected_end_reason}` + + Notes: + + {notes} + + ## Inputs + + Replace these placeholder files after the official hosted recording finishes: + + - `inputs/event_stream_stop-response.json`: final `event_stream_stop` or completed `event_stream_status` JSON from the official Codex Record & Replay tool. + {transcript_input} + + `capture-contract.json` records the expected scenario, handoff path evidence, transcript evidence requirement, action events, and end reason for this packet. `verify-inputs.sh` checks that the status JSON contains Record & Replay handoff path evidence and, when transcript input is enabled, that `inputs/mcp-transcript.json` looks like an MCP transcript. + + `capture-contract.json` also includes `postCaptureWorkflow`, the machine-readable command order to run after hosted official recording completes. + + `verify-workflow.sh` checks that this packet's machine-readable workflow still matches the generated inputs and wrapper scripts. It does not inspect hosted JSON and can run before recording. + + Do not commit raw official recording JSON, screenshots, secrets, private text, or machine-local paths. Run inspect-only first, then let the import script create a sanitized fixture. + + ## Commands + + If you captured hosted Codex tool responses as separate JSON files, first finalize packet inputs from the repository root: + + ```sh + python3 scripts/finalize-record-and-replay-official-capture-packet.py \\ + --packet-dir \\ + --start-json \\ + --status-json \\ + --stop-json \\ + --final-status-json + ``` + + From this packet directory, run: + + ```sh + ./verify-inputs.sh + ./verify-workflow.sh + ./inspect-only.sh + ./import-fixture.sh + ./check-coverage.sh + ./check-fixture-set.sh + ./strict-golden-gate.sh + ./strict-expected-failure-audit.sh + ``` + {ocu_candidate_step} + + The generated shell wrappers embed the repository path captured when this packet was created. Set `REPO_ROOT=/path/to/repo` if you move the packet or run it against another checkout. + + Equivalent inspect-only command: + + ```sh + {commands["inspectOnlyShell"]} + ``` + + Equivalent import command: + + ```sh + {commands["importFixtureShell"]} + ``` + + Equivalent fixture-set gate: + + ```sh + {commands["fixtureSetShell"]} + ``` + + Equivalent strict golden gate audit: + + ```sh + {commands["strictOfficialGoldenGateShell"]} + ``` + + Equivalent strict expected-failure audit while required official golden is still missing: + + ```sh + {commands["strictOfficialGoldenExpectedFailureAuditShell"]} + ``` + """ + ) + + +def write_capture_packet( + packet_dir: pathlib.Path, + result: dict[str, Any], + args: argparse.Namespace, + repo: pathlib.Path, +) -> dict[str, Any]: + packet_dir.mkdir(parents=True, exist_ok=True) + inputs_dir = packet_dir / "inputs" + inputs_dir.mkdir(parents=True, exist_ok=True) + + commands = result["commands"] + files: list[str] = [] + + def record(path: pathlib.Path) -> None: + files.append(display_path(path, repo)) + + readme = packet_dir / "README.md" + write_text(readme, render_capture_readme(result, include_transcript=args.include_transcript)) + record(readme) + + recipe_path = packet_dir / "scenario-recipe.json" + write_text(recipe_path, json.dumps(result["scenarioRecipe"], indent=2, sort_keys=True) + "\n") + record(recipe_path) + + contract = capture_contract(result, include_transcript=args.include_transcript) + contract_path = packet_dir / "capture-contract.json" + write_text(contract_path, json.dumps(contract, indent=2, sort_keys=True) + "\n") + record(contract_path) + + preflight_path = packet_dir / "preflight.json" + write_text(preflight_path, json.dumps(result, indent=2, sort_keys=True) + "\n") + record(preflight_path) + + status_path = inputs_dir / "event_stream_stop-response.json" + write_text(status_path, placeholder_json("official event_stream_stop or completed event_stream_status response JSON")) + record(status_path) + + transcript_path = inputs_dir / "mcp-transcript.json" + if args.include_transcript: + write_text(transcript_path, placeholder_json("same-session official MCP transcript JSON")) + record(transcript_path) + + scripts = { + "verify-inputs.sh": None, + "verify-workflow.sh": None, + "inspect-only.sh": commands.get("inspectOnly"), + "import-fixture.sh": commands.get("importFixture"), + "check-coverage.sh": commands.get("coverageReadiness"), + "check-fixture-set.sh": commands.get("fixtureSet"), + "strict-golden-gate.sh": commands.get("strictOfficialGoldenGate"), + "strict-expected-failure-audit.sh": commands.get( + "strictOfficialGoldenExpectedFailureAudit" + ), + } + if commands.get("ocuCandidate") is not None: + scripts["ingest-ocu-candidate.sh"] = commands.get("ocuCandidate") + for name, command in scripts.items(): + path = packet_dir / name + if name == "verify-inputs.sh": + write_text(path, render_packet_inputs_script(args), executable=True) + elif name == "verify-workflow.sh": + write_text(path, render_workflow_verifier_script(packet_set=False), executable=True) + else: + write_text(path, render_packet_script(command, args, repo), executable=True) + record(path) + + return { + "path": display_path(packet_dir, repo), + "files": files, + "includeTranscript": bool(args.include_transcript), + "requiresMcpTranscriptInput": bool(args.include_transcript), + "postCaptureWorkflow": contract["postCaptureWorkflow"], + "captureContractPath": display_path(contract_path, repo), + "statusResponseInputPath": display_path(status_path, repo), + "mcpTranscriptInputPath": display_path(transcript_path, repo) + if args.include_transcript + else None, + "verifyInputsShell": f"bash {display_path(packet_dir / 'verify-inputs.sh', repo)}", + "verifyWorkflowShell": f"bash {display_path(packet_dir / 'verify-workflow.sh', repo)}", + "inspectOnlyShell": f"bash {display_path(packet_dir / 'inspect-only.sh', repo)}", + "importFixtureShell": f"bash {display_path(packet_dir / 'import-fixture.sh', repo)}", + "checkCoverageShell": f"bash {display_path(packet_dir / 'check-coverage.sh', repo)}", + "checkFixtureSetShell": f"bash {display_path(packet_dir / 'check-fixture-set.sh', repo)}", + "strictGoldenGateShell": f"bash {display_path(packet_dir / 'strict-golden-gate.sh', repo)}", + "strictExpectedFailureAuditShell": ( + f"bash {display_path(packet_dir / 'strict-expected-failure-audit.sh', repo)}" + ), + "ingestOcuCandidateShell": ( + f"bash {display_path(packet_dir / 'ingest-ocu-candidate.sh', repo)}" + if commands.get("ocuCandidate") is not None + else None + ), + } + + +def write_capture_packet_set( + root: pathlib.Path, + scenarios: list[str], + results: list[dict[str, Any]], + repo: pathlib.Path, + *, + include_transcript: bool, +) -> dict[str, Any]: + root.mkdir(parents=True, exist_ok=True) + packets = { + result["scenario"]: result.get("capturePacket") + for result in results + if isinstance(result.get("capturePacket"), dict) + } + contracts = { + result["scenario"]: capture_contract(result, include_transcript=include_transcript) + for result in results + if isinstance(result.get("capturePacket"), dict) + } + workflows = { + scenario: contract.get("postCaptureWorkflow", []) + for scenario, contract in contracts.items() + } + contract_paths = { + scenario: packet.get("captureContractPath") + for scenario, packet in packets.items() + if isinstance(packet.get("captureContractPath"), str) + } + scripts = { + "verifyAll": root / "verify-all.sh", + "verifyWorkflow": root / "verify-workflow.sh", + "inspectAll": root / "inspect-all.sh", + "importAll": root / "import-all.sh", + "checkAll": root / "check-all.sh", + "ingestOcuCandidates": root / "ingest-ocu-candidates.sh", + "strictExpectedFailureAudit": root / "strict-expected-failure-audit.sh", + } + write_text( + scripts["verifyAll"], + render_packet_set_script(scenarios, "verify-inputs.sh", include_transcript=include_transcript), + executable=True, + ) + write_text( + scripts["verifyWorkflow"], + render_workflow_verifier_script(packet_set=True), + executable=True, + ) + write_text( + scripts["inspectAll"], + render_packet_set_script(scenarios, "inspect-only.sh", include_transcript=include_transcript), + executable=True, + ) + write_text( + scripts["importAll"], + render_packet_set_script(scenarios, "import-fixture.sh", include_transcript=include_transcript), + executable=True, + ) + write_text( + scripts["checkAll"], + render_packet_set_script(scenarios, "check-coverage.sh", include_transcript=include_transcript), + executable=True, + ) + write_text( + scripts["ingestOcuCandidates"], + render_packet_set_optional_script(scenarios, "ingest-ocu-candidate.sh"), + executable=True, + ) + strict_expected_failure_command = [ + "python3", + "scripts/check-record-and-replay-baseline-summary.py", + "dist/record-and-replay-official-golden-gate-summary.json", + "--allow-strict-official-golden-missing", + ] + write_text( + scripts["strictExpectedFailureAudit"], + render_repo_script(strict_expected_failure_command, repo), + executable=True, + ) + + manifest = { + "stage": "capture-packet-set", + "scenarios": scenarios, + "includeTranscript": include_transcript, + "requiresMcpTranscriptInput": include_transcript, + "capturePackets": packets, + "captureContracts": contracts, + "captureContractPaths": contract_paths, + "postCaptureWorkflow": workflows, + "scripts": {name: display_path(path, repo) for name, path in scripts.items()}, + "results": results, + } + write_text(root / "capture-packets.json", json.dumps(manifest, indent=2, sort_keys=True) + "\n") + scenario_lines = "\n".join(f"- `{scenario}`: `{scenario}/README.md`" for scenario in scenarios) + transcript_requirement = "yes" if include_transcript else "no" + if include_transcript: + transcript_step = ( + "2. Replace `inputs/mcp-transcript.json` with same-session transcript evidence." + ) + else: + transcript_step = ( + "2. This packet set was generated with `--no-include-transcript`; it does not " + "create or require `inputs/mcp-transcript.json`." + ) + write_text( + root / "README.md", + textwrap.dedent( + f"""\ + # Record & Replay Official Capture Packet Set + + This directory contains one official capture packet per scenario. These packets do not start official recording and do not write fixtures. Use them after each Codex-hosted official Record & Replay recording completes. + + MCP transcript input required: `{transcript_requirement}` + + `capture-packets.json` includes `captureContracts`, a machine-readable per-scenario copy of each packet's `capture-contract.json`. + + It also includes `postCaptureWorkflow`, the ordered per-scenario command plan to run after hosted official recording completes. + + `verify-workflow.sh` checks that `capture-packets.json`, each child `capture-contract.json`, and the generated wrapper scripts still agree. It does not inspect hosted JSON and can run before recording. + + ## Scenarios + + {scenario_lines} + + ## Workflow + + For each scenario directory: + + 1. Replace `inputs/event_stream_stop-response.json` with the hosted official `event_stream_stop` or completed `event_stream_status` JSON. + {transcript_step} + 3. Run `./inspect-only.sh`. + 4. If inspect-only succeeds, run `./import-fixture.sh`. + 5. Run `./check-coverage.sh` after importing required samples. + + Do not commit raw hosted JSON, screenshots, secrets, private text, or machine-local paths. Commit only sanitized fixtures produced by the import script. + + After replacing inputs, the root helpers can run every scenario in order: + + ```sh + ./verify-all.sh + ./verify-workflow.sh + ./inspect-all.sh + ./import-all.sh + ./check-all.sh + ./strict-expected-failure-audit.sh + ``` + + For scenarios that support synthetic OCU candidate sampling, run: + + ```sh + ./ingest-ocu-candidates.sh + ``` + + The candidate helper imports same-scenario OCU candidate fixtures but does not make candidate comparison part of the required official golden gate. It skips scenarios without a generated per-scenario OCU candidate wrapper, such as recording-required keyboard, cancel, or timeout scenarios. To use candidates for calibration, run the `fixtureSetGateShell` or `pairingPreflightShell` printed by each candidate import. + """ + ), + ) + return { + "path": display_path(root, repo), + "manifestPath": display_path(root / "capture-packets.json", repo), + "readmePath": display_path(root / "README.md", repo), + "scenarios": scenarios, + "includeTranscript": include_transcript, + "requiresMcpTranscriptInput": include_transcript, + "capturePackets": packets, + "captureContracts": contracts, + "captureContractPaths": contract_paths, + "postCaptureWorkflow": workflows, + "scripts": {name: display_path(path, repo) for name, path in scripts.items()}, + "verifyWorkflowShell": ( + f"bash {display_path(root / 'verify-workflow.sh', repo)}" + ), + "ingestOcuCandidatesShell": ( + f"bash {display_path(root / 'ingest-ocu-candidates.sh', repo)}" + ), + "strictExpectedFailureAuditShell": ( + f"bash {display_path(root / 'strict-expected-failure-audit.sh', repo)}" + ), + } + + +def preflight_packet_set(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + repo = pathlib.Path(__file__).resolve().parents[1] + if args.capture_packet_dir is None: + return 1, { + "ok": False, + "stage": "capture-packet-set", + "errors": ["--capture-packet-dir is required with --capture-packet-recommended-scenarios"], + } + if args.name: + return 1, { + "ok": False, + "stage": "capture-packet-set", + "errors": ["--name cannot be used with --capture-packet-recommended-scenarios"], + } + + scenarios = list(dict.fromkeys(args.recommended_scenario)) + results: list[dict[str, Any]] = [] + ok = True + for scenario in scenarios: + scenario_args = argparse.Namespace(**vars(args)) + scenario_args.scenario = scenario + scenario_args.name = None + scenario_args.capture_packet_dir = args.capture_packet_dir / scenario + exit_code, result = preflight(scenario_args) + results.append(result) + ok = ok and exit_code == 0 + + capture_packet_set = write_capture_packet_set( + args.capture_packet_dir, + scenarios, + results, + repo, + include_transcript=args.include_transcript, + ) + result = { + "ok": ok, + "stage": "capture-packet-set", + "capturePacketSet": capture_packet_set, + "scenarios": scenarios, + "results": results, + "nextActions": [ + "Capture each hosted official Record & Replay scenario, replace the matching packet inputs, then run that packet's inspect-only.sh.", + "Import required scenarios first; recommended scenarios calibrate keyboard, drag, cancel, and timeout behavior.", + ], + } + return 0 if ok else 1, result + + +def readiness_status(coverage: dict[str, Any], scenario: str) -> dict[str, Any]: + missing = coverage.get("missingOfficialScenarios") + not_ready = coverage.get("notReadyOfficialScenarios") + available = coverage.get("availableOfficialScenarios") + if not isinstance(missing, list): + missing = [] + if not isinstance(not_ready, list): + not_ready = [] + if not isinstance(available, list): + available = [] + scenario_available = scenario in available + scenario_missing = scenario in missing or not scenario_available + scenario_not_ready = scenario in not_ready + ready = ( + scenario_available + and not scenario_missing + and not scenario_not_ready + and coverage.get("coverageOk") is True + and coverage.get("requiredOfficialReadinessOk") is True + ) + return { + "scenario": scenario, + "available": scenario_available, + "missing": scenario_missing, + "notReady": scenario_not_ready, + "ready": ready, + } + + +def preflight(args: argparse.Namespace) -> tuple[int, dict[str, Any]]: + repo = pathlib.Path(__file__).resolve().parents[1] + fixture_root = args.fixture_root + fixture_name = args.name or default_fixture_name(args.scenario, args.official_plugin_version) + + coverage_command = [ + sys.executable, + str(repo / "scripts/check-event-stream-official-fixture-coverage.py"), + "--fixture-root", + str(fixture_root), + "--allow-missing", + "--check-readiness", + ] + for scenario in args.require_scenario: + coverage_command.extend(["--require-scenario", scenario]) + for scenario in args.recommended_scenario: + coverage_command.extend(["--recommended-scenario", scenario]) + coverage_ok, coverage = run_json(coverage_command, repo) + + plugin_versions = discover_official_plugin_versions(args.official_plugin_root, repo) + plugin_available = any(version.get("hasSkill") for version in plugin_versions) + scenario_status = readiness_status(coverage, args.scenario) + commands = build_commands(repo, args, fixture_name) + missing_required = coverage.get("missingOfficialScenarios") + missing_recommended = coverage.get("missingRecommendedOfficialScenarios") + not_ready = coverage.get("notReadyOfficialScenarios") + coverage_errors = coverage.get("errors") + if not isinstance(coverage_errors, list): + coverage_errors = [] + + result: dict[str, Any] = { + "ok": coverage_ok and (plugin_available or args.allow_missing_official_plugin), + "stage": "preflight", + "scenario": args.scenario, + "fixtureName": fixture_name, + "fixtureRoot": display_path(fixture_root, repo), + "officialPlugin": { + "root": display_path(args.official_plugin_root, repo), + "available": plugin_available, + "versions": plugin_versions, + "allowedMissing": args.allow_missing_official_plugin, + }, + "scenarioRecipe": scenario_recipe(args.scenario), + "coverage": coverage, + "coverageErrors": coverage_errors, + "scenarioStatus": scenario_status, + "nextActions": [], + "commands": commands, + } + + if args.capture_packet_dir is not None: + result["capturePacket"] = write_capture_packet(args.capture_packet_dir, result, args, repo) + result["nextActions"].append( + "Replace capture packet input placeholders with hosted official JSON, run capturePacket.verifyInputsShell, then run capturePacket.inspectOnlyShell." + ) + + if not plugin_available: + result["nextActions"].append( + "Install or restore the official bundled record-and-replay plugin cache, " + "or pass --allow-missing-official-plugin when preparing commands only." + ) + if coverage_errors: + result["nextActions"].append( + "Fix official fixture coverage errors before importing or pairing candidates: " + + " | ".join(str(error) for error in coverage_errors) + ) + if scenario_status["missing"]: + result["nextActions"].append( + f"Capture hosted official Record & Replay scenario {args.scenario!r}, then run commands.inspectOnly." + ) + elif scenario_status["notReady"]: + result["nextActions"].append( + f"Re-import or repair official scenario {args.scenario!r}; existing fixture does not pass readiness." + ) + else: + result["nextActions"].append( + f"Official scenario {args.scenario!r} is present; run commands.strictOfficialGoldenGate before release to refresh the strict summary artifact." + ) + + if missing_required: + result["nextActions"].append(f"Missing required official scenarios: {', '.join(missing_required)}.") + if not_ready: + result["nextActions"].append(f"Not-ready official scenarios: {', '.join(not_ready)}.") + if missing_recommended: + result["nextActions"].append( + f"Recommended follow-up official scenarios still missing: {', '.join(missing_recommended)}." + ) + + if args.require_ready and not scenario_status["ready"]: + result["ok"] = False + result["requireReady"] = True + else: + result["requireReady"] = False + + return 0 if result["ok"] else 1, result + + +def main() -> int: + parser = argparse.ArgumentParser( + description=( + "Prepare the next official Record & Replay successful recording golden capture. " + "This command is read-only: it reports current coverage and prints inspect/import commands." + ) + ) + parser.add_argument( + "--scenario", + default=DEFAULT_SCENARIO, + help=( + f"Official scenario to prepare. Defaults to {DEFAULT_SCENARIO}. " + f"Required scenarios: {DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument("--name", help="Fixture name to use in generated ingest commands.") + parser.add_argument( + "--fixture-root", + type=pathlib.Path, + default=DEFAULT_FIXTURE_ROOT, + help=f"Official recording fixture root. Defaults to {DEFAULT_FIXTURE_ROOT}.", + ) + parser.add_argument( + "--official-plugin-root", + type=pathlib.Path, + default=DEFAULT_OFFICIAL_PLUGIN_ROOT, + help=f"Bundled official record-and-replay plugin cache root. Defaults to {DEFAULT_OFFICIAL_PLUGIN_ROOT}.", + ) + parser.add_argument( + "--official-plugin-version", + default=DEFAULT_OFFICIAL_PLUGIN_VERSION, + help=f"Version string to write during import. Defaults to {DEFAULT_OFFICIAL_PLUGIN_VERSION!r}.", + ) + parser.add_argument( + "--status-json-placeholder", + default="", + help="Placeholder path used in generated inspect/import commands.", + ) + parser.add_argument( + "--status-json-base-dir-placeholder", + default=None, + help="Optional placeholder for --status-json-base-dir in generated commands.", + ) + parser.add_argument( + "--mcp-transcript-placeholder", + default="", + help="Placeholder path used when --include-transcript is enabled.", + ) + parser.add_argument( + "--include-transcript", + action="store_true", + default=True, + help="Include --mcp-transcript and --require-mcp-transcript-evidence in generated commands.", + ) + parser.add_argument( + "--no-include-transcript", + action="store_false", + dest="include_transcript", + help="Do not include transcript flags in generated inspect/import commands.", + ) + parser.add_argument( + "--require-coverage-on-import", + action="store_true", + help="Add --require-coverage to the generated import command.", + ) + parser.add_argument( + "--require-scenario", + action="append", + default=[], + help=( + "Required scenario to pass to coverage report. Defaults to " + f"{DEFAULT_REQUIRED_SCENARIO_HELP}." + ), + ) + parser.add_argument( + "--recommended-scenario", + action="append", + default=[], + help="Recommended scenario to pass to coverage report. Defaults to the coverage script defaults.", + ) + parser.add_argument( + "--allow-missing-official-plugin", + action="store_true", + help="Do not fail when the local official bundled plugin cache is absent.", + ) + parser.add_argument( + "--require-ready", + action="store_true", + help="Fail unless the selected scenario is already present and passes required readiness.", + ) + parser.add_argument( + "--capture-packet-dir", + type=pathlib.Path, + default=None, + help=( + "Write a local capture packet with scenario recipe, placeholder inputs, and inspect/import wrappers. " + "No official recording is started." + ), + ) + parser.add_argument( + "--capture-packet-recommended-scenarios", + action="store_true", + help=( + "With --capture-packet-dir, write one capture packet per recommended scenario. " + "No official recording is started." + ), + ) + args = parser.parse_args() + if not args.require_scenario: + args.require_scenario = list(DEFAULT_REQUIRED_SCENARIOS) + if not args.recommended_scenario: + args.recommended_scenario = list(DEFAULT_RECOMMENDED_SCENARIOS) + if args.capture_packet_recommended_scenarios: + exit_code, result = preflight_packet_set(args) + else: + exit_code, result = preflight(args) + output = json.dumps(result, indent=2, sort_keys=True) + if exit_code == 0: + print(output) + else: + print(output, file=sys.stderr) + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/probe-event-stream-recording.py b/scripts/probe-event-stream-recording.py new file mode 100755 index 00000000..e4d2e0f6 --- /dev/null +++ b/scripts/probe-event-stream-recording.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import select +import subprocess +import sys +import tempfile +import time +from typing import Any + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +DEFAULT_OFFICIAL_PLUGIN_DIR = ( + pathlib.Path.home() + / ".codex/plugins/cache/openai-bundled/record-and-replay/1.0.857" +) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Probe a Record & Replay event-stream MCP server and optionally run a " + "short start/stop recording cycle." + ) + ) + parser.add_argument( + "--target", + choices=["local", "official"], + default="official", + help="Which event-stream MCP server to probe.", + ) + parser.add_argument( + "--client", + help="Executable to launch. Defaults to local .build/debug/OpenComputerUse or the official bundled client.", + ) + parser.add_argument( + "--plugin-dir", + default=os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_PLUGIN_DIR"), + help="Official record-and-replay plugin directory.", + ) + parser.add_argument( + "--recordings-dir", + help="Recording output directory to pass through OPEN_COMPUTER_USE_EVENT_STREAM_DIR.", + ) + parser.add_argument( + "--start-stop", + action="store_true", + help="Call event_stream_start and then event_stream_stop. Without this flag only initialize/tools-list are probed.", + ) + parser.add_argument( + "--decline-elicitation", + action="store_true", + help="Deprecated alias for --elicitation-action decline.", + ) + parser.add_argument( + "--elicitation-action", + choices=["accept", "decline", "cancel"], + default="accept", + help="Action to return when the server sends elicitation/create.", + ) + parser.add_argument( + "--timeout", + type=float, + default=float(os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT", "10")), + help="Seconds to wait for each MCP response.", + ) + parser.add_argument( + "--output", + help="Optional path to write the full JSON probe result.", + ) + parser.add_argument( + "--fixture-output", + help="Optional path to write a sanitized, repo-stable probe fixture.", + ) + parser.add_argument( + "--official-plugin-version", + default="1.0.857", + help="Official Record & Replay plugin version to record in fixture output.", + ) + parser.add_argument( + "--allow-start-timeout", + action="store_true", + help="Return success when initialize/tools-list work but event_stream_start times out. Useful for documenting official raw-client host constraints.", + ) + args = parser.parse_args() + if args.decline_elicitation: + if args.elicitation_action != "accept": + parser.error("--decline-elicitation cannot be combined with --elicitation-action") + args.elicitation_action = "decline" + return args + + +def official_client(plugin_dir: pathlib.Path) -> pathlib.Path: + return ( + plugin_dir + / "Codex Computer Use.app/Contents/SharedSupport/SkyComputerUseClient.app/Contents/MacOS/SkyComputerUseClient" + ) + + +def launch_config(args: argparse.Namespace) -> tuple[list[str], pathlib.Path, dict[str, str], pathlib.Path]: + env = os.environ.copy() + recordings_dir = pathlib.Path(args.recordings_dir) if args.recordings_dir else pathlib.Path( + tempfile.mkdtemp(prefix="event-stream-probe-") + ) / "recordings" + env["OPEN_COMPUTER_USE_EVENT_STREAM_DIR"] = str(recordings_dir) + + if args.target == "local": + client = pathlib.Path(args.client) if args.client else REPO_ROOT / ".build/debug/OpenComputerUse" + env.setdefault("OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY", "1") + env.setdefault("OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL", "approve") + env.setdefault("OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS", "0") + env.setdefault("OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS", "never") + return [str(client), "event-stream", "mcp"], REPO_ROOT, env, recordings_dir + + plugin_dir = pathlib.Path(args.plugin_dir) if args.plugin_dir else DEFAULT_OFFICIAL_PLUGIN_DIR + client = pathlib.Path(args.client) if args.client else official_client(plugin_dir) + return [str(client), "event-stream", "mcp"], plugin_dir, env, recordings_dir + + +class MCPProbe: + def __init__(self, command: list[str], cwd: pathlib.Path, env: dict[str, str], timeout: float): + self.command = command + self.cwd = cwd + self.timeout = timeout + self.next_id = 1 + self.transcript: list[dict[str, Any]] = [] + self.proc = subprocess.Popen( + command, + cwd=str(cwd), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + def send(self, message: dict[str, Any]) -> None: + assert self.proc.stdin is not None + self.transcript.append({"direction": "send", "message": message}) + self.proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + self.proc.stdin.flush() + + def request(self, method: str, params: dict[str, Any] | None = None) -> int: + message_id = self.next_id + self.next_id += 1 + self.send( + { + "jsonrpc": "2.0", + "id": message_id, + "method": method, + "params": params or {}, + } + ) + return message_id + + def notify(self, method: str, params: dict[str, Any] | None = None) -> None: + self.send({"jsonrpc": "2.0", "method": method, "params": params or {}}) + + def read(self) -> dict[str, Any]: + assert self.proc.stdout is not None + ready, _, _ = select.select([self.proc.stdout], [], [], self.timeout) + if not ready: + message = {"timeout": self.timeout} + self.transcript.append({"direction": "receive", "message": message}) + return message + line = self.proc.stdout.readline() + if not line: + message = {"eof": True} + self.transcript.append({"direction": "receive", "message": message}) + return message + try: + message = json.loads(line) + except json.JSONDecodeError as error: + message = {"raw": line, "parseError": str(error)} + self.transcript.append({"direction": "receive", "message": message}) + return message + + def close(self) -> dict[str, Any]: + try: + if self.proc.stdin: + self.proc.stdin.close() + except BrokenPipeError: + pass + try: + self.proc.wait(timeout=3) + except subprocess.TimeoutExpired: + self.proc.kill() + self.proc.wait(timeout=3) + stderr = self.proc.stderr.read() if self.proc.stderr else "" + return {"returncode": self.proc.returncode, "stderr": stderr} + + +def decode_tool_text(result: dict[str, Any] | None) -> dict[str, Any] | None: + if not isinstance(result, dict): + return None + content = result.get("content") + if not isinstance(content, list): + return None + for item in content: + if not isinstance(item, dict) or item.get("type") != "text": + continue + text = item.get("text") + if not isinstance(text, str): + continue + try: + decoded = json.loads(text) + except json.JSONDecodeError: + continue + if isinstance(decoded, dict): + return decoded + return None + + +def sanitize_probe_value(value: Any) -> Any: + if isinstance(value, dict): + result: dict[str, Any] = {} + for key, child in value.items(): + if key in {"command", "cwd", "recordingsDir"}: + continue + if key == "text" and isinstance(child, str): + try: + decoded_text = json.loads(child) + except json.JSONDecodeError: + result[key] = child + else: + result["textJSON"] = sanitize_probe_value(decoded_text) + continue + if key in { + "eventsPath", + "metadataPath", + "sessionPath", + "suppressedEventsPath", + "currentSegmentEventsPath", + "currentSegmentMetadataPath", + }: + result[key] = f"" + continue + if key in {"sessionId", "sessionID"} and isinstance(child, str): + result[key] = "" + continue + if key.endswith("At") and isinstance(child, str): + result[key] = "" + continue + result[key] = sanitize_probe_value(child) + return result + if isinstance(value, list): + return [sanitize_probe_value(child) for child in value] + return value + + +def summarize_probe(result: dict[str, Any], official_plugin_version: str) -> dict[str, Any]: + initialize = result.get("initializeResponse", {}) + initialize_result = initialize.get("result", {}) if isinstance(initialize, dict) else {} + tools_response = result.get("toolsResponse", {}) + tools_result = tools_response.get("result", {}) if isinstance(tools_response, dict) else {} + tools = tools_result.get("tools", []) if isinstance(tools_result, dict) else [] + transcript = [] + elicitation_requests = [] + for entry in result.get("transcript", []): + if not isinstance(entry, dict): + continue + message = entry.get("message", {}) + if not isinstance(message, dict): + continue + if entry.get("direction") == "receive" and message.get("method") == "elicitation/create": + params = message.get("params") + if isinstance(params, dict): + requested_schema = params.get("requestedSchema") + schema_properties = ( + requested_schema.get("properties") + if isinstance(requested_schema, dict) + else None + ) + elicitation_requests.append( + { + "mode": params.get("mode"), + "message": params.get("message"), + "requestedSchemaType": requested_schema.get("type") + if isinstance(requested_schema, dict) + else None, + "requestedSchemaPropertyNames": sorted(schema_properties.keys()) + if isinstance(schema_properties, dict) + else None, + "requestedSchemaRequired": requested_schema.get("required") + if isinstance(requested_schema, dict) + else None, + "requestedSchemaHasAdditionalProperties": "additionalProperties" + in requested_schema + if isinstance(requested_schema, dict) + else None, + } + ) + transcript.append( + { + "direction": entry.get("direction"), + "id": message.get("id"), + "method": message.get("method"), + "hasResult": "result" in message, + "hasError": "error" in message, + "timeout": message.get("timeout"), + "eof": message.get("eof"), + } + ) + return { + "fixtureFormatVersion": 1, + "kind": "record-and-replay-event-stream-probe", + "source": result.get("target"), + "officialPluginVersion": official_plugin_version if result.get("target") == "official" else None, + "ok": result.get("ok"), + "startStopRequested": result.get("startStopRequested"), + "startStopCompleted": result.get("startStopCompleted"), + "startTimedOut": result.get("startTimedOut"), + "serverInfo": sanitize_probe_value(initialize_result.get("serverInfo")), + "protocolVersion": initialize_result.get("protocolVersion"), + "toolNames": [tool.get("name") for tool in tools if isinstance(tool, dict)], + "startResponseShape": sanitize_probe_value(result.get("startResponse")), + "repeatStartResponseShape": sanitize_probe_value(result.get("repeatStartResponse")), + "statusResponseShape": sanitize_probe_value(result.get("statusResponse")), + "stopResponseShape": sanitize_probe_value(result.get("stopResponse")), + "repeatStopResponseShape": sanitize_probe_value(result.get("repeatStopResponse")), + "finalStatusResponseShape": sanitize_probe_value(result.get("finalStatusResponse")), + "elicitationRequests": sanitize_probe_value(elicitation_requests), + "process": sanitize_probe_value(result.get("process")), + "transcriptShape": transcript, + } + + +def wait_for_response( + probe: MCPProbe, + response_id: int, + elicitation_action: str, +) -> dict[str, Any]: + while True: + message = probe.read() + if message.get("method") == "elicitation/create": + content = {} if elicitation_action == "accept" else None + probe.send( + { + "jsonrpc": "2.0", + "id": message["id"], + "result": {"action": elicitation_action, "content": content}, + } + ) + continue + if message.get("id") == response_id: + return message + if "timeout" in message or "eof" in message or "parseError" in message: + return message + + +def probe_server(args: argparse.Namespace) -> dict[str, Any]: + command, cwd, env, recordings_dir = launch_config(args) + result: dict[str, Any] = { + "ok": False, + "target": args.target, + "command": command, + "cwd": str(cwd), + "recordingsDir": str(recordings_dir), + "startStopRequested": args.start_stop, + } + client_path = pathlib.Path(command[0]) + if not client_path.exists(): + result.update({"error": "clientNotFound", "client": str(client_path)}) + return result + + probe = MCPProbe(command, cwd, env, args.timeout) + elicitation_action = args.elicitation_action + try: + initialize_id = probe.request( + "initialize", + { + "protocolVersion": "2025-11-25", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "event-stream-recording-probe", "version": "0"}, + }, + ) + initialize = wait_for_response(probe, initialize_id, elicitation_action) + result["initializeResponse"] = initialize + if "result" not in initialize: + return result + + probe.notify("notifications/initialized") + tools_id = probe.request("tools/list") + tools = wait_for_response(probe, tools_id, elicitation_action) + result["toolsResponse"] = tools + + if args.start_stop: + start_id = probe.request( + "tools/call", + {"name": "event_stream_start", "arguments": {}}, + ) + start = wait_for_response(probe, start_id, elicitation_action) + result["startResponse"] = start + decoded_start = decode_tool_text(start.get("result") if isinstance(start, dict) else None) + if decoded_start: + result["startStatus"] = decoded_start + + if "result" in start: + repeat_start_id = probe.request( + "tools/call", + {"name": "event_stream_start", "arguments": {}}, + ) + repeat_start = wait_for_response( + probe, + repeat_start_id, + elicitation_action, + ) + result["repeatStartResponse"] = repeat_start + decoded_repeat_start = decode_tool_text( + repeat_start.get("result") if isinstance(repeat_start, dict) else None + ) + if decoded_repeat_start: + result["repeatStartStatus"] = decoded_repeat_start + + should_try_stop = "result" in start or "timeout" in start + if should_try_stop: + # Give the recorder a moment to flush session.started before the stop request. + time.sleep(0.2) + status_id = probe.request( + "tools/call", + {"name": "event_stream_status", "arguments": {}}, + ) + status = wait_for_response(probe, status_id, elicitation_action) + result["statusResponse"] = status + decoded_status = decode_tool_text(status.get("result") if isinstance(status, dict) else None) + if decoded_status: + result["statusStatus"] = decoded_status + + stop_id = probe.request( + "tools/call", + {"name": "event_stream_stop", "arguments": {}}, + ) + stop = wait_for_response(probe, stop_id, elicitation_action) + result["stopResponse"] = stop + decoded_stop = decode_tool_text(stop.get("result") if isinstance(stop, dict) else None) + if decoded_stop: + result["stopStatus"] = decoded_stop + + if "result" in stop: + repeat_stop_id = probe.request( + "tools/call", + {"name": "event_stream_stop", "arguments": {}}, + ) + repeat_stop = wait_for_response( + probe, + repeat_stop_id, + elicitation_action, + ) + result["repeatStopResponse"] = repeat_stop + decoded_repeat_stop = decode_tool_text( + repeat_stop.get("result") + if isinstance(repeat_stop, dict) + else None + ) + if decoded_repeat_stop: + result["repeatStopStatus"] = decoded_repeat_stop + + final_status_id = probe.request( + "tools/call", + {"name": "event_stream_status", "arguments": {}}, + ) + final_status = wait_for_response( + probe, + final_status_id, + elicitation_action, + ) + result["finalStatusResponse"] = final_status + decoded_final_status = decode_tool_text( + final_status.get("result") + if isinstance(final_status, dict) + else None + ) + if decoded_final_status: + result["finalStatusStatus"] = decoded_final_status + + surface_ok = "result" in result.get("initializeResponse", {}) and "result" in result.get( + "toolsResponse", {} + ) + result["startStopCompleted"] = False + result["startTimedOut"] = False + if args.start_stop: + start_response = result.get("startResponse", {}) + stop_response = result.get("stopResponse", {}) + result["startStopCompleted"] = "result" in start_response and "result" in stop_response + result["startTimedOut"] = "timeout" in start_response + result["ok"] = surface_ok and ( + result["startStopCompleted"] + or (args.allow_start_timeout and result["startTimedOut"]) + ) + else: + result["ok"] = surface_ok + finally: + result["process"] = probe.close() + result["transcript"] = probe.transcript + return result + + +def main() -> int: + args = parse_args() + result = probe_server(args) + output = json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False) + if args.output: + pathlib.Path(args.output).write_text(output + "\n") + if args.fixture_output: + fixture = summarize_probe(result, args.official_plugin_version) + fixture_path = pathlib.Path(args.fixture_output) + fixture_path.parent.mkdir(parents=True, exist_ok=True) + fixture_path.write_text( + json.dumps(fixture, indent=2, sort_keys=True, ensure_ascii=False) + "\n" + ) + print(output) + return 0 if result.get("ok") else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/record_and_replay_baseline_contract.py b/scripts/record_and_replay_baseline_contract.py new file mode 100644 index 00000000..3b186037 --- /dev/null +++ b/scripts/record_and_replay_baseline_contract.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 + +REQUIRED_BASELINE_CHECKS = ( + "baseline-contract-smoke", + "event-stream-smoke-matrix", + "event-stream-screenshot-context-smoke", + "event-stream-action-smoke", + "official-surface-compare", + "official-no-active-response-compare", + "official-raw-start-timeout-fixture-smoke", + "official-fixture-set-smoke", + "official-fixture-coverage-report", + "official-fixture-ingest-smoke", + "ocu-candidate-ingest-smoke", + "official-golden-capture-preflight-smoke", + "ocu-candidate-pairing-preflight-smoke", + "baseline-audit-targets-smoke", + "standalone-skill-repo-smoke", + "npm-staged-skill-repo-smoke", +) + +STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS = ( + "checkedGeneratedRepoSelfCheck", + "checkedManifestContract", + "checkedReadmeHandoffContract", + "checkedReadmeOfficialEvidenceHandoff", + "checkedReadmeOfficialGoldenGap", + "checkedReadmeWaitNotifyBoundary", + "checkedGeneratedReadmePrerequisites", + "checkedGeneratedReadmeScenarioList", + "checkedOfficialEvidenceManifest", + "checkedOfficialEvidenceScenarioManifest", + "checkedOfficialEvidencePreflightManifest", + "checkedOfficialEvidenceAuditManifest", + "checkedOfficialFixtureSetComparePolicyManifest", + "checkedSourceBaselineSummaryEvidence", + "checkedSourceBaselineSummaryOfficialGoldenState", + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff", + "checkedSourceBaselineSummaryCapturePacketSetContractManifest", + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff", + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff", + "checkedPackageArtifact", + "checkedRuntimeContract", + "checkedRuntimeTimeoutDiagnostics", + "checkedInitializeSurfaceContract", + "checkedToolMetadataContract", + "checkedToolInputSchemaNoArguments", + "checkedNoActiveStatusStop", + "checkedRequiresObjectParams", + "checkedRequiresStringToolName", + "checkedRequiresObjectArguments", + "checkedRejectsUnexpectedArguments", + "checkedRejectsNonObjectArguments", + "checkedRejectedRequestsDoNotCreateSessionFiles", + "checkedStatusNotUsedAsWaitLoopGuard", + "checkedMcpNoDirectEventContentsGuard", + "checkedWaitNotifyContract", + "checkedNotifySuppressedEventsPathEnv", + "checkedNotifyCallbackFailureExit", + "checkedNotifyCallbackFailureReason", + "checkedNotifyCallbackTimeoutFailureExit", + "checkedNotifyCallbackTimeoutReason", + "checkedRecordingToSkillHandoff", + "checkedStrictValidation", + "checkedDeclaredHandoffPaths", + "checkedScreenshotPathContainment", + "checkedEventsOnlyValidation", + "checkedScaffoldSkill", + "checkedScaffoldSkillFailureExit", + "checkedSkillCreatorHandoff", + "checkedCancelledRecordingContract", + "checkedCancelledRecordingRejected", + "checkedLifecycleSmoke", + "checkedOneActive", + "checkedIdempotentStop", + "checkedFinalStatus", +) + +NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS = ( + "checkedNpmLauncher", + "checkedNpmPythonLauncherDiagnostics", + "checkedStandaloneRepoScaffold", + "checkedManifestContract", + "checkedReadmeHandoffContract", + "checkedReadmeOfficialEvidenceHandoff", + "checkedReadmeOfficialGoldenGap", + "checkedReadmeWaitNotifyBoundary", + "checkedGeneratedReadmePrerequisites", + "checkedGeneratedReadmeScenarioList", + "checkedOfficialEvidenceManifest", + "checkedOfficialEvidenceScenarioManifest", + "checkedOfficialEvidencePreflightManifest", + "checkedOfficialEvidenceAuditManifest", + "checkedOfficialFixtureSetComparePolicyManifest", + "checkedSourceBaselineSummaryEvidence", + "checkedSourceBaselineSummaryOfficialGoldenState", + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff", + "checkedSourceBaselineSummaryCapturePacketSetContractManifest", + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff", + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff", + "checkedPackageArtifact", + "checkedGeneratedRepoSelfCheck", + "checkedRuntimeTimeoutDiagnostics", + "checkedInitializeSurfaceContract", + "checkedToolMetadataContract", + "checkedToolInputSchemaNoArguments", + "checkedNoActiveStatusStop", + "checkedRequiresObjectParams", + "checkedRequiresStringToolName", + "checkedRequiresObjectArguments", + "checkedRejectsUnexpectedArguments", + "checkedRejectsNonObjectArguments", + "checkedRejectedRequestsDoNotCreateSessionFiles", + "checkedSkillWorkflow", + "checkedStatusNotUsedAsWaitLoopGuard", + "checkedMcpNoDirectEventContentsGuard", + "checkedWaitNotifyContract", + "checkedNotifySuppressedEventsPathEnv", + "checkedNotifyCallbackFailureExit", + "checkedNotifyCallbackFailureReason", + "checkedNotifyCallbackTimeoutFailureExit", + "checkedNotifyCallbackTimeoutReason", + "checkedSkillPackaging", + "checkedRecordingToSkillManifestContract", + "checkedCancelledRecordingContract", + "checkedStrictValidation", + "checkedDeclaredHandoffPaths", + "checkedScreenshotPathContainment", + "checkedEventsOnlyValidation", + "checkedScaffoldSkill", + "checkedScaffoldSkillFailureExit", + "checkedCancelledRecordingRejected", + "checkedRecordingToSkillHandoff", + "checkedSkillCreatorHandoff", +) + +STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS = ( + "checkedGeneratedRepoSelfCheck", + "checkedManifestContract", + "checkedReadmeHandoffContract", + "checkedReadmeOfficialEvidenceHandoff", + "checkedReadmeOfficialGoldenGap", + "checkedReadmeWaitNotifyBoundary", + "checkedGeneratedReadmePrerequisites", + "checkedGeneratedReadmeScenarioList", + "checkedOfficialEvidenceManifest", + "checkedOfficialEvidenceScenarioManifest", + "checkedOfficialEvidencePreflightManifest", + "checkedOfficialEvidenceAuditManifest", + "checkedOfficialFixtureSetComparePolicyManifest", + "checkedSourceBaselineSummaryEvidence", + "checkedSourceBaselineSummaryOfficialGoldenState", + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff", + "checkedSourceBaselineSummaryCapturePacketSetContractManifest", + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow", + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier", + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff", + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff", + "checkedPackageArtifact", + "checkedRuntimeContract", + "checkedRuntimeTimeoutDiagnostics", + "checkedInitializeSurfaceContract", + "checkedToolMetadataContract", + "checkedToolInputSchemaNoArguments", + "checkedNoActiveStatusStop", + "checkedRequiresObjectParams", + "checkedRequiresStringToolName", + "checkedRequiresObjectArguments", + "checkedRejectsUnexpectedArguments", + "checkedRejectsNonObjectArguments", + "checkedRejectedRequestsDoNotCreateSessionFiles", + "checkedStatusNotUsedAsWaitLoopGuard", + "checkedMcpNoDirectEventContentsGuard", + "checkedWaitNotifyContract", + "checkedNotifySuppressedEventsPathEnv", + "checkedNotifyCallbackFailureExit", + "checkedNotifyCallbackFailureReason", + "checkedNotifyCallbackTimeoutFailureExit", + "checkedNotifyCallbackTimeoutReason", + "checkedRecordingToSkillHandoff", + "checkedStrictValidation", + "checkedDeclaredHandoffPaths", + "checkedScreenshotPathContainment", + "checkedEventsOnlyValidation", + "checkedScaffoldSkill", + "checkedScaffoldSkillFailureExit", + "checkedSkillCreatorHandoff", + "checkedCancelledRecordingContract", + "checkedCancelledRecordingRejected", + "checkedLifecycleSmoke", + "checkedLifecycleOneActive", + "checkedLifecycleIdempotentStop", + "checkedLifecycleFinalStatus", +) + +NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS = NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS diff --git a/scripts/record_and_replay_scenarios.py b/scripts/record_and_replay_scenarios.py new file mode 100644 index 00000000..c9ef746a --- /dev/null +++ b/scripts/record_and_replay_scenarios.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from copy import deepcopy +from typing import Any + + +DEFAULT_REQUIRED_SCENARIOS = ("simple-action-stop",) +DEFAULT_RECOMMENDED_SCENARIOS = ( + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", +) + +ACTION_SMOKE_SCENARIOS = {"simple-action-stop", "drag-stop"} + +BASE_GOLDEN_READINESS_ARGS = ( + "--require-fixture-manifest", + "--require-session-alias", + "--require-metadata-counts", + "--require-handoff-paths", +) + +COMPLETED_MCP_RESPONSE_SHAPES = ( + "startResponseShape", + "statusResponseShape", + "stopResponseShape", + "finalStatusResponseShape", +) + +_SCENARIO_RECIPES: dict[str, dict[str, Any]] = { + "simple-action-stop": { + "scenario": "simple-action-stop", + "priority": "required", + "captureGoal": "Record one low-risk left click and finish with the Record & Replay Done control.", + "userAction": "Click a stable, harmless UI target once, then click Done in the official recording controls.", + "expectedActionEvents": ["mouse.click"], + "expectedEndReason": "recording_controls_stopped", + "expectedEvidence": [ + "event_stream_start/status/stop hosted MCP response shape", + "metadata/session/events/suppressed handoff paths", + "one session.started as the first event", + "one session.ended as the final event", + "mouse.click action event", + "AX payload for the clicked target/window", + ], + "ocuCandidateSourceKind": "run-action-smoke", + "notes": [ + "Use this as the minimum official successful recording golden fixture.", + "Keep the clicked target harmless and easy to describe after redaction.", + ], + }, + "keyboard-input-stop": { + "scenario": "keyboard-input-stop", + "priority": "recommended", + "captureGoal": "Record a small text input and finish with the Record & Replay Done control.", + "userAction": "Type a short non-sensitive string into a harmless text field, then click Done.", + "expectedActionEvents": ["keyboard.text_input"], + "expectedEndReason": "recording_controls_stopped", + "expectedEvidence": [ + "keyboard.text_input action event", + "focusedAccessibilityElement or equivalent keyboard target evidence", + "AX payload for the edited field/window", + "completed hosted MCP response shape", + ], + "ocuCandidateSourceKind": "recording-required", + "notes": [ + "Do not use synthetic --run-action-smoke for this scenario; macOS may filter synthetic keyboard events.", + "Use non-sensitive placeholder text only.", + ], + }, + "drag-stop": { + "scenario": "drag-stop", + "priority": "recommended", + "captureGoal": "Record one low-risk drag gesture and finish with the Record & Replay Done control.", + "userAction": "Drag a harmless target a short distance, release it, then click Done.", + "expectedActionEvents": ["mouse.drag"], + "expectedEndReason": "recording_controls_stopped", + "expectedEvidence": [ + "mouse.drag action event", + "start/end location evidence", + "targetAccessibilityElement or equivalent drag target evidence", + "AX payload for the affected window", + "completed hosted MCP response shape", + ], + "ocuCandidateSourceKind": "run-action-smoke", + "notes": [ + "Use a fixture or harmless UI target where drag has no external side effects.", + ], + }, + "cancel": { + "scenario": "cancel", + "priority": "recommended", + "captureGoal": "Record the official cancellation path.", + "userAction": "Start recording, optionally perform no action, then click Discard/Cancel in the recording controls.", + "expectedActionEvents": [], + "expectedEndReason": "recording_controls_cancelled", + "expectedEvidence": [ + "session.ended endReason=recording_controls_cancelled", + "cancelled hosted MCP/status response shape when available", + "no skill creation from the cancelled recording", + ], + "ocuCandidateSourceKind": "recording-required", + "notes": [ + "Cancelled recordings are evidence for lifecycle semantics only and must not be used to scaffold a skill.", + ], + }, + "timeout": { + "scenario": "timeout", + "priority": "recommended", + "captureGoal": "Record the official maximum-duration timeout path.", + "userAction": "Start recording and let the official time limit expire without using Done or Discard.", + "expectedActionEvents": [], + "expectedEndReason": None, + "expectedEvidence": [ + "one session.started as the first event", + "one session.ended as the final event", + "official timeout endReason once observed", + "completed or timeout hosted MCP/status response shape", + ], + "ocuCandidateSourceKind": "recording-required", + "notes": [ + "OCU currently uses recording_time_limit_reached as a baseline; official timeout endReason still needs calibration.", + "This sample may be slow because the official limit is expected to be long.", + ], + }, +} + + +def scenario_recipe(scenario: str) -> dict[str, Any]: + recipe = _SCENARIO_RECIPES.get(scenario) + if recipe is None: + return { + "scenario": scenario, + "priority": "custom", + "captureGoal": "Capture and document this custom Record & Replay scenario before importing it.", + "userAction": "Use a minimal, low-risk demonstration and finish through the official recording controls.", + "expectedActionEvents": [], + "expectedEndReason": None, + "expectedEvidence": [ + "hosted MCP response shape", + "metadata/session/events/suppressed handoff paths", + "session.started and session.ended lifecycle evidence", + "AX payload when the scenario includes UI interaction", + ], + "ocuCandidateSourceKind": "recording-required", + "notes": [ + "No scenario-specific readiness policy is encoded for this custom scenario.", + ], + } + return deepcopy(recipe) + + +def scenario_readiness_args( + scenario: str, + *, + source: str, + require_mcp_transcript_evidence: bool = False, +) -> list[str]: + recipe = scenario_recipe(scenario) + args = [ + *BASE_GOLDEN_READINESS_ARGS, + "--require-source", + source, + ] + + expected_action_events = recipe.get("expectedActionEvents") + if isinstance(expected_action_events, list): + for event_type in expected_action_events: + if isinstance(event_type, str) and event_type: + args.extend(["--require-event-type", event_type]) + + expected_end_reason = recipe.get("expectedEndReason") + if isinstance(expected_end_reason, str) and expected_end_reason: + args.extend(["--require-end-reason", expected_end_reason]) + + scenario_name = recipe.get("scenario") + if scenario_name in {"simple-action-stop", "keyboard-input-stop", "drag-stop"}: + args.append("--require-full-ax-payload") + elif scenario_name in {"cancel", "timeout"}: + args.extend(["--allow-no-action", "--allow-no-ax-payload"]) + + if require_mcp_transcript_evidence: + args.append("--require-mcp-transcript") + if scenario_name in {"simple-action-stop", "keyboard-input-stop", "drag-stop"}: + for response_shape in COMPLETED_MCP_RESPONSE_SHAPES: + args.extend(["--require-mcp-response-shape", response_shape]) + + return args diff --git a/scripts/run-event-stream-smoke-matrix.sh b/scripts/run-event-stream-smoke-matrix.sh new file mode 100755 index 00000000..b4eebd3e --- /dev/null +++ b/scripts/run-event-stream-smoke-matrix.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +run_step() { + local name="$1" + shift + echo "==> ${name}" >&2 + "$@" +} + +run_step "event-stream default MCP lifecycle" ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream no-active status/stop" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1 ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream maximum-duration timeout" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT=1 ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream wait timeout" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT=1 ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream approval denied" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=deny ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream approval cancelled" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL=cancel ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream MCP elicitation approval" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1 ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream app-agent wait/notify" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT=1 ./scripts/run-event-stream-smoke-tests.sh +run_step "event-stream fixture import" ./scripts/test-event-stream-fixture-import.py +run_step "event-stream recording compare" ./scripts/test-event-stream-recording-compare.py + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_SCREENSHOTS:-0}" == "1" ]]; then + run_step "event-stream screenshot context" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS=1 ./scripts/run-event-stream-smoke-tests.sh +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_MATRIX_OFFICIAL:-0}" == "1" ]]; then + run_step "event-stream official surface compare" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1 ./scripts/run-event-stream-smoke-tests.sh +fi + +echo '{"ok":true,"matrix":"event-stream"}' diff --git a/scripts/run-event-stream-smoke-tests.sh b/scripts/run-event-stream-smoke-tests.sh new file mode 100755 index 00000000..e3f366e6 --- /dev/null +++ b/scripts/run-event-stream-smoke-tests.sh @@ -0,0 +1,1803 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +if [[ -n "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR:-}" ]]; then + tmpdir="${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TMPDIR}" + mkdir -p "${tmpdir}" + cleanup_tmpdir=0 +else + tmpdir="$(mktemp -d)" + cleanup_tmpdir=1 +fi +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_KEEP_TMP:-0}" == "1" ]]; then + cleanup_tmpdir=0 +fi +if [[ "${cleanup_tmpdir}" == "1" ]]; then + trap 'rm -rf "${tmpdir}"' EXIT +fi + +cd "${repo_root}" + +swift build --product OpenComputerUse >/dev/null + +messages_file="${tmpdir}/messages.jsonl" +cat >"${messages_file}" <<'JSONL' +{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-11-25","capabilities":{},"clientInfo":{"name":"event-stream-smoke","version":"0"}}} +{"jsonrpc":"2.0","method":"notifications/initialized","params":{}} +{"jsonrpc":"2.0","id":2,"method":"tools/list","params":{}} +{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}} +{"jsonrpc":"2.0","id":4,"method":"tools/call","params":{"name":"event_stream_start","arguments":{}}} +{"jsonrpc":"2.0","id":5,"method":"tools/call","params":{"name":"event_stream_status","arguments":{}}} +{"jsonrpc":"2.0","id":6,"method":"tools/call","params":{"name":"event_stream_stop","arguments":{}}} +{"jsonrpc":"2.0","id":7,"method":"tools/call","params":{"name":"event_stream_stop","arguments":{}}} +{"jsonrpc":"2.0","id":8,"method":"tools/call","params":{"name":"event_stream_status","arguments":{}}} +JSONL + +screenshot_policy="never" +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS:-0}" == "1" ]]; then + screenshot_policy="always" +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS:-0}" == "1" ]]; then + python3 - "${tmpdir}" "${screenshot_policy}" <<'PY' +import json +import os +import pathlib +import socket +import subprocess +import sys +import time + +root = pathlib.Path(sys.argv[1]) +screenshot_policy = sys.argv[2] +recordings = root / "recordings" +socket_path = root / "open-computer-use-agent.sock" +mcp_transcript_path = root / "action-mcp-transcript.json" +action_scenario = os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO", "mixed-action-stop") +supported_action_scenarios = { + "mixed-action-stop", + "simple-action-stop", + "drag-stop", +} +assert action_scenario in supported_action_scenarios, action_scenario + + +def start_process(command, env=None): + return subprocess.Popen( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + + +agent = start_process([".build/debug/OpenComputerUse", "__open-computer-use-app-agent", str(socket_path)]) + + +def request(payload, persistent_socket=None): + data = (json.dumps(payload, separators=(",", ":")) + "\n").encode() + if persistent_socket is not None: + persistent_socket.sendall(data) + chunks = [] + while True: + chunk = persistent_socket.recv(65536) + assert chunk, "app-agent closed persistent connection before newline response" + if b"\n" in chunk: + before, _sep, _after = chunk.partition(b"\n") + chunks.append(before) + break + chunks.append(chunk) + response = json.loads(b"".join(chunks).decode()) + if "error" in response: + raise AssertionError(response["error"]) + return response + + deadline = time.time() + 10 + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + while True: + try: + client.connect(str(socket_path)) + break + except OSError as error: + if time.time() >= deadline: + raise AssertionError(f"failed to connect to app-agent socket: {error}") + time.sleep(0.05) + client.sendall(data) + chunks = [] + while True: + chunk = client.recv(65536) + assert chunk, "app-agent closed connection before newline response" + if b"\n" in chunk: + before, _sep, _after = chunk.partition(b"\n") + chunks.append(before) + break + chunks.append(chunk) + response = json.loads(b"".join(chunks).decode()) + if "error" in response: + raise AssertionError(response["error"]) + return response + + +def cli(arguments, environment=None): + response = request({ + "kind": "cli", + "arguments": arguments, + "environment": environment or {}, + }) + assert response["exitCode"] == 0, response + stdout = response["stdout"].strip() + assert stdout, response + return json.loads(stdout) + + +mcp_next_id = 1 +mcp_transcript = [] +mcp_response_shapes = {} + + +def mcp_exchange(message, environment=None, expect_response=True): + line = json.dumps(message, separators=(",", ":")) + mcp_transcript.append({"direction": "send", "message": message}) + response = request({ + "kind": "eventStreamMCP", + "line": line, + "environment": environment or {}, + }).get("response") + if response is None: + assert not expect_response, message + return None + parsed = json.loads(response) + mcp_transcript.append({"direction": "receive", "message": parsed}) + return parsed + + +def mcp_request(method, params=None, environment=None, expect_response=True): + global mcp_next_id + message = { + "jsonrpc": "2.0", + "id": mcp_next_id, + "method": method, + } + mcp_next_id += 1 + if params is not None: + message["params"] = params + return mcp_exchange(message, environment=environment, expect_response=expect_response) + + +def mcp_notify(method, params=None, environment=None): + message = { + "jsonrpc": "2.0", + "method": method, + } + if params is not None: + message["params"] = params + return mcp_exchange(message, environment=environment, expect_response=False) + + +def decode_mcp_tool_text(response): + assert isinstance(response, dict), response + assert "result" in response, response + content = response["result"].get("content", []) + assert content, response + text = content[0].get("text") + assert isinstance(text, str) and text.strip(), response + return json.loads(text) + + +def mcp_tool_call(tool_name, shape_key, environment=None): + response = mcp_request( + "tools/call", + {"name": tool_name, "arguments": {}}, + environment=environment, + ) + mcp_response_shapes[shape_key] = response + return decode_mcp_tool_text(response) + + +def write_mcp_transcript(): + payload = { + "target": "local-open-computer-use-action-smoke", + "transcript": mcp_transcript, + **mcp_response_shapes, + } + mcp_transcript_path.write_text(json.dumps(payload, indent=2, sort_keys=True) + "\n") + + +try: + deadline = time.time() + 10 + while not socket_path.exists(): + if time.time() >= deadline: + raise AssertionError("timed out waiting for app-agent socket") + time.sleep(0.05) + + start_env = { + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": screenshot_policy, + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + if action_scenario != "mixed-action-stop": + start_env["OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS"] = "0" + initialized = mcp_request( + "initialize", + { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-action-smoke", "version": "0"}, + }, + environment=start_env, + ) + assert initialized["result"]["serverInfo"]["name"] == "Record & Replay", initialized + mcp_notify("notifications/initialized", {}, environment=start_env) + listed = mcp_request("tools/list", {}, environment=start_env) + assert [tool["name"] for tool in listed["result"]["tools"]] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ], listed + + started = mcp_tool_call("event_stream_start", "startResponseShape", environment=start_env) + session_id = started["sessionId"] + assert started["state"] == "recording" + repeated_start = mcp_tool_call("event_stream_start", "repeatStartResponseShape", environment=start_env) + assert repeated_start["sessionId"] == session_id + status = mcp_tool_call("event_stream_status", "statusResponseShape", environment=start_env) + assert status["sessionId"] == session_id + assert status["state"] == "recording" + + subprocess.run([ + "swift", + "-e", + """ +import CoreGraphics +import Darwin +import Foundation +let point = CGEvent(source: nil)?.location ?? CGPoint(x: 240, y: 240) +guard let source = CGEventSource(stateID: .hidSystemState) else { + exit(2) +} +let scenario = ProcessInfo.processInfo.environment["OCU_ACTION_SCENARIO"] ?? "mixed-action-stop" + +func postClick() { + guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left), + let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: point, mouseButton: .left) else { + exit(2) + } + down.setIntegerValueField(.mouseEventClickState, value: 1) + up.setIntegerValueField(.mouseEventClickState, value: 1) + down.post(tap: .cghidEventTap) + usleep(50_000) + up.post(tap: .cghidEventTap) + usleep(50_000) +} + +func postScroll() { + guard let scroll = CGEvent(scrollWheelEvent2Source: source, units: .line, wheelCount: 2, wheel1: -4, wheel2: 0, wheel3: 0) else { + exit(2) + } + scroll.post(tap: .cghidEventTap) + usleep(50_000) +} + +func postShortcut() { + for _ in 0..<2 { + guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: 79, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: source, virtualKey: 79, keyDown: false) else { + exit(2) + } + keyDown.flags = [.maskShift] + keyUp.flags = [.maskShift] + keyDown.post(tap: .cgSessionEventTap) + usleep(100_000) + keyUp.post(tap: .cgSessionEventTap) + usleep(150_000) + } +} + +func postTextInput() { + for _ in 0..<3 { + guard let keyDown = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: true), + let keyUp = CGEvent(keyboardEventSource: source, virtualKey: 0, keyDown: false) else { + exit(2) + } + keyDown.flags = [] + keyUp.flags = [] + var text = Array("a".utf16) + text.withUnsafeMutableBufferPointer { buffer in + guard let baseAddress = buffer.baseAddress else { + exit(2) + } + keyDown.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: baseAddress) + keyUp.keyboardSetUnicodeString(stringLength: buffer.count, unicodeString: baseAddress) + } + keyDown.post(tap: .cghidEventTap) + usleep(100_000) + keyUp.post(tap: .cghidEventTap) + usleep(150_000) + } +} + +func postDrag() { + let end = CGPoint(x: point.x + 48, y: point.y + 36) + let mid = CGPoint(x: point.x + 24, y: point.y + 18) + guard let down = CGEvent(mouseEventSource: source, mouseType: .leftMouseDown, mouseCursorPosition: point, mouseButton: .left), + let dragged = CGEvent(mouseEventSource: source, mouseType: .leftMouseDragged, mouseCursorPosition: mid, mouseButton: .left), + let draggedEnd = CGEvent(mouseEventSource: source, mouseType: .leftMouseDragged, mouseCursorPosition: end, mouseButton: .left), + let up = CGEvent(mouseEventSource: source, mouseType: .leftMouseUp, mouseCursorPosition: end, mouseButton: .left) else { + exit(2) + } + down.post(tap: .cghidEventTap) + usleep(50_000) + dragged.post(tap: .cghidEventTap) + usleep(50_000) + draggedEnd.post(tap: .cghidEventTap) + usleep(50_000) + up.post(tap: .cghidEventTap) + usleep(50_000) +} + +switch scenario { +case "mixed-action-stop": + postClick() + postScroll() +case "simple-action-stop": + postClick() +case "drag-stop": + postDrag() +default: + exit(3) +} +""", + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OCU_ACTION_SCENARIO": action_scenario, + }) + + time.sleep(0.5) + stopped = mcp_tool_call("event_stream_stop", "stopResponseShape", environment=start_env) + assert stopped["sessionId"] == session_id + assert stopped["state"] == "stopped" + repeated_stop = mcp_tool_call("event_stream_stop", "repeatStopResponseShape", environment=start_env) + assert repeated_stop["sessionId"] == session_id + final_status = mcp_tool_call("event_stream_status", "finalStatusResponseShape", environment=start_env) + assert final_status["sessionId"] == session_id + assert final_status["state"] == "stopped" + required_mcp_response_shape_keys = { + "startResponseShape", + "repeatStartResponseShape", + "statusResponseShape", + "stopResponseShape", + "repeatStopResponseShape", + "finalStatusResponseShape", + } + missing_response_shape_keys = sorted(required_mcp_response_shape_keys - set(mcp_response_shapes)) + assert not missing_response_shape_keys, { + "missingResponseShapes": missing_response_shape_keys, + "mcpResponseShapes": sorted(mcp_response_shapes), + } + checked_mcp_response_shapes_captured = True + write_mcp_transcript() + events_path = pathlib.Path(stopped["eventsPath"]) + handoff_metadata_path = pathlib.Path(stopped["metadataPath"]) + session_dir = pathlib.Path(stopped.get("sessionDirectoryPath") or handoff_metadata_path.parent) + metadata_path = session_dir / "metadata.json" + events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if line.strip() + ] + event_types = [event.get("type") for event in events] + required_event_types = { + "mixed-action-stop": {"mouse.click"}, + "simple-action-stop": {"mouse.click"}, + "drag-stop": {"mouse.drag"}, + }[action_scenario] + for required_event_type in required_event_types: + assert required_event_type in event_types, event_types + assert "AX.focusedWindowChanged" in event_types, event_types + validation_required_event_types = [ + "AX.focusedWindowChanged", + "session.ended", + *sorted(required_event_types), + ] + + validate_command = [ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(metadata_path), + "--strict-ocu", + "--require-skill-draft", + ] + for required_event_type in validation_required_event_types: + validate_command.extend(["--require-event-type", required_event_type]) + subprocess.run(validate_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + runtime_validate_command = [ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-skill-draft", + str(metadata_path), + ] + for required_event_type in validation_required_event_types: + runtime_validate_command[-1:-1] = ["--require-event-type", required_event_type] + subprocess.run(runtime_validate_command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }) + subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "summarize", + "--json", + "--require-action", + str(metadata_path), + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }) + skill_dir = root / "generated-action-skill" + scaffold = subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "scaffold-skill", + "--json", + str(metadata_path), + "--skill-name", + "recorded-action-smoke", + "--description", + "Replay the action smoke workflow captured by Open Computer Use Record & Replay.", + "--output-dir", + str(skill_dir), + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }) + scaffold_payload = json.loads(scaffold.stdout) + assert scaffold_payload["ok"] is True, scaffold_payload + skill_path = pathlib.Path(scaffold_payload["skillPath"]) + summary_path = pathlib.Path(scaffold_payload["summaryPath"]) + assert skill_path.exists(), scaffold_payload + assert summary_path.exists(), scaffold_payload + skill_text = skill_path.read_text() + summary_text = summary_path.read_text() + summary_json = json.loads(summary_text) + skill_creator_handoff_snippets = [ + "connector, API, or dedicated tool", + "skill-creator", + "Complete the `skill-creator` workflow, including validation", + "not a standalone runbook or replay plan", + ] + for snippet in skill_creator_handoff_snippets: + assert snippet in skill_text, skill_text + checked_skill_creator_finalization_handoff = True + assert "Replay Steps" in skill_text, skill_text + expected_replay_text = { + "mixed-action-stop": ("Click", "Scroll in"), + "simple-action-stop": ("Click",), + "drag-stop": ("Drag",), + }[action_scenario] + for expected_text in expected_replay_text: + assert expected_text in skill_text, skill_text + readiness = summary_json.get("skillReadiness", {}) + assert readiness.get("status") in ("ready", "needsReview"), readiness + assert readiness.get("canCreateSkillDraft") is True, readiness + checked_skill_readiness_can_create_draft = True + combined_scaffold_text = skill_text + "\n" + summary_text + forbidden_path_fragments = [ + "/Users/", + "/var/folders/", + "/private/var/", + "/tmp/", + str(metadata_path), + str(events_path), + str(session_dir), + ] + for fragment in forbidden_path_fragments: + assert fragment not in combined_scaffold_text, fragment + assert summary_json.get("metadataPath") == "", summary_json + assert summary_json.get("eventsPath") == "", summary_json + assert summary_json.get("sessionDir") == "", summary_json + checked_generated_skill_path_redaction = True + + request({"kind": "terminate"}) +finally: + for process in [agent]: + if process.poll() is None: + process.terminate() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + agent_stderr = agent.stderr.read() if agent.stderr else "" + if agent.returncode not in (0, -15, None): + raise AssertionError(f"app-agent exited with {agent.returncode}: {agent_stderr}") + +print(json.dumps({ + "ok": True, + "mode": "actions", + "actionScenario": action_scenario, + "sessionId": session_id, + "eventCount": stopped["eventCount"], + "eventTypes": sorted(set(event_types)), + "recordingsRoot": str(recordings), + "screenshotPolicy": screenshot_policy, + "skillPath": str(skill_path), + "mcpTranscriptPath": str(mcp_transcript_path), + "checkedGeneratedSkillPathRedaction": checked_generated_skill_path_redaction, + "checkedMcpResponseShapesCaptured": checked_mcp_response_shapes_captured, + "checkedSkillCreatorFinalizationHandoff": checked_skill_creator_finalization_handoff, + "checkedSkillReadinessCanCreateDraft": checked_skill_readiness_can_create_draft, +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION:-0}" == "1" ]]; then + python3 - "${tmpdir}" <<'PY' +import json +import os +import pathlib +import select +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]) +recordings = root / "recordings" + +env = os.environ.copy() +env.update({ + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "mcp", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), +}) + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "event-stream", "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, +) + +def read_response(timeout=5): + ready, _, _ = select.select([proc.stdout], [], [], timeout) + assert ready, f"MCP server did not respond within {timeout}s" + line = proc.stdout.readline() + assert line, "MCP server exited before responding" + return json.loads(line) + +def send(message): + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + +def request(message, timeout=5): + send(message) + return read_response(timeout=timeout) + +def tool_text(response): + content = response["result"]["content"] + assert content and content[0]["type"] == "text" + return json.loads(content[0]["text"]) + +try: + initialize_response = request({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "event-stream-mcp-elicitation-smoke", "version": "0"}, + }, + }) + send({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}) + send({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }) + elicitation_request = read_response() + assert elicitation_request["method"] == "elicitation/create" + assert elicitation_request["params"]["mode"] == "form" + assert "Record & Replay" in elicitation_request["params"]["message"] + assert elicitation_request["params"]["requestedSchema"]["type"] == "object" + assert "additionalProperties" not in elicitation_request["params"]["requestedSchema"] + send({ + "jsonrpc": "2.0", + "id": elicitation_request["id"], + "result": { + "action": "accept", + "content": {}, + }, + }) + start_response = read_response() + stop_response = request({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }) +finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + raise + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +initialize = initialize_response["result"] +assert initialize["protocolVersion"] == "2025-11-25" +assert initialize["serverInfo"]["name"] == "Record & Replay" +assert "instructions" not in initialize + +started = tool_text(start_response) +stopped = tool_text(stop_response) +session_id = started["sessionId"] +assert started["state"] == "recording" +assert stopped["state"] == "stopped" +assert stopped["sessionId"] == session_id +assert stopped["endReason"] == "recording_controls_stopped" +session_path = pathlib.Path(stopped["sessionPath"]) +metadata_path = session_path.parent / "metadata.json" +assert metadata_path.exists() +subprocess.run([ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(metadata_path), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + str(metadata_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) + +print(json.dumps({ + "ok": True, + "mode": "mcp-elicitation", + "sessionId": session_id, + "eventCount": stopped["eventCount"], + "recordingsRoot": str(recordings), +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APP_AGENT_WAIT:-0}" == "1" ]]; then + python3 - "${tmpdir}" "${screenshot_policy}" <<'PY' +import json +import os +import pathlib +import socket +import subprocess +import sys +import threading +import time + +root = pathlib.Path(sys.argv[1]) +screenshot_policy = sys.argv[2] +recordings = root / "recordings" +socket_path = root / "open-computer-use-agent.sock" + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "__open-computer-use-app-agent", str(socket_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +def connect_with_retry(): + deadline = time.time() + 10 + last_error = None + while time.time() < deadline: + try: + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM).connect(str(socket_path)) + except OSError as error: + last_error = error + time.sleep(0.05) + raise AssertionError(f"timed out waiting for app-agent socket: {last_error}") + +def request(payload): + deadline = time.time() + 10 + data = (json.dumps(payload, separators=(",", ":")) + "\n").encode() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + while True: + try: + client.connect(str(socket_path)) + break + except OSError as error: + if time.time() >= deadline: + raise AssertionError(f"failed to connect to app-agent socket: {error}") + time.sleep(0.05) + client.sendall(data) + chunks = [] + while True: + chunk = client.recv(4096) + assert chunk, "app-agent closed connection before newline response" + if b"\n" in chunk: + before, _sep, _after = chunk.partition(b"\n") + chunks.append(before) + break + chunks.append(chunk) + response = json.loads(b"".join(chunks).decode()) + if "error" in response: + raise AssertionError(response["error"]) + return response + +def cli(arguments, environment=None): + response = request({ + "kind": "cli", + "arguments": arguments, + "environment": environment or {}, + }) + assert response["exitCode"] == 0, response + stdout = response["stdout"].strip() + assert stdout, response + return json.loads(stdout) + +def event_stream_mcp(message, environment=None): + response = request({ + "kind": "eventStreamMCP", + "line": json.dumps(message, separators=(",", ":")), + "environment": environment or {}, + }) + if response["response"] is None: + return None + return json.loads(response["response"]) + +def tool_text(response): + content = response["result"]["content"] + assert content and content[0]["type"] == "text" + return json.loads(content[0]["text"]) + +try: + connect_with_retry() + start_env = { + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": screenshot_policy, + "OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + initialize_response = event_stream_mcp({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-app-agent-wait-smoke", "version": "0"}, + }, + }, start_env) + event_stream_mcp({ + "jsonrpc": "2.0", + "method": "notifications/initialized", + "params": {}, + }, start_env) + start_response = event_stream_mcp({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }, start_env) + + initialize = initialize_response["result"] + assert initialize["protocolVersion"] == "2025-11-25" + assert initialize["serverInfo"]["name"] == "Record & Replay" + assert "instructions" not in initialize + + started = tool_text(start_response) + session_id = started["sessionId"] + assert started["state"] == "recording" + assert started["active"] is True + assert pathlib.Path(started["eventsPath"]).is_relative_to(recordings) + + notify_script = root / "notify.sh" + notify_status_path = root / "notify-status.json" + notify_session_path = root / "notify-session.txt" + notify_state_path = root / "notify-state.txt" + notify_script.write_text( + "#!/bin/sh\n" + "cat > \"$1\"\n" + "printf '%s' \"$OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID\" > \"$2\"\n" + "printf '%s' \"$OPEN_COMPUTER_USE_EVENT_STREAM_STATE\" > \"$3\"\n" + "test -n \"$OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH\"\n" + "test -n \"$OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH\"\n", + encoding="utf-8", + ) + notify_script.chmod(0o755) + + waited = {} + wait_error = {} + + def wait_for_stop(): + try: + waited["status"] = cli([ + "event-stream", + "wait", + "--json", + "--session-id", + session_id, + "--timeout", + "5", + "--notify-command", + json.dumps([ + str(notify_script), + str(notify_status_path), + str(notify_session_path), + str(notify_state_path), + ]), + ]) + except BaseException as error: + wait_error["error"] = error + + waiter = threading.Thread(target=wait_for_stop) + waiter.start() + time.sleep(0.3) + stopped = cli(["event-stream", "stop", "--json"]) + waiter.join(timeout=8) + + assert not waiter.is_alive(), "wait command did not return after stop" + if "error" in wait_error: + raise wait_error["error"] + waited_status = waited["status"] + + assert stopped["state"] == "stopped" + assert stopped["sessionId"] == session_id + assert stopped["endReason"] == "recording_controls_stopped" + assert waited_status["state"] == "stopped" + assert waited_status["sessionId"] == session_id + assert waited_status["endReason"] == stopped["endReason"] + assert waited_status["waitTimedOut"] is False + assert waited_status["waitSessionMatched"] is True + notification = waited_status["notification"] + assert notification["attempted"] is True + assert notification["skipped"] is False + assert notification["ok"] is True + assert notification["exitCode"] == 0 + + notify_status = json.loads(notify_status_path.read_text()) + assert notify_status["sessionId"] == session_id + assert notify_status["state"] == "stopped" + assert notify_status["waitTimedOut"] is False + assert notify_status["waitSessionMatched"] is True + assert notify_session_path.read_text() == session_id + assert notify_state_path.read_text() == "stopped" + + completed_wait = cli([ + "event-stream", + "wait", + "--json", + "--session-id", + session_id, + "--timeout", + "5", + ]) + assert completed_wait["state"] == "stopped" + assert completed_wait["sessionId"] == session_id + assert completed_wait["endReason"] == stopped["endReason"] + assert completed_wait["waitTimedOut"] is False + assert completed_wait["waitSessionMatched"] is True + + events_path = pathlib.Path(stopped["eventsPath"]) + session_path = pathlib.Path(stopped["sessionPath"]) + metadata_path = session_path.parent / "metadata.json" + session_alias_path = metadata_path.parent / "session.json" + assert events_path.exists() + assert metadata_path.exists() + metadata = json.loads(metadata_path.read_text()) + session_alias = json.loads(session_alias_path.read_text()) + assert session_alias.get("id") == metadata.get("sessionId") + assert session_alias.get("eventsPath") == metadata.get("eventsPath") + assert session_alias.get("startedAt") == metadata.get("startedAt") + assert session_alias.get("endReason") == metadata.get("endReason") + assert not (recordings / "active-session.json").exists() + subprocess.run([ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(metadata_path), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + str(metadata_path), + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }) + + request({"kind": "terminate"}) +finally: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +print(json.dumps({ + "ok": True, + "mode": "app-agent-wait", + "sessionId": session_id, + "eventCount": stopped["eventCount"], + "recordingsRoot": str(recordings), + "screenshotPolicy": screenshot_policy, +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_WAIT_TIMEOUT:-0}" == "1" ]]; then + python3 - "${tmpdir}" "${screenshot_policy}" <<'PY' +import json +import os +import pathlib +import socket +import subprocess +import sys +import time + +root = pathlib.Path(sys.argv[1]) +screenshot_policy = sys.argv[2] +recordings = root / "recordings" +socket_path = root / "open-computer-use-agent.sock" + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "__open-computer-use-app-agent", str(socket_path)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, +) + +def request(payload): + deadline = time.time() + 10 + data = (json.dumps(payload, separators=(",", ":")) + "\n").encode() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as client: + while True: + try: + client.connect(str(socket_path)) + break + except OSError as error: + if time.time() >= deadline: + raise AssertionError(f"failed to connect to app-agent socket: {error}") + time.sleep(0.05) + client.sendall(data) + chunks = [] + while True: + chunk = client.recv(4096) + assert chunk, "app-agent closed connection before newline response" + if b"\n" in chunk: + before, _sep, _after = chunk.partition(b"\n") + chunks.append(before) + break + chunks.append(chunk) + response = json.loads(b"".join(chunks).decode()) + if "error" in response: + raise AssertionError(response["error"]) + return response + +def cli(arguments, environment=None): + response = request({ + "kind": "cli", + "arguments": arguments, + "environment": environment or {}, + }) + assert response["exitCode"] == 0, response + stdout = response["stdout"].strip() + assert stdout, response + return json.loads(stdout) + +try: + start_env = { + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": screenshot_policy, + "OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + started = cli(["event-stream", "start", "--json"], start_env) + session_id = started["sessionId"] + assert started["state"] == "recording" + assert pathlib.Path(started["eventsPath"]).is_relative_to(recordings) + + waited = cli([ + "event-stream", + "wait", + "--json", + "--session-id", + session_id, + "--timeout", + "0.2", + ]) + assert waited["sessionId"] == session_id + assert waited["state"] == "recording" + assert waited["active"] is True + assert waited["waitTimedOut"] is True + assert waited["waitSessionMatched"] is True + + cancelled = cli(["event-stream", "cancel", "--json"]) + assert cancelled["sessionId"] == session_id + assert cancelled["state"] == "cancelled" + assert cancelled["endReason"] == "recording_controls_cancelled" + assert not (recordings / "active-session.json").exists() + subprocess.run([ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(cancelled["metadataPath"]), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + str(cancelled["metadataPath"]), + ], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }) + + request({"kind": "terminate"}) +finally: + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + proc.wait(timeout=5) + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +print(json.dumps({ + "ok": True, + "mode": "wait-timeout", + "sessionId": session_id, + "waitTimedOut": waited["waitTimedOut"], + "recordingsRoot": str(recordings), + "screenshotPolicy": screenshot_policy, +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ -n "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL:-}" ]]; then + approval_policy="${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL}" + if [[ "${approval_policy}" != "deny" && "${approval_policy}" != "cancel" ]]; then + echo "OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_APPROVAL must be 'deny' or 'cancel'" >&2 + exit 2 + fi +python3 - "${tmpdir}" "${approval_policy}" <<'PY' +import json +import os +import pathlib +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]) +approval_policy = sys.argv[2] +recordings = root / "recordings" + +env = os.environ.copy() +env.update({ + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": approval_policy, + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), +}) + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "event-stream", "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, +) + +def request(message, expect_response=True): + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + line = proc.stdout.readline() + assert line, "MCP server exited before responding" + return json.loads(line) + +try: + initialize_response = request({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-approval-smoke", "version": "0"}, + }, + }) + request({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, expect_response=False) + start_response = request({ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }) +finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + raise + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +initialize = initialize_response["result"] +assert initialize["protocolVersion"] == "2025-11-25" +assert initialize["serverInfo"]["name"] == "Record & Replay" +assert "instructions" not in initialize + +result = start_response["result"] +assert result["isError"] is True +content = result["content"] +assert content and content[0]["type"] == "text" +text = content[0]["text"] +if approval_policy == "deny": + expected = "Record & Replay approval denied via MCP elicitation." +else: + expected = "Record & Replay approval cancelled via MCP elicitation." +assert text == expected +assert not (recordings / "latest-session.json").exists() +assert not (recordings / "active-session.json").exists() + +print(json.dumps({ + "ok": True, + "mode": "approval", + "approvalPolicy": approval_policy, + "message": text, + "recordingsRoot": str(recordings), +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE:-0}" == "1" ]]; then + python3 - "${tmpdir}" <<'PY' +import json +import os +import pathlib +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]) +recordings = root / "recordings" + +env = os.environ.copy() +env.update({ + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), +}) + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "event-stream", "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, +) + +def request(message, expect_response=True): + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + line = proc.stdout.readline() + assert line, "MCP server exited before responding" + return json.loads(line) + +def tool_text(response): + content = response["result"]["content"] + assert content and content[0]["type"] == "text" + return json.loads(content[0]["text"]) + +try: + initialize_response = request({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-no-active-smoke", "version": "0"}, + }, + }) + request({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, expect_response=False) + tools_response = request({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + status_response = request({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }) + stop_response = request({ + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }) +finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + raise + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +initialize = initialize_response["result"] +assert initialize["protocolVersion"] == "2025-11-25" +assert initialize["serverInfo"]["name"] == "Record & Replay" +assert "instructions" not in initialize + +tools = tools_response["result"]["tools"] +assert [tool["name"] for tool in tools] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", +] + +status = tool_text(status_response) +stopped = tool_text(stop_response) + +for payload in [status, stopped]: + assert payload == { + "isRecording": False, + "maxDurationSeconds": 1800, + }, payload + assert "sessionId" not in payload + assert "eventsPath" not in payload + assert "metadataPath" not in payload + +assert not (recordings / "latest-session.json").exists() +assert not (recordings / "active-session.json").exists() + +print(json.dumps({ + "ok": True, + "mode": "no-active", + "isRecording": stopped["isRecording"], + "maxDurationSeconds": stopped["maxDurationSeconds"], + "recordingsRoot": str(recordings), +}, sort_keys=True)) +PY + exit 0 +fi + +if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT:-0}" == "1" ]]; then + timeout_seconds="${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_TIMEOUT_SECONDS:-1.5}" + python3 - "${tmpdir}" "${screenshot_policy}" "${timeout_seconds}" <<'PY' +import json +import os +import pathlib +import subprocess +import sys +import time + +root = pathlib.Path(sys.argv[1]) +screenshot_policy = sys.argv[2] +timeout_seconds = float(sys.argv[3]) +recordings = root / "recordings" +responses = [] + +env = os.environ.copy() +env.update({ + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": screenshot_policy, + "OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + "OPEN_COMPUTER_USE_EVENT_STREAM_MAX_DURATION_SECONDS": str(timeout_seconds), +}) + +proc = subprocess.Popen( + [".build/debug/OpenComputerUse", "event-stream", "mcp"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, +) + +def request(message, expect_response=True): + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + line = proc.stdout.readline() + assert line, "MCP server exited before responding" + response = json.loads(line) + responses.append(response) + return response + +def tool_text(response): + content = response["result"]["content"] + assert content and content[0]["type"] == "text" + return json.loads(content[0]["text"]) + +try: + initialize_response = request({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-timeout-smoke", "version": "0"}, + }, + }) + request({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, expect_response=False) + tools_response = request({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + start_response = request({ + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }) + + time.sleep(timeout_seconds + 0.8) + + status_response = request({ + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }) + stop_after_timeout_response = request({ + "jsonrpc": "2.0", + "id": 5, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }) +finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + raise + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + +initialize = initialize_response["result"] +assert initialize["protocolVersion"] == "2025-11-25" +assert initialize["serverInfo"]["name"] == "Record & Replay" +assert "instructions" not in initialize + +tools = tools_response["result"]["tools"] +assert [tool["name"] for tool in tools] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", +] + +started = tool_text(start_response) +stopped = tool_text(status_response) +stop_after_timeout = tool_text(stop_after_timeout_response) + +assert started["state"] == "recording" +assert started["active"] is True +assert stopped["state"] == "stopped" +assert stopped["active"] is False +assert stopped["endReason"] == "recording_time_limit_reached" +assert stop_after_timeout["state"] == "stopped" +assert stop_after_timeout["sessionId"] == stopped["sessionId"] +assert stop_after_timeout["endReason"] == stopped["endReason"] + +events_path = pathlib.Path(stopped["eventsPath"]) +session_path = pathlib.Path(stopped["sessionPath"]) +metadata_path = session_path.parent / "metadata.json" +suppressed_path = pathlib.Path(stopped["suppressedEventsPath"]) +session_alias_path = metadata_path.parent / "session.json" + +assert events_path.exists() +assert metadata_path.exists() +assert session_path.exists() +assert suppressed_path.exists() +assert session_alias_path.exists() +assert session_path == session_alias_path +metadata = json.loads(metadata_path.read_text()) +session_alias = json.loads(session_alias_path.read_text()) +assert session_alias.get("id") == metadata.get("sessionId") +assert session_alias.get("eventsPath") == metadata.get("eventsPath") +assert session_alias.get("startedAt") == metadata.get("startedAt") +assert session_alias.get("endReason") == metadata.get("endReason") + +events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if line.strip() +] +event_types = [event["type"] for event in events] +assert "session.started" in event_types +assert "window.changed" in event_types +assert "AX.focusedWindowChanged" in event_types +assert "session.ended" in event_types +assert any( + event.get("type") == "session.ended" + and event.get("endReason") == "recording_time_limit_reached" + for event in events +) +subprocess.run([ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(metadata_path), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-event-type", + "session.ended", +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-event-type", + "session.ended", + str(metadata_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) + +print(json.dumps({ + "ok": True, + "mode": "timeout", + "sessionId": stopped["sessionId"], + "eventCount": stopped["eventCount"], + "recordingsRoot": str(recordings), + "screenshotPolicy": screenshot_policy, + "timeoutSeconds": timeout_seconds, +}, sort_keys=True)) +PY + exit 0 +fi + +OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY=1 \ +OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS=0 \ +OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=approve \ +OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS="${screenshot_policy}" \ +OPEN_COMPUTER_USE_EVENT_STREAM_RAW_EVENTS=0 \ +OPEN_COMPUTER_USE_EVENT_STREAM_DIR="${tmpdir}/recordings" \ + ".build/debug/OpenComputerUse" event-stream mcp \ + <"${messages_file}" >"${tmpdir}/responses.jsonl" + +python3 - "${tmpdir}" "${screenshot_policy}" <<'PY' +import json +import os +import pathlib +import select +import subprocess +import sys + +root = pathlib.Path(sys.argv[1]) +screenshot_policy = sys.argv[2] +recordings = root / "recordings" +responses_path = root / "responses.jsonl" + +responses = [ + json.loads(line) + for line in responses_path.read_text().splitlines() + if line.strip() +] +by_id = {response.get("id"): response for response in responses if "id" in response} + +initialize = by_id[1]["result"] +assert initialize["protocolVersion"] == "2025-11-25" +assert initialize["serverInfo"]["name"] == "Record & Replay" +assert "instructions" not in initialize + +tools = by_id[2]["result"]["tools"] +assert [tool["name"] for tool in tools] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", +] + +def tool_text(response_id): + content = by_id[response_id]["result"]["content"] + assert content and content[0]["type"] == "text" + return json.loads(content[0]["text"]) + +started = tool_text(3) +repeated_start = tool_text(4) +status = tool_text(5) +stopped = tool_text(6) +repeated_stop = tool_text(7) +status_after_stop = tool_text(8) + +assert started["state"] == "recording" +assert started["active"] is True +assert repeated_start["state"] == "recording" +assert repeated_start["sessionId"] == started["sessionId"] +assert status["state"] == "recording" +assert status["sessionId"] == started["sessionId"] +assert stopped["state"] == "stopped" +assert stopped["active"] is False +assert stopped["endReason"] == "recording_controls_stopped" +assert repeated_stop["state"] == "stopped" +assert repeated_stop["sessionId"] == stopped["sessionId"] +assert repeated_stop["endReason"] == stopped["endReason"] +assert status_after_stop["state"] == "stopped" +assert status_after_stop["sessionId"] == stopped["sessionId"] + +events_path = pathlib.Path(stopped["eventsPath"]) +session_path = pathlib.Path(stopped["sessionPath"]) +metadata_path = session_path.parent / "metadata.json" +suppressed_path = pathlib.Path(stopped["suppressedEventsPath"]) +session_alias_path = metadata_path.parent / "session.json" + +assert events_path.exists() +assert metadata_path.exists() +assert session_path.exists() +assert suppressed_path.exists() +assert session_alias_path.exists() +assert session_path == session_alias_path +metadata = json.loads(metadata_path.read_text()) +session_alias = json.loads(session_alias_path.read_text()) +assert session_alias.get("id") == metadata.get("sessionId") +assert session_alias.get("eventsPath") == metadata.get("eventsPath") +assert session_alias.get("startedAt") == metadata.get("startedAt") +assert session_alias.get("endReason") == metadata.get("endReason") + +events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if line.strip() +] +event_types = [event["type"] for event in events] +assert "session.started" in event_types +assert "window.changed" in event_types +assert "AX.focusedWindowChanged" in event_types +assert "session.ended" in event_types +subprocess.run([ + sys.executable, + "scripts/validate-event-stream-recording.py", + str(metadata_path), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-event-type", + "session.ended", +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "AX.focusedWindowChanged", + "--require-event-type", + "session.ended", + str(metadata_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "validate", + "--json", + "--strict-ocu", + str(session_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) +subprocess.run([ + sys.executable, + "scripts/summarize-event-stream-recording.py", + str(metadata_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "summarize", + "--json", + str(metadata_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) +subprocess.run([ + ".build/debug/OpenComputerUse", + "event-stream", + "summarize", + "--json", + str(session_path), +], check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", +}) + +ax_payloads = [ + event.get("accessibilityInspectorPayload") + for event in events + if event.get("type") == "AX.focusedWindowChanged" +] +ax_payloads = [payload for payload in ax_payloads if payload] +assert ax_payloads, "expected at least one AX payload" +screenshot_needed_count = sum( + 1 for payload in ax_payloads if payload.get("screenshotNeededForContext") +) +screenshot_available_payloads = [ + payload for payload in ax_payloads if payload.get("screenshotAvailable") +] +screenshot_available_count = len(screenshot_available_payloads) +screenshot_paths = [ + pathlib.Path(payload["screenshotPath"]) + for payload in screenshot_available_payloads + if payload.get("screenshotPath") +] +screenshot_path_count = len(screenshot_paths) + +if screenshot_policy == "always": + assert screenshot_needed_count > 0, "expected screenshotNeededForContext with always policy" + if screenshot_available_payloads: + assert screenshot_paths, "expected screenshotPath when screenshot is available" + assert all(path.exists() for path in screenshot_paths) + +if os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL") == "1": + official_timeout = float(os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_TIMEOUT", "10")) + plugin_dir = pathlib.Path(os.environ.get( + "OPEN_COMPUTER_USE_EVENT_STREAM_OFFICIAL_PLUGIN_DIR", + str(pathlib.Path.home() / ".codex/plugins/cache/openai-bundled/record-and-replay/1.0.857"), + )) + official_client = plugin_dir / "Codex Computer Use.app/Contents/SharedSupport/SkyComputerUseClient.app/Contents/MacOS/SkyComputerUseClient" + assert official_client.exists(), f"official Record & Replay client not found: {official_client}" + + proc = subprocess.Popen( + [str(official_client), "event-stream", "mcp"], + cwd=str(plugin_dir), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def official_request(message, expect_response=True): + proc.stdin.write(json.dumps(message, separators=(",", ":")) + "\n") + proc.stdin.flush() + if not expect_response: + return None + ready, _, _ = select.select([proc.stdout], [], [], official_timeout) + assert ready, f"official MCP server did not respond within {official_timeout}s" + line = proc.stdout.readline() + assert line, "official MCP server exited before responding" + return json.loads(line) + + try: + official_initialize_response = official_request({ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "event-stream-smoke-official-compare", "version": "0"}, + }, + }) + official_request({"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, expect_response=False) + official_tools_response = official_request({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}) + finally: + if proc.stdin: + proc.stdin.close() + try: + proc.wait(timeout=5) + except subprocess.TimeoutExpired: + proc.kill() + raise + stderr = proc.stderr.read() if proc.stderr else "" + assert proc.returncode == 0, stderr + + official_initialize = official_initialize_response["result"] + assert official_initialize["protocolVersion"] == initialize["protocolVersion"] + assert official_initialize["serverInfo"]["name"] == initialize["serverInfo"]["name"] + assert "instructions" not in official_initialize + + official_tools = official_tools_response["result"]["tools"] + comparable_local_tools = [ + {key: tool[key] for key in ["name", "description", "inputSchema", "annotations"]} + for tool in tools + ] + comparable_official_tools = [ + {key: tool[key] for key in ["name", "description", "inputSchema", "annotations"]} + for tool in official_tools + ] + assert comparable_local_tools == comparable_official_tools + +print(json.dumps({ + "ok": True, + "sessionId": stopped["sessionId"], + "eventCount": stopped["eventCount"], + "recordingsRoot": str(recordings), + "screenshotPolicy": screenshot_policy, + "screenshotContextChecked": screenshot_policy == "always", + "screenshotNeededForContextCount": screenshot_needed_count, + "screenshotAvailableCount": screenshot_available_count, + "screenshotPathCount": screenshot_path_count, + "handoffChecked": True, + "officialCompared": os.environ.get("OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL") == "1", +}, sort_keys=True)) +PY diff --git a/scripts/run-record-and-replay-baseline-smoke.sh b/scripts/run-record-and-replay-baseline-smoke.sh new file mode 100755 index 00000000..cbaf71f8 --- /dev/null +++ b/scripts/run-record-and-replay-baseline-smoke.sh @@ -0,0 +1,517 @@ +#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${repo_root}" + +require_official_golden=0 +summary_json="" + +usage() { + cat <<'EOF' +Usage: scripts/run-record-and-replay-baseline-smoke.sh [--require-official-golden] [--summary-json ] + +Runs the opt-in Record & Replay baseline smoke suite. + +Options: + --require-official-golden Return non-zero unless required official successful + recording fixtures are present in the repository. + --summary-json Write the final machine-readable baseline summary + JSON to this path while still printing it to stdout. + -h, --help Show this help. +EOF +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --require-official-golden) + require_official_golden=1 + shift + ;; + --summary-json) + if [[ $# -lt 2 ]]; then + echo "--summary-json requires a path" >&2 + usage >&2 + exit 2 + fi + summary_json="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +run_step() { + local name="$1" + shift + echo "==> ${name}" >&2 + "$@" +} + +run_json_step() { + local name="$1" + local output_file="$2" + shift 2 + echo "==> ${name}" >&2 + "$@" | tee "${output_file}" +} + +assert_json_true_keys() { + local json_file="$1" + shift + python3 - "${json_file}" "$@" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +keys = sys.argv[2:] +payload = json.loads(path.read_text()) +missing = [key for key in keys if payload.get(key) is not True] +if missing: + raise SystemExit( + "expected true JSON keys missing from " + + str(path) + + ": " + + ", ".join(missing) + + "\n" + + json.dumps(payload, indent=2, sort_keys=True) + ) +PY +} + +assert_json_contract_true_keys() { + local json_file="$1" + local contract_name="$2" + python3 - "${json_file}" "${contract_name}" <<'PY' +import json +import pathlib +import sys + +sys.path.insert(0, str(pathlib.Path.cwd() / "scripts")) +import record_and_replay_baseline_contract as contract + +path = pathlib.Path(sys.argv[1]) +contract_name = sys.argv[2] +keys = getattr(contract, contract_name) +payload = json.loads(path.read_text()) +missing = [key for key in keys if payload.get(key) is not True] +if missing: + raise SystemExit( + "expected true JSON keys missing from " + + str(path) + + ": " + + ", ".join(missing) + + "\n" + + json.dumps(payload, indent=2, sort_keys=True) + ) +PY +} + +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/ocu-rnr-baseline-smoke.XXXXXX")" +trap 'rm -rf "${tmp_dir}"' EXIT +standalone_json="${tmp_dir}/standalone-skill-repo-smoke.json" +npm_json="${tmp_dir}/npm-staged-skill-repo-smoke.json" +official_surface_json="${tmp_dir}/official-surface-compare.json" +official_no_active_json="${tmp_dir}/official-no-active-response.json" +official_raw_timeout_json="${tmp_dir}/official-raw-start-timeout.json" +official_fixture_set_json="${tmp_dir}/official-fixture-set-smoke.json" +official_fixture_coverage_json="${tmp_dir}/official-fixture-coverage.json" +official_fixture_ingest_json="${tmp_dir}/official-fixture-ingest-smoke.json" +ocu_candidate_ingest_json="${tmp_dir}/ocu-candidate-ingest-smoke.json" +official_capture_preflight_json="${tmp_dir}/official-capture-preflight-smoke.json" +ocu_pairing_preflight_json="${tmp_dir}/ocu-pairing-preflight-smoke.json" +baseline_audit_targets_json="${tmp_dir}/baseline-audit-targets-smoke.json" +baseline_contract_json="${tmp_dir}/baseline-contract-smoke.json" +matrix_jsonl="${tmp_dir}/event-stream-matrix.jsonl" +screenshot_jsonl="${tmp_dir}/event-stream-screenshot-smoke.jsonl" +action_jsonl="${tmp_dir}/event-stream-action-smoke.jsonl" +action_simple_jsonl="${tmp_dir}/event-stream-action-simple-smoke.jsonl" +action_drag_jsonl="${tmp_dir}/event-stream-action-drag-smoke.jsonl" +generated_summary_json="${tmp_dir}/record-and-replay-baseline-summary.json" + +run_json_step "baseline contract smoke" "${baseline_contract_json}" ./scripts/test-record-and-replay-baseline-contract.py +assert_json_true_keys \ + "${baseline_contract_json}" \ + checkedRequiredBaselineChecks \ + checkedNoDuplicateRequiredBaselineChecks \ + checkedStandaloneSmokeRequiredKeys \ + checkedStandaloneSummaryEvidenceKeys \ + checkedStandaloneLifecycleSummaryRenames \ + checkedNpmStagedSmokeRequiredKeys \ + checkedNpmStagedSummaryEvidenceKeys \ + checkedNpmStagedSummaryMatchesSmoke +run_json_step "event-stream smoke matrix" "${matrix_jsonl}" ./scripts/run-event-stream-smoke-matrix.sh +run_json_step "event-stream screenshot context smoke" "${screenshot_jsonl}" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS=1 ./scripts/run-event-stream-smoke-tests.sh +run_json_step "event-stream real input action smoke" "${action_jsonl}" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 ./scripts/run-event-stream-smoke-tests.sh +run_json_step "event-stream simple action candidate smoke" "${action_simple_jsonl}" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=simple-action-stop ./scripts/run-event-stream-smoke-tests.sh +cat "${action_simple_jsonl}" >>"${action_jsonl}" +run_json_step "event-stream drag action candidate smoke" "${action_drag_jsonl}" env OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS=1 OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO=drag-stop ./scripts/run-event-stream-smoke-tests.sh +cat "${action_drag_jsonl}" >>"${action_jsonl}" +python3 - "${matrix_jsonl}" "${screenshot_jsonl}" "${action_jsonl}" <<'PY' +import json +import pathlib +import sys + + +def json_lines(path): + records = [] + for line in pathlib.Path(path).read_text().splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + records.append(json.loads(line)) + return records + + +matrix = json_lines(sys.argv[1]) +screenshot = json_lines(sys.argv[2]) +action = json_lines(sys.argv[3]) +matrix_modes = {record.get("mode") for record in matrix if record.get("mode")} +required_modes = { + "no-active", + "timeout", + "wait-timeout", + "approval", + "mcp-elicitation", + "app-agent-wait", +} +missing_modes = sorted(required_modes - matrix_modes) +has_default_lifecycle = any( + record.get("ok") is True and record.get("handoffChecked") is True for record in matrix +) +has_matrix_summary = any(record.get("ok") is True and record.get("matrix") == "event-stream" for record in matrix) +if missing_modes or not has_default_lifecycle or not has_matrix_summary: + raise SystemExit( + "event-stream matrix evidence incomplete\n" + + json.dumps( + { + "missingModes": missing_modes, + "hasDefaultLifecycle": has_default_lifecycle, + "hasMatrixSummary": has_matrix_summary, + "records": matrix, + }, + indent=2, + sort_keys=True, + ) + ) + +screenshot_records = [ + record + for record in screenshot + if record.get("ok") is True and record.get("screenshotPolicy") == "always" +] +if not screenshot_records: + raise SystemExit( + "missing screenshot context smoke JSON record\n" + + json.dumps(screenshot, indent=2, sort_keys=True) + ) +screenshot_record = screenshot_records[-1] +if ( + screenshot_record.get("screenshotContextChecked") is not True + or not isinstance(screenshot_record.get("screenshotNeededForContextCount"), int) + or screenshot_record.get("screenshotNeededForContextCount") <= 0 +): + raise SystemExit( + "event-stream screenshot context evidence incomplete\n" + + json.dumps(screenshot_record, indent=2, sort_keys=True) + ) +if ( + isinstance(screenshot_record.get("screenshotAvailableCount"), int) + and screenshot_record.get("screenshotAvailableCount") > 0 + and screenshot_record.get("screenshotPathCount", 0) <= 0 +): + raise SystemExit( + "event-stream screenshot context reported available screenshots without paths\n" + + json.dumps(screenshot_record, indent=2, sort_keys=True) + ) + +action_records = [record for record in action if record.get("mode") == "actions"] +if not action_records: + raise SystemExit("missing action smoke JSON record\n" + json.dumps(action, indent=2, sort_keys=True)) +mixed_records = [ + record + for record in action_records + if record.get("actionScenario") in (None, "mixed-action-stop") +] +action_record = mixed_records[-1] if mixed_records else action_records[-1] +required_event_types = { + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.click", + "session.ended", +} +action_event_types = set(action_record.get("eventTypes", [])) +missing_event_types = sorted(required_event_types - action_event_types) +if ( + action_record.get("ok") is not True + or missing_event_types + or not action_record.get("skillPath") + or not action_record.get("mcpTranscriptPath") + or action_record.get("checkedMcpResponseShapesCaptured") is not True + or action_record.get("checkedSkillReadinessCanCreateDraft") is not True + or action_record.get("checkedSkillCreatorFinalizationHandoff") is not True + or action_record.get("checkedGeneratedSkillPathRedaction") is not True +): + raise SystemExit( + "event-stream action evidence incomplete\n" + + json.dumps( + { + "missingEventTypes": missing_event_types, + "record": action_record, + }, + indent=2, + sort_keys=True, + ) + ) + +scenario_requirements = { + "simple-action-stop": {"mouse.click"}, + "drag-stop": {"mouse.drag"}, +} +for scenario, required_types in scenario_requirements.items(): + scenario_records = [ + record for record in action_records if record.get("actionScenario") == scenario + ] + if not scenario_records: + raise SystemExit( + f"missing event-stream action scenario evidence for {scenario}\n" + + json.dumps(action_records, indent=2, sort_keys=True) + ) + scenario_record = scenario_records[-1] + scenario_event_types = set(scenario_record.get("eventTypes", [])) + scenario_required_event_types = ( + {"session.started", "window.changed", "AX.focusedWindowChanged", "session.ended"} + | required_types + ) + scenario_missing_event_types = sorted( + scenario_required_event_types - scenario_event_types + ) + if ( + scenario_record.get("ok") is not True + or scenario_missing_event_types + or scenario_record.get("checkedMcpResponseShapesCaptured") is not True + or scenario_record.get("checkedSkillReadinessCanCreateDraft") is not True + or scenario_record.get("checkedSkillCreatorFinalizationHandoff") is not True + or scenario_record.get("checkedGeneratedSkillPathRedaction") is not True + ): + raise SystemExit( + f"event-stream action scenario evidence incomplete for {scenario}\n" + + json.dumps( + { + "missingEventTypes": scenario_missing_event_types, + "record": scenario_record, + }, + indent=2, + sort_keys=True, + ) + ) +PY +run_json_step "official Record & Replay surface compare" "${official_surface_json}" ./scripts/compare-event-stream-surface.py --use-default-official +python3 - "${official_surface_json}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +by_label = {result.get("label"): result for result in payload.get("results", [])} +required = ["local-open-computer-use", "official-record-and-replay"] +missing = [label for label in required if by_label.get(label, {}).get("ok") is not True] +if payload.get("ok") is not True or missing: + raise SystemExit( + "official surface compare did not prove required labels: " + + ", ".join(missing) + + "\n" + + json.dumps(payload, indent=2, sort_keys=True) + ) +PY +run_json_step "official no-active response compare" "${official_no_active_json}" ./scripts/compare-event-stream-no-active.py +python3 - "${official_no_active_json}" <<'PY' +import json +import pathlib +import sys + +payload = json.loads(pathlib.Path(sys.argv[1]).read_text()) +expected_response = {"isRecording": False, "maxDurationSeconds": 1800} +actual = payload.get("actual", {}) +checked_tools = payload.get("checkedTools", []) +missing_tools = [ + tool for tool in ["event_stream_status", "event_stream_stop"] if tool not in checked_tools +] +if ( + payload.get("ok") is not True + or payload.get("createdSessionFiles") is not False + or missing_tools + or actual.get("event_stream_status") != expected_response + or actual.get("event_stream_stop") != expected_response +): + raise SystemExit( + "official no-active response compare did not prove required shape\n" + + json.dumps( + { + "missingTools": missing_tools, + "payload": payload, + }, + indent=2, + sort_keys=True, + ) + ) +PY +run_json_step "official raw start timeout fixture smoke" "${official_raw_timeout_json}" ./scripts/test-event-stream-probe-fixtures.py +assert_json_true_keys \ + "${official_raw_timeout_json}" \ + checkedOfficialRawStartTimeoutBoundary \ + checkedOfficialRawStartStatusStopTimeout \ + checkedOfficialRawStartDoesNotReturnRecordingPaths \ + checkedOfficialRawSurface \ + checkedRawProbeFixtureRedaction +run_json_step "official fixture set gate smoke" "${official_fixture_set_json}" ./scripts/test-event-stream-official-fixture-set.py +assert_json_true_keys \ + "${official_fixture_set_json}" \ + checkedFixtureSetGate \ + checkedRequiredSimpleActionStopScenario \ + checkedCandidatePairing \ + checkedCandidateReadinessFailure \ + checkedMissingCandidateFailure \ + checkedAxDiffComparisonPolicy \ + checkedSuppressedStreamComparisonPolicy \ + checkedAxDiffComparisonFailure \ + checkedStopEndReasonPolicy \ + checkedCancelScenarioPolicy \ + checkedKeyboardInputScenarioPolicy \ + checkedDragScenarioPolicy \ + checkedTimeoutScenarioPolicy \ + checkedMissingScenarioFailure \ + checkedImportScenarioManifest +run_json_step "official fixture coverage report" "${official_fixture_coverage_json}" ./scripts/check-event-stream-official-fixture-coverage.py --allow-missing --check-readiness +run_json_step "official fixture ingest smoke" "${official_fixture_ingest_json}" ./scripts/test-official-record-and-replay-fixture-ingest.py +assert_json_true_keys \ + "${official_fixture_ingest_json}" \ + checkedOfficialFixtureIngest \ + checkedOfficialFixtureInspectOnly \ + checkedOfficialSessionDirectoryPathHandoff \ + checkedPostIngestCoverageReport \ + checkedPostIngestCoverageReadiness \ + checkedPostIngestRequireCoverageFailure +run_json_step "OCU candidate ingest smoke" "${ocu_candidate_ingest_json}" ./scripts/test-ocu-record-and-replay-candidate-ingest.py +assert_json_true_keys \ + "${ocu_candidate_ingest_json}" \ + checkedOcuCandidateIngest \ + checkedCandidateIngestHandoffCommands \ + checkedSmokeJsonImport \ + checkedOfficialCandidatePairing \ + checkedCandidateRedaction \ + checkedKeyboardInputScenarioReadiness \ + checkedDragScenarioReadiness +run_json_step "official golden capture preflight smoke" "${official_capture_preflight_json}" ./scripts/test-record-and-replay-official-golden-capture-preflight.py +assert_json_true_keys \ + "${official_capture_preflight_json}" \ + checkedMissingScenarioPreflight \ + checkedCapturePacket \ + checkedCapturePacketPostCaptureWorkflow \ + checkedCapturePacketHandoffScripts \ + checkedCapturePacketStrictAuditHandoff \ + checkedCapturePacketStrictExpectedFailureAuditHandoff \ + checkedCapturePacketOcuCandidateOutputDir \ + checkedCapturePacketNoTranscript \ + checkedCapturePacketSet \ + checkedCapturePacketSetPostCaptureWorkflow \ + checkedCapturePacketSetOcuCandidateHandoff \ + checkedCapturePacketSetContractManifest \ + checkedCapturePacketTranscriptManifest \ + checkedMakeCapturePacketTargets \ + checkedCapturePacketInputSemanticGuard \ + checkedCapturePacketPlaceholderGuard \ + checkedCapturePacketVerifyInputs \ + checkedCapturePacketImportPlaceholderGuard \ + checkedCapturePacketTranscriptPlaceholderGuard \ + checkedCapturePacketSetRootPlaceholderGuard \ + checkedCapturePacketSetRootPreflightPlaceholderGuard \ + checkedCapturePacketSetVerifyAll \ + checkedRequireReadyFailure \ + checkedMissingPluginFailure \ + checkedAllowedMissingPluginKeyboardScenario \ + checkedCoverageErrorFailure +run_json_step "OCU candidate pairing preflight smoke" "${ocu_pairing_preflight_json}" ./scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py +assert_json_true_keys \ + "${ocu_pairing_preflight_json}" \ + checkedMissingOfficialPreflight \ + checkedNoCandidatePreflight \ + checkedPairedCandidatePreflight \ + checkedKeyboardRecordingRequiredScenario \ + checkedCoverageErrorReport +run_json_step "baseline audit Make targets smoke" "${baseline_audit_targets_json}" ./scripts/test-record-and-replay-baseline-audit-make-targets.py +assert_json_true_keys \ + "${baseline_audit_targets_json}" \ + checkedBaselineAuditMakeTarget \ + checkedBaselineAuditDefaultSummaryPath \ + checkedBaselineAuditCustomSummaryPath \ + checkedBaselineAuditIgnoresStrictSummaryVar \ + checkedStrictOfficialGoldenAuditMakeTarget \ + checkedStrictOfficialGoldenAuditDefaultSummaryPath \ + checkedStrictOfficialGoldenAuditCustomSummaryPath \ + checkedStrictOfficialGoldenAuditIgnoresBaselineSummaryVar \ + checkedStrictOfficialGoldenAuditSeparateSummaryPath +run_json_step "standalone Record & Replay skill repo smoke" "${standalone_json}" ./scripts/test-record-and-replay-skill-repo-scaffold.py +assert_json_contract_true_keys \ + "${standalone_json}" \ + STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS +run_json_step "npm staged Record & Replay skill repo smoke" "${npm_json}" node ./scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs +assert_json_contract_true_keys \ + "${npm_json}" \ + NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS + +summary_args=( + --standalone-json "${standalone_json}" + --npm-json "${npm_json}" + --official-surface-json "${official_surface_json}" + --official-no-active-json "${official_no_active_json}" + --official-raw-timeout-json "${official_raw_timeout_json}" + --official-fixture-set-json "${official_fixture_set_json}" + --official-fixture-coverage-json "${official_fixture_coverage_json}" + --official-fixture-ingest-json "${official_fixture_ingest_json}" + --ocu-candidate-ingest-json "${ocu_candidate_ingest_json}" + --official-capture-preflight-json "${official_capture_preflight_json}" + --ocu-pairing-preflight-json "${ocu_pairing_preflight_json}" + --baseline-audit-targets-json "${baseline_audit_targets_json}" + --baseline-contract-json "${baseline_contract_json}" + --matrix-jsonl "${matrix_jsonl}" + --screenshot-jsonl "${screenshot_jsonl}" + --action-jsonl "${action_jsonl}" +) +if [[ "${require_official_golden}" == "1" ]]; then + summary_args+=(--require-official-golden) +fi +set +e +if [[ -n "${summary_json}" ]]; then + mkdir -p "$(dirname "${summary_json}")" + ./scripts/build-record-and-replay-baseline-summary.py "${summary_args[@]}" | tee "${generated_summary_json}" "${summary_json}" +else + ./scripts/build-record-and-replay-baseline-summary.py "${summary_args[@]}" | tee "${generated_summary_json}" +fi +summary_status="${PIPESTATUS[0]}" +set -e + +audit_args=("${generated_summary_json}") +if [[ "${require_official_golden}" == "1" ]]; then + audit_args+=(--require-official-golden) +fi +set +e +./scripts/check-record-and-replay-baseline-summary.py "${audit_args[@]}" >&2 +audit_status="$?" +set -e + +if [[ "${summary_status}" != "0" ]]; then + exit "${summary_status}" +fi +exit "${audit_status}" diff --git a/scripts/scaffold-event-stream-skill.py b/scripts/scaffold-event-stream-skill.py new file mode 100755 index 00000000..2059e74c --- /dev/null +++ b/scripts/scaffold-event-stream-skill.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import importlib.util +import json +import pathlib +import re +import shutil +import subprocess +import sys +from typing import Any + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +SUMMARY_SCRIPT = REPO_ROOT / "scripts/summarize-event-stream-recording.py" +VALIDATION_SCRIPT = REPO_ROOT / "scripts/validate-event-stream-recording.py" +SKILL_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{1,62}[a-z0-9]$") +PATH_KEYS = { + "sessionDir", + "metadataPath", + "eventsPath", +} + + +def load_validation_module(): + spec = importlib.util.spec_from_file_location( + "event_stream_recording_validator", + VALIDATION_SCRIPT, + ) + if spec is None or spec.loader is None: + raise ValueError(f"could not load validator script: {VALIDATION_SCRIPT}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Create a reusable skill draft from a Record & Replay event-stream recording." + ) + parser.add_argument( + "recording", + type=pathlib.Path, + help="Session directory, metadata.json, session.json, or events.jsonl.", + ) + parser.add_argument( + "--skill-name", + required=True, + help="Generated skill name, using lowercase letters, numbers, and hyphens.", + ) + parser.add_argument( + "--description", + help="Skill description. Defaults to a generic Record & Replay replay description.", + ) + parser.add_argument( + "--output-dir", + type=pathlib.Path, + required=True, + help="Directory to create. The directory must not exist unless --overwrite is set.", + ) + parser.add_argument( + "--overwrite", + action="store_true", + help="Replace an existing output directory.", + ) + parser.add_argument( + "--include-text", + action="store_true", + help=( + "Include raw AX value/selectedText summary fields. Use only with deliberately " + "sanitized recordings." + ), + ) + return parser.parse_args() + + +def run_summary(recording: pathlib.Path, include_text: bool) -> dict[str, Any]: + command = [ + sys.executable, + str(SUMMARY_SCRIPT), + "--require-action", + ] + if include_text: + command.append("--include-text") + command.append(str(recording)) + completed = subprocess.run( + command, + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + payload = completed.stdout if completed.returncode == 0 else completed.stderr + try: + summary = json.loads(payload) + except json.JSONDecodeError as error: + raise ValueError(f"summary command did not return JSON: {error}") from error + if not isinstance(summary, dict): + raise ValueError("summary command returned non-object JSON") + if not summary.get("ok"): + errors = summary.get("errors") if isinstance(summary.get("errors"), list) else [] + raise ValueError("; ".join(str(error) for error in errors) or "recording summary failed") + return summary + + +def run_validation(recording: pathlib.Path) -> dict[str, Any]: + module = load_validation_module() + result = module.validate( + recording, + strict_ocu=False, + required_event_types=[], + require_skill_draft=True, + ) + if not isinstance(result, dict): + raise ValueError("validation command returned non-object JSON") + if not result.get("ok"): + errors = result.get("errors") if isinstance(result.get("errors"), list) else [] + reasons = ( + result.get("skillDraftReasons") + if isinstance(result.get("skillDraftReasons"), list) + else [] + ) + message_parts = [str(item) for item in [*errors, *reasons] if str(item)] + raise ValueError("; ".join(dict.fromkeys(message_parts)) or "recording validation failed") + return result + + +def sanitize_summary(value: Any) -> Any: + if isinstance(value, dict): + sanitized: dict[str, Any] = {} + for key, child in value.items(): + if key in PATH_KEYS: + sanitized[key] = f"" + continue + if key == "screenshotPaths" and isinstance(child, list): + sanitized[key] = [ + pathlib.Path(item).name if isinstance(item, str) else item + for item in child + ] + continue + sanitized[key] = sanitize_summary(child) + return sanitized + if isinstance(value, list): + return [sanitize_summary(child) for child in value] + return value + + +def frontmatter_value(value: str) -> str: + return value.replace("\n", " ").strip() + + +def title_from_skill_name(skill_name: str) -> str: + return " ".join(part.capitalize() for part in skill_name.split("-")) + + +def target_description(element: dict[str, Any]) -> str: + target_bits = [] + for key in ["role", "title", "label", "description"]: + raw = element.get(key) + if isinstance(raw, str) and raw: + target_bits.append(f"{key}={raw!r}") + return ", ".join(target_bits) if target_bits else "the observed target" + + +def action_description(action: dict[str, Any]) -> str: + action_type = action.get("type") + line = action.get("line") + prefix = f"Line {line}: " if isinstance(line, int) else "" + element = action.get("element") if isinstance(action.get("element"), dict) else {} + window = action.get("window") if isinstance(action.get("window"), dict) else {} + target = target_description(element) + app = window.get("appName") or window.get("bundleIdentifier") or "the recorded app" + + if action_type == "mouse.click": + return f"{prefix}Click {target} in {app}." + if action_type == "mouse.context_menu": + return f"{prefix}Open the context menu on {target} in {app}." + if action_type == "mouse.drag": + return f"{prefix}Drag from the recorded start location to end location in {app}." + if action_type == "keyboard.text_input": + text_length = action.get("textLength") + if isinstance(text_length, int): + return f"{prefix}Enter user-provided text into {target} ({text_length} recorded characters)." + return f"{prefix}Enter user-provided text into {target}." + if action_type == "keyboard.submit": + return f"{prefix}Submit the current focused control in {app}." + if action_type == "keyboard.shortcut": + key = action.get("key") + modifiers = action.get("modifiers") + chord = "+".join([*(modifiers if isinstance(modifiers, list) else []), str(key)]) + return f"{prefix}Press shortcut `{chord}` in {app}." + if action_type == "terminal.value_changed": + return f"{prefix}Observe terminal output/state change; keep command/output values sanitized." + if action_type == "selection.changed": + return f"{prefix}Use the recorded selection change as context, but do not rely on stale selected text." + if action_type == "experimentalRawEvents": + if action.get("reason") == "scrollWheel": + return f"{prefix}Scroll in {app} using the recorded wheel direction; re-check the visible state after scrolling." + return f"{prefix}Replay raw input evidence in {app} only after inspecting the original `events.jsonl`." + return f"{prefix}Replay `{action_type}` using the recorded context." + + +def input_description(action: dict[str, Any]) -> str | None: + action_type = action.get("type") + element = action.get("element") if isinstance(action.get("element"), dict) else {} + target = target_description(element) + + if action_type == "keyboard.text_input": + text_length = action.get("textLength") + if isinstance(text_length, int): + return ( + f"Runtime text for {target} ({text_length} recorded characters; " + "recorded literal text is omitted by default)." + ) + return f"Runtime text for {target}; recorded literal text is omitted by default." + if action_type == "selection.changed": + return "Current selection or selected content semantics if the workflow depends on the recorded selection." + return None + + +def runtime_input_description(runtime_input: dict[str, Any]) -> str: + kind = str(runtime_input.get("kind") or "value") + target = target_description( + runtime_input.get("target") if isinstance(runtime_input.get("target"), dict) else {} + ) + if isinstance(runtime_input.get("textLength"), int): + suffix = f" ({runtime_input['textLength']} recorded characters; use a fresh runtime value)." + elif isinstance(runtime_input.get("selectedTextLength"), int): + suffix = ( + f" ({runtime_input['selectedTextLength']} recorded selected characters; " + "confirm current semantics)." + ) + else: + suffix = "." + sensitivity = " Treat this as sensitive input." if runtime_input.get("sensitive") is True else "" + + if kind == "text": + return f"Runtime text for {target}{suffix}{sensitivity}" + if kind == "selection": + return f"Runtime selection or selected-content meaning for {target}{suffix}" + return f"Runtime {kind} for {target}{suffix}" + + +def safety_signal_description(signal: dict[str, Any]) -> str: + source_event_type = signal.get("sourceEventType") or "recorded action" + reason = signal.get("reason") or "confirmationRequired" + target = target_description(signal.get("target") if isinstance(signal.get("target"), dict) else {}) + line = signal.get("line") + prefix = f"Line {line}: " if isinstance(line, int) else "" + return ( + f"{prefix}`{source_event_type}` matched {reason} on {target}; " + "ask for explicit confirmation before replaying this step." + ) + + +def summary_limit_descriptions(summary_limits: dict[str, Any]) -> list[str]: + if summary_limits.get("hasTruncatedSummary") is not True: + return [] + omitted_counts = ( + summary_limits.get("omittedCounts") + if isinstance(summary_limits.get("omittedCounts"), dict) + else {} + ) + items = [] + for key, value in sorted(omitted_counts.items()): + if isinstance(value, int) and value > 0: + items.append( + f"{value} {key} item(s) were omitted from `references/recording-summary.json`; " + "inspect the original `events.jsonl` before finalizing replay steps." + ) + return items + + +def markdown_list(items: list[str], fallback: str) -> str: + if not items: + return f"- {fallback}" + return "\n".join(f"- {item}" for item in items) + + +def render_skill(skill_name: str, description: str, summary: dict[str, Any]) -> str: + windows = summary.get("windows") if isinstance(summary.get("windows"), list) else [] + window_items = [] + for window in windows[:10]: + if not isinstance(window, dict): + continue + app = window.get("appName") or window.get("bundleIdentifier") or "Unknown app" + title = window.get("windowTitle") + window_items.append(f"{app}" + (f" / {title}" if title else "")) + + actions = summary.get("actionSequence") if isinstance(summary.get("actionSequence"), list) else [] + action_items = [ + action_description(action) + for action in actions[:30] + if isinstance(action, dict) + ] + runtime_inputs = summary.get("runtimeInputs") if isinstance(summary.get("runtimeInputs"), list) else [] + input_items = [ + runtime_input_description(runtime_input) + for runtime_input in runtime_inputs[:30] + if isinstance(runtime_input, dict) + ] + if not input_items: + input_items = [ + item + for action in actions[:30] + if isinstance(action, dict) + for item in [input_description(action)] + if item is not None + ] + warnings = summary.get("warnings") if isinstance(summary.get("warnings"), list) else [] + evidence = summary.get("skillEvidence") if isinstance(summary.get("skillEvidence"), dict) else {} + readiness = summary.get("skillReadiness") if isinstance(summary.get("skillReadiness"), dict) else {} + readiness_reasons = readiness.get("reasons") if isinstance(readiness.get("reasons"), list) else [] + safety_signals = summary.get("safetySignals") if isinstance(summary.get("safetySignals"), list) else [] + safety_items = [ + safety_signal_description(signal) + for signal in safety_signals[:20] + if isinstance(signal, dict) + ] + summary_limits = summary.get("summaryLimits") if isinstance(summary.get("summaryLimits"), dict) else {} + summary_limit_items = summary_limit_descriptions(summary_limits) + + return f"""--- +name: {frontmatter_value(skill_name)} +description: {frontmatter_value(description)} +--- + +# {title_from_skill_name(skill_name)} + +## Purpose + +Replay or adapt the workflow captured with Open Computer Use Record & Replay. This is a generated draft: preserve the observed sequence, but replace user-specific values with explicit inputs before using it on real data. + +## Inputs To Confirm + +- The user's concrete goal for this replay. +- Required app/account/login state. +- Any names, paths, message text, dates, or account-specific values that were redacted or should be parameterized. +- Whether any step sends, deletes, purchases, uploads, approves, or otherwise changes external state. + +## Runtime Inputs + +{markdown_list(input_items, "No text or selection inputs were inferred. Still ask the user for any values that should vary between replays.")} + +## Workflow Readiness + +- Status: {readiness.get("status") or "unknown"} +- Can create skill draft: {str(readiness.get("canCreateSkillDraft")).lower()} +- Requires human review: {str(readiness.get("requiresHumanReview")).lower()} +- Recommended next step: {readiness.get("recommendedNextStep") or "Inspect `events.jsonl` before finalizing this skill."} + +{markdown_list([str(reason) for reason in readiness_reasons], "No readiness issues were detected by the summary helper.")} + +## Summary Limits + +{markdown_list(summary_limit_items, "No high-volume summary fields were truncated.")} + +## Recorded Context + +{markdown_list(window_items, "No stable app/window context was captured; inspect `references/recording-summary.json`.")} + +## Agent Replay Procedure + +- Before replaying UI actions, check whether a connector, API, or dedicated tool can perform the stable semantic operation more reliably than desktop automation; use Computer Use for visually dependent verification, unsupported UI interactions, or when manipulating the interface is itself the workflow. +- Start by calling `get_app_state` for the recorded app or the app the user names; verify the expected window or equivalent screen is present. +- For each observed step, resolve the current target by visible label, role, title, or neighboring text from `references/recording-summary.json`; use `element_index` actions when a semantic element is available. +- Use coordinate clicks or drags only as a last resort after refreshing the screenshot and confirming the coordinate system still matches the current window. +- For text entry, collect the runtime value from the user or task context, focus the observed input target, then use `type_text` or `set_value` according to the current app state. +- Pause and ask for explicit confirmation before sends, deletes, purchases, approvals, uploads, or other externally visible changes. + +## Replay Steps + +{markdown_list(action_items, "No high-level action sequence was captured; ask the user to re-record after fixing permissions.")} + +## Evidence + +- Event count: {summary.get("eventCount")} +- Action event count: {summary.get("actionEventCount")} +- End reason: {summary.get("endReason") or "unknown"} +- Has AX context: {str(evidence.get("hasAXContext")).lower()} +- Has target elements: {str(evidence.get("hasTargetElements")).lower()} +- Has redaction signals: {str(evidence.get("hasRedactionSignals")).lower()} +- Has safety signals: {str(evidence.get("hasSafetySignals")).lower()} + +## Warnings + +{markdown_list([str(warning) for warning in warnings], "No summary warnings were reported.")} + +## Safety + +- Do not reconstruct passwords, OTPs, API keys, terminal buffers, or other redacted values. +- Ask for confirmation before externally visible or destructive actions. +- Prefer semantic targets, labels, app names, and keyboard shortcuts over raw coordinates. +- Re-check the current app state before executing each step; recorded UI positions may drift. + +### Confirmation Signals + +{markdown_list(safety_items, "No specific confirmation-sensitive actions were detected by the summary helper. Still ask before externally visible changes.")} + +## Verification + +- Before packaging this skill, rerun `open-computer-use event-stream validate --json --strict-ocu --require-skill-draft ` on the source recording. +- Compare the final app state with the user's requested outcome, not just the recorded event sequence. +- If replay behavior depends on missing AX targets, screenshots, or redacted text, refine this draft against `events.jsonl` before installing it. +- If `summaryLimits.hasTruncatedSummary` is true, inspect the original `events.jsonl` for omitted steps before finalizing this skill. + +## Finalizing The Skill + +- Treat this directory as a skill draft, not a standalone runbook or replay plan. +- Read and follow the `skill-creator` skill before packaging or installing the skill. +- Complete the `skill-creator` workflow, including validation, before reporting that this skill was created. +- Keep reusable workflow intent, prerequisites, runtime inputs, safety confirmations, and verification steps in the skill; omit raw event logs and user-specific values. +- The final deliverable should be an actual discoverable skill directory, not only this generated Markdown draft. + +## Source + +The generated summary is stored in `references/recording-summary.json`. Treat `events.jsonl` from the original recording as the primary evidence when refining this draft. +""" + + +def write_skill(output_dir: pathlib.Path, skill_name: str, description: str, summary: dict[str, Any], overwrite: bool) -> None: + readiness = summary.get("skillReadiness") if isinstance(summary.get("skillReadiness"), dict) else {} + if readiness.get("canCreateSkillDraft") is not True: + message = readiness.get("recommendedNextStep") or ( + "Recording does not contain enough usable evidence to create a skill draft." + ) + raise ValueError(str(message)) + + if output_dir.exists(): + if not overwrite: + raise ValueError(f"output directory already exists: {output_dir}") + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) + references_dir = output_dir / "references" + references_dir.mkdir() + + sanitized = sanitize_summary(summary) + (output_dir / "SKILL.md").write_text( + render_skill(skill_name, description, sanitized), + encoding="utf-8", + ) + (references_dir / "recording-summary.json").write_text( + json.dumps(sanitized, indent=2, sort_keys=True, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + + +def main() -> int: + args = parse_args() + skill_name = args.skill_name.strip() + if not SKILL_NAME_PATTERN.fullmatch(skill_name): + print( + json.dumps( + { + "ok": False, + "error": "invalidSkillName", + "message": "skill name must use lowercase letters, numbers, and hyphens", + } + ), + file=sys.stderr, + ) + return 1 + + description = args.description or ( + f"Replay a workflow captured by Open Computer Use Record & Replay for {skill_name}." + ) + + try: + run_validation(args.recording) + summary = run_summary(args.recording, include_text=args.include_text) + write_skill(args.output_dir, skill_name, description, summary, overwrite=args.overwrite) + except ValueError as error: + print(json.dumps({"ok": False, "error": str(error)}, ensure_ascii=False), file=sys.stderr) + return 1 + + print( + json.dumps( + { + "ok": True, + "skillName": skill_name, + "outputDir": str(args.output_dir), + "skillPath": str(args.output_dir / "SKILL.md"), + "summaryPath": str(args.output_dir / "references/recording-summary.json"), + }, + indent=2, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scaffold-record-and-replay-skill-repo.py b/scripts/scaffold-record-and-replay-skill-repo.py new file mode 100755 index 00000000..1cdc3fca --- /dev/null +++ b/scripts/scaffold-record-and-replay-skill-repo.py @@ -0,0 +1,3197 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import shutil +import sys +import textwrap + +from record_and_replay_scenarios import ( + DEFAULT_RECOMMENDED_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_recipe, +) + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +SOURCE_SKILL_NAME = "open-computer-use-record-and-replay" +SOURCE_SKILL_DIR = REPO_ROOT / "skills" / SOURCE_SKILL_NAME + + +PACKAGE_SCRIPT = """#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +skills_root="${repo_root}/skills" +dist_dir="${repo_root}/dist/skills" + +if ! command -v zip >/dev/null 2>&1; then + echo "zip is required" >&2 + exit 1 +fi + +rm -rf "${dist_dir}" +mkdir -p "${dist_dir}" + +while IFS= read -r skill_md; do + skill_dir="$(dirname "${skill_md}")" + skill_name="$(basename "${skill_dir}")" + python3 - "${skill_md}" "${skill_name}" <<'PY' +import pathlib +import sys + +skill_path = pathlib.Path(sys.argv[1]) +expected_name = sys.argv[2] +content = skill_path.read_text() +if not content.startswith("---\\n"): + raise SystemExit(f"{skill_path} is missing YAML frontmatter") +try: + _prefix, frontmatter, _rest = content.split("---", 2) +except ValueError as error: + raise SystemExit(f"{skill_path} has malformed YAML frontmatter") from error + +fields = {} +for line in frontmatter.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + fields[key.strip()] = value.strip() + +if fields.get("name") != expected_name: + raise SystemExit( + f"{skill_path} frontmatter name must be {expected_name!r}, got {fields.get('name')!r}" + ) +if not fields.get("description"): + raise SystemExit(f"{skill_path} frontmatter description is required") +PY + zip_path="${dist_dir}/${skill_name}-skill.zip" + skill_path="${dist_dir}/${skill_name}.skill" + ( + cd "${skills_root}" + zip -q -r "${zip_path}" "${skill_name}" + ) + cp "${zip_path}" "${skill_path}" + echo "${zip_path}" + echo "${skill_path}" +done < <(find "${skills_root}" -mindepth 2 -maxdepth 2 -name SKILL.md -print | sort) +""" + + +CHECK_SCRIPT = """#!/usr/bin/env bash + +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"${repo_root}/scripts/package-skill.sh" +"${repo_root}/scripts/verify-package-artifact.py" +"${repo_root}/scripts/verify-manifest.py" +"${repo_root}/scripts/verify-source-baseline-summary.py" +"${repo_root}/scripts/verify-readme-handoff.py" +"${repo_root}/scripts/verify-runtime.py" +"${repo_root}/scripts/verify-skill-workflow.py" +"${repo_root}/scripts/wait-notify-contract-smoke.py" +"${repo_root}/scripts/recording-to-skill-smoke.py" +""" + + +VERIFY_PACKAGE_ARTIFACT_SCRIPT = """#!/usr/bin/env python3 + +import json +import pathlib +import sys +import zipfile + + +REQUIRED_SKILL_SNIPPETS = [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + "open-computer-use event-stream validate", + "open-computer-use event-stream scaffold-skill", + "OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH", +] + + +def require(failures, condition, message): + if not condition: + failures.append(message) + + +def parse_frontmatter(text): + if not text.startswith("---\\n"): + raise ValueError("SKILL.md is missing YAML frontmatter") + try: + _prefix, frontmatter, _rest = text.split("---", 2) + except ValueError as error: + raise ValueError("SKILL.md has malformed YAML frontmatter") from error + fields = {} + for line in frontmatter.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + fields[key.strip()] = value.strip() + return fields + + +def main(): + repo_root = pathlib.Path(__file__).resolve().parents[1] + manifest = json.loads((repo_root / "record-and-replay-skill-repo.json").read_text()) + skill_name = manifest.get("skill", {}).get("name") or "open-computer-use-record-and-replay" + dist_dir = repo_root / "dist" / "skills" + zip_path = dist_dir / f"{skill_name}-skill.zip" + skill_path = dist_dir / f"{skill_name}.skill" + expected_skill_md = f"{skill_name}/SKILL.md" + failures = [] + + require(failures, zip_path.exists(), f"missing packaged zip: {zip_path}") + require(failures, skill_path.exists(), f"missing .skill artifact: {skill_path}") + if not zip_path.exists() or not skill_path.exists(): + raise AssertionError(json.dumps({"failures": failures}, indent=2, sort_keys=True)) + + require( + failures, + zip_path.read_bytes() == skill_path.read_bytes(), + ".skill artifact must match packaged zip bytes", + ) + + try: + with zipfile.ZipFile(skill_path) as archive: + names = sorted(archive.namelist()) + require(failures, expected_skill_md in names, f"missing {expected_skill_md} in package") + unsafe = [ + name + for name in names + if name.startswith("/") + or name.startswith("../") + or "/../" in name + or not name.startswith(f"{skill_name}/") + ] + require(failures, not unsafe, f"unsafe or unexpected package paths: {unsafe}") + skill_text = archive.read(expected_skill_md).decode("utf-8") + except Exception as error: + raise AssertionError( + json.dumps( + {"failures": failures + [f"failed to read skill package: {error}"]}, + indent=2, + sort_keys=True, + ) + ) from error + + try: + frontmatter = parse_frontmatter(skill_text) + except ValueError as error: + failures.append(str(error)) + frontmatter = {} + + require( + failures, + frontmatter.get("name") == skill_name, + f"packaged SKILL.md frontmatter name must be {skill_name!r}", + ) + require( + failures, + bool(frontmatter.get("description")), + "packaged SKILL.md frontmatter description is required", + ) + missing_snippets = [snippet for snippet in REQUIRED_SKILL_SNIPPETS if snippet not in skill_text] + require( + failures, + not missing_snippets, + f"packaged SKILL.md missing handoff snippets: {missing_snippets}", + ) + + if failures: + raise AssertionError(json.dumps({"failures": failures}, indent=2, sort_keys=True)) + + print( + json.dumps( + { + "ok": True, + "checkedPackageArtifact": True, + "checkedPackageSkillArchive": True, + "checkedPackageSkillFrontmatter": True, + "checkedPackageSkillHandoff": True, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +VERIFY_MANIFEST_SCRIPT = """#!/usr/bin/env python3 + +import json +import pathlib +import sys + + +EXPECTED_TOOLS = [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", +] +EXPECTED_REQUIRED_SCENARIOS = ["simple-action-stop"] +EXPECTED_RECOMMENDED_SCENARIOS = [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", +] +EXPECTED_STRICT_MISSING_AUDIT = ( + "scripts/check-record-and-replay-baseline-summary.py " + "dist/record-and-replay-official-golden-gate-summary.json " + "--allow-strict-official-golden-missing" +) + + +def require(failures, condition, message): + if not condition: + failures.append(message) + + +def main(): + repo_root = pathlib.Path(__file__).resolve().parents[1] + manifest_path = repo_root / "record-and-replay-skill-repo.json" + manifest = json.loads(manifest_path.read_text()) + failures = [] + + require( + failures, + manifest.get("kind") == "open-computer-use-record-and-replay-thin-skill-repo", + "unexpected repository kind", + ) + require(failures, manifest.get("runtimeDependency") == "open-computer-use", "runtime dependency drifted") + + official = manifest.get("officialEvidence", {}) + source_summary_path = repo_root / "evidence" / "source-baseline-summary.json" + source_summary = {} + if source_summary_path.exists(): + try: + source_summary = json.loads(source_summary_path.read_text()) + except Exception: + source_summary = {} + source_golden_complete = ( + (source_summary.get("status") or {}).get("officialSuccessfulRecordingGoldenComplete") + ) + require( + failures, + official.get("baselineVersion") == "record-and-replay/1.0.857", + "official baseline version drifted", + ) + require( + failures, + isinstance(official.get("hasSuccessfulRecordingGolden"), bool), + "standalone manifest official successful recording golden state must be boolean", + ) + if isinstance(source_golden_complete, bool): + require( + failures, + official.get("hasSuccessfulRecordingGolden") == source_golden_complete, + "standalone manifest official successful recording golden state drifted from source baseline summary", + ) + require( + failures, + official.get("requiredSuccessfulRecordingScenarios") == EXPECTED_REQUIRED_SCENARIOS, + "required official scenario list drifted", + ) + require( + failures, + official.get("recommendedSuccessfulRecordingScenarios") == EXPECTED_RECOMMENDED_SCENARIOS, + "recommended official scenario list drifted", + ) + require( + failures, + sorted((official.get("scenarioRecipes") or {}).keys()) == sorted(EXPECTED_RECOMMENDED_SCENARIOS), + "scenario recipe list drifted", + ) + + baseline_checks = official.get("sourceRepoBaselineChecks", {}) + fixture_set_gate = baseline_checks.get("officialFixtureSetGate", {}) + fixture_ingest = baseline_checks.get("officialFixtureIngest", {}) + fixture_compare_policy = fixture_set_gate.get("sameScenarioComparePolicy", {}) + capture_preflight = baseline_checks.get("officialGoldenCapturePreflight", {}) + require( + failures, + fixture_set_gate.get("check") == "official-fixture-set-smoke", + "official fixture set gate smoke drifted", + ) + require( + failures, + fixture_compare_policy.get("requiresAxDiffEvidence") is True, + "official fixture set AX diff evidence policy missing", + ) + require( + failures, + fixture_compare_policy.get("requiresSameAxDiffMarkers") is True, + "official fixture set AX diff marker policy missing", + ) + require( + failures, + fixture_compare_policy.get("requiresSameSuppressedEventSequence") is True, + "official fixture set suppressed event sequence policy missing", + ) + require( + failures, + fixture_compare_policy.get("requiresSameSuppressedSchema") is True, + "official fixture set suppressed schema policy missing", + ) + require( + failures, + fixture_ingest.get("check") == "official-fixture-ingest-smoke", + "official fixture ingest smoke drifted", + ) + fixture_ingest_evidence = fixture_ingest.get("requiredEvidence") or [] + require( + failures, + "checkedOfficialSessionDirectoryPathHandoff" in fixture_ingest_evidence, + "official sessionDirectoryPath handoff evidence missing", + ) + require( + failures, + capture_preflight.get("check") == "official-golden-capture-preflight-smoke", + "official capture preflight smoke drifted", + ) + capture_preflight_evidence = capture_preflight.get("requiredEvidence") or [] + require( + failures, + "checkedOfficialCapturePacketInputSemanticGuard" in capture_preflight_evidence, + "official capture semantic guard evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketSetContractManifest" in capture_preflight_evidence, + "official capture packet set contract manifest evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketPostCaptureWorkflow" in capture_preflight_evidence, + "official capture packet post-capture workflow evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketWorkflowVerifier" in capture_preflight_evidence, + "official capture packet workflow verifier evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketSetPostCaptureWorkflow" in capture_preflight_evidence, + "official capture packet set post-capture workflow evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketSetWorkflowVerifier" in capture_preflight_evidence, + "official capture packet set workflow verifier evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketStrictAuditHandoff" in capture_preflight_evidence, + "official capture packet strict audit handoff evidence missing", + ) + require( + failures, + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + in capture_preflight_evidence, + "official capture packet strict expected-failure audit handoff evidence missing", + ) + + audit = official.get("sourceRepoBaselineAudit", {}) + require( + failures, + audit.get("usableBaseline") == "make record-and-replay-baseline-audit", + "source repo baseline audit command drifted", + ) + require( + failures, + audit.get("strictOfficialGoldenGate") == "make record-and-replay-official-golden-gate-audit", + "strict official golden audit command drifted", + ) + require( + failures, + audit.get("auditTargetDryRunSmoke") == "make record-and-replay-baseline-audit-targets-smoke", + "audit target dry-run smoke command drifted", + ) + require( + failures, + audit.get("baselineSummaryArtifact") == "dist/record-and-replay-baseline-summary.json", + "baseline summary artifact drifted", + ) + require( + failures, + audit.get("strictOfficialGoldenSummaryArtifact") + == "dist/record-and-replay-official-golden-gate-summary.json", + "strict official golden summary artifact drifted", + ) + require( + failures, + audit.get("baselineSummaryEnvVar") == "RNR_BASELINE_SUMMARY_JSON", + "baseline summary env var drifted", + ) + require( + failures, + audit.get("strictOfficialGoldenSummaryEnvVar") == "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON", + "strict official golden summary env var drifted", + ) + require( + failures, + audit.get("strictOfficialGoldenExpectedFailureAudit") == EXPECTED_STRICT_MISSING_AUDIT, + "strict expected-failure audit command drifted", + ) + require( + failures, + audit.get("verifiesSummaryArtifactSeparation") is True, + "summary artifact separation evidence missing", + ) + require( + failures, + audit.get("verifiesSummaryEnvVarIsolation") is True, + "summary env var isolation evidence missing", + ) + + mcp = manifest.get("mcpServer", {}) + require(failures, mcp.get("tools") == EXPECTED_TOOLS, "official MCP tool list drifted") + require(failures, mcp.get("requiresObjectParams") is True, "object params guard missing") + require(failures, mcp.get("requiresStringToolName") is True, "string tool name guard missing") + require(failures, mcp.get("requiresObjectArguments") is True, "object arguments guard missing") + require(failures, mcp.get("rejectsUnexpectedArguments") is True, "unexpected arguments guard missing") + require( + failures, + mcp.get("rejectedRequestsDoNotCreateSessionFiles") is True, + "malformed MCP request side-effect guard missing", + ) + require( + failures, + mcp.get("noActiveResponse", {}).get("event_stream_status") + == {"isRecording": False, "maxDurationSeconds": 1800}, + "no-active status response drifted", + ) + require( + failures, + mcp.get("noActiveResponse", {}).get("event_stream_stop") + == {"isRecording": False, "maxDurationSeconds": 1800}, + "no-active stop response drifted", + ) + + extension = manifest.get("extensionLayer", {}) + wait_notify = extension.get("waitNotify", {}) + require( + failures, + extension.get("officialCompatibleMcpSurface") is False, + "extension layer must remain outside official MCP surface", + ) + require( + failures, + "OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH" + in (wait_notify.get("environmentVariables") or []), + "notify suppressed events path handoff missing", + ) + require( + failures, + wait_notify.get("callbackFailureMakesCliFail") is True, + "notify callback failure contract missing", + ) + require( + failures, + wait_notify.get("callbackTimeoutMakesCliFail") is True, + "notify callback timeout contract missing", + ) + + recording_to_skill = manifest.get("recordingToSkill", {}) + strict_validation = recording_to_skill.get("strictValidation", {}) + require( + failures, + strict_validation.get("requiresDeclaredHandoffPaths") + == ["metadataPath", "sessionPath", "eventsPath", "suppressedEventsPath"], + "recording strict validation handoff paths drifted", + ) + require( + failures, + strict_validation.get("requiresScreenshotPathsInsideSession") is True, + "screenshot containment gate missing", + ) + require( + failures, + recording_to_skill.get("rejectsCancelledRecordings") is True, + "cancelled recording rejection contract missing", + ) + + checks = manifest.get("checks", {}) + require(failures, checks.get("manifestContract") == "scripts/verify-manifest.py", "manifest self-check missing") + require( + failures, + checks.get("packageArtifact") == "scripts/verify-package-artifact.py", + "package artifact self-check missing", + ) + require( + failures, + checks.get("sourceBaselineSummaryEvidence") == "scripts/verify-source-baseline-summary.py", + "source baseline summary evidence check missing", + ) + require( + failures, + checks.get("readmeHandoffContract") == "scripts/verify-readme-handoff.py", + "README handoff self-check missing", + ) + require(failures, checks.get("runtimeContract") == "scripts/verify-runtime.py", "runtime check missing") + require(failures, checks.get("skillWorkflow") == "scripts/verify-skill-workflow.py", "skill workflow check missing") + require(failures, checks.get("waitNotifyContractSmoke") == "scripts/wait-notify-contract-smoke.py", "wait/notify check missing") + require(failures, checks.get("recordingToSkillSmoke") == "scripts/recording-to-skill-smoke.py", "recording-to-skill check missing") + require(failures, checks.get("selfCheck") == "scripts/check.sh", "self-check entry missing") + + if failures: + raise AssertionError(json.dumps({"failures": failures}, indent=2, sort_keys=True)) + + print( + json.dumps( + { + "ok": True, + "checkedManifestContract": True, + "checkedPackageArtifactCheck": True, + "checkedOfficialEvidenceAuditManifest": True, + "checkedStrictExpectedFailureAudit": True, + "checkedOfficialFixtureSetComparePolicy": True, + "checkedSourceBaselineSummaryEvidenceCheck": True, + "checkedMcpNoArgSurfaceContract": True, + "checkedWaitNotifyExtensionContract": True, + "checkedRecordingToSkillContract": True, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +VERIFY_SOURCE_BASELINE_SUMMARY_SCRIPT = """#!/usr/bin/env python3 + +import json +import pathlib +import sys + + +EXPECTED_REQUIRED_GAPS = ["simple-action-stop"] +EXPECTED_RECOMMENDED_GAPS = [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", +] +EXPECTED_SUMMARY_EVIDENCE_PATH = "evidence/source-baseline-summary.json" + + +def require(failures, condition, message): + if not condition: + failures.append(message) + + +def bool_at(root, *path): + value = root + for key in path: + if not isinstance(value, dict): + return False + value = value.get(key) + return value is True + + +def official_golden_state(status): + missing_required = status.get("missingRequiredOfficialSuccessfulRecordingScenarios") or [] + not_ready_required = status.get("notReadyRequiredOfficialSuccessfulRecordingScenarios") or [] + coverage_errors = status.get("officialFixtureCoverageErrors") or [] + missing_recommended = status.get("missingRecommendedOfficialSuccessfulRecordingScenarios") or [] + + current_required_gap = ( + status.get("officialGoldenGatePassed") is False + and status.get("officialSuccessfulRecordingGoldenComplete") is False + and status.get("officialSuccessfulRecordingEquivalenceReady") is False + and status.get("requiresOfficialGoldenCapture") is True + and missing_required == EXPECTED_REQUIRED_GAPS + ) + completed_equivalence = ( + status.get("officialGoldenGatePassed") is True + and status.get("officialSuccessfulRecordingGoldenComplete") is True + and status.get("officialSuccessfulRecordingEquivalenceReady") is True + and status.get("requiresOfficialGoldenCapture") is False + and missing_required == [] + and not_ready_required == [] + and coverage_errors == [] + ) + current_recommended_gap = ( + sorted(missing_recommended) == sorted(EXPECTED_RECOMMENDED_GAPS) + ) + return current_required_gap, current_recommended_gap, completed_equivalence + + +def main(): + repo_root = pathlib.Path(__file__).resolve().parents[1] + manifest = json.loads((repo_root / "record-and-replay-skill-repo.json").read_text()) + evidence_path = repo_root / EXPECTED_SUMMARY_EVIDENCE_PATH + failures = [] + + source_audit = ( + manifest.get("officialEvidence", {}) + .get("sourceRepoBaselineAudit", {}) + ) + require( + failures, + source_audit.get("copiedBaselineSummaryEvidence") + == EXPECTED_SUMMARY_EVIDENCE_PATH, + "copied baseline summary evidence path drifted", + ) + require( + failures, + manifest.get("checks", {}).get("sourceBaselineSummaryEvidence") + == "scripts/verify-source-baseline-summary.py", + "source baseline summary check not declared", + ) + require( + failures, + evidence_path.exists(), + "missing copied source baseline summary evidence", + ) + if evidence_path.exists(): + summary = json.loads(evidence_path.read_text()) + status = summary.get("status", {}) + evidence = summary.get("evidence", {}) + require(failures, summary.get("baseline") == "record-and-replay", "baseline name drifted") + require(failures, summary.get("ok") is True, "baseline summary must be usable in default mode") + require(failures, status.get("usableBaseline") is True, "usable baseline evidence missing") + require( + failures, + status.get("standaloneRepoBaselineReady") is True, + "standalone repo baseline readiness missing", + ) + require( + failures, + status.get("officialGoldenRequirementSatisfied") is True, + "default official golden requirement policy drifted", + ) + ( + current_required_gap, + current_recommended_gap, + completed_equivalence, + ) = official_golden_state(status) + require( + failures, + current_required_gap or completed_equivalence, + "official golden state must be either current required gap or completed equivalence", + ) + if current_required_gap: + require( + failures, + current_recommended_gap, + "recommended official golden gap drifted", + ) + require( + failures, + bool_at(evidence, "officialFixtureSetGate", "checkedAxDiffComparisonPolicy"), + "official fixture set AX diff compare policy evidence missing", + ) + require( + failures, + bool_at(evidence, "officialFixtureSetGate", "checkedSuppressedStreamComparisonPolicy"), + "official fixture set suppressed stream compare policy evidence missing", + ) + require( + failures, + bool_at(evidence, "fixtureIngestPipelines", "checkedOfficialSessionDirectoryPathHandoff"), + "official sessionDirectoryPath handoff evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketSetContractManifest"), + "capture packet set contract manifest evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketPostCaptureWorkflow"), + "capture packet post-capture workflow evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketWorkflowVerifier"), + "capture packet workflow verifier evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketSetPostCaptureWorkflow"), + "capture packet set post-capture workflow evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketSetWorkflowVerifier"), + "capture packet set workflow verifier evidence missing", + ) + require( + failures, + bool_at(evidence, "preflightPipelines", "checkedOfficialCapturePacketStrictAuditHandoff"), + "capture packet strict audit handoff evidence missing", + ) + require( + failures, + bool_at( + evidence, + "preflightPipelines", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ), + "capture packet strict expected-failure audit handoff evidence missing", + ) + require( + failures, + bool_at(evidence, "standaloneSkillRepo", "checkedOfficialFixtureSetComparePolicyManifest"), + "source standalone fixture-set compare policy manifest evidence missing", + ) + require( + failures, + bool_at(evidence, "npmStagedSkillRepo", "checkedOfficialFixtureSetComparePolicyManifest"), + "npm staged fixture-set compare policy manifest evidence missing", + ) + require( + failures, + bool_at(evidence, "standaloneSkillRepo", "checkedGeneratedRepoSelfCheck"), + "source standalone generated repo self-check evidence missing", + ) + require( + failures, + bool_at(evidence, "npmStagedSkillRepo", "checkedGeneratedRepoSelfCheck"), + "npm staged generated repo self-check evidence missing", + ) + + if failures: + raise AssertionError(json.dumps({"failures": failures}, indent=2, sort_keys=True)) + + print( + json.dumps( + { + "ok": True, + "checkedSourceBaselineSummaryEvidence": True, + "checkedSourceBaselineSummaryDefaultUsable": True, + "checkedSourceBaselineSummaryOfficialGoldenState": True, + "checkedSourceBaselineSummaryOfficialGoldenGap": current_required_gap, + "checkedSourceBaselineSummaryOfficialGoldenComplete": completed_equivalence, + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": True, + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": True, + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": True, + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedSourceBaselineSummaryStandaloneEvidence": True, + "evidencePath": EXPECTED_SUMMARY_EVIDENCE_PATH, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +VERIFY_SKILL_WORKFLOW_SCRIPT = """#!/usr/bin/env python3 + +import json +import pathlib +import sys + + +REQUIRED_SNIPPETS = [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + "Do not add cancel, wait, callback, webhook, or extra arguments to this MCP surface.", + "Call `event_stream_start` once.", + "If it reports an already active recording, do not start another one.", + "End the turn and let the user perform the workflow.", + "recording can last up to 30 minutes", + "Do not poll while the user is recording.", + "Use `event_stream_status` only when the user asks for status or returns after recording", + "do not use it to wait for completion", + "When the user says they are done, call `event_stream_stop`.", + "read `eventsPath`, `metadataPath`, and, when present, `sessionPath`", + "The MCP server does not expose event-stream contents directly.", + "Treat `events.jsonl` as the primary evidence.", + "open-computer-use event-stream validate --json --strict-ocu --require-skill-draft ", + "open-computer-use event-stream validate --json --require-skill-draft ", + "open-computer-use event-stream summarize --json ", + "open-computer-use event-stream scaffold-skill --json ", + "If the user says they clicked Discard or cancelled recording, do not call `event_stream_stop` again.", + "Before finalizing the skill, read and follow the `skill-creator` skill.", + "Use `runtimeInputs`", + "Use `safetySignals`", + "If `summaryLimits.hasTruncatedSummary=true`", + "session.started` is missing, duplicated, or not the first event", + "session.ended` appears more than once, or whose `session.ended` is not the final event", + "## Independent Wait / Notify Integration", + "open-computer-use event-stream wait --json --session-id --notify-command", + "`wait --json` adds `waitTimedOut` and `waitSessionMatched`.", + "`--notify-command` receives the final status JSON on stdin", + "OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON", + "OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH", + "Keep this listener path outside the official-compatible MCP tool surface.", +] + +FORBIDDEN_SNIPPETS = [ + "event_stream_cancel", + "event_stream_wait", +] + + +def main(): + repo_root = pathlib.Path(__file__).resolve().parents[1] + skill_paths = sorted((repo_root / "skills").glob("*/SKILL.md")) + if len(skill_paths) != 1: + raise AssertionError(f"expected exactly one skill entrypoint, found {skill_paths!r}") + skill_path = skill_paths[0] + text = skill_path.read_text() + + missing = [snippet for snippet in REQUIRED_SNIPPETS if snippet not in text] + forbidden = [snippet for snippet in FORBIDDEN_SNIPPETS if snippet in text] + if missing or forbidden: + raise AssertionError( + json.dumps( + { + "missingRequiredSnippets": missing, + "forbiddenSnippets": forbidden, + "skillPath": str(skill_path.relative_to(repo_root)), + }, + sort_keys=True, + ) + ) + + print( + json.dumps( + { + "ok": True, + "checkedSkillWorkflow": True, + "checkedStatusNotUsedAsWaitLoopGuard": True, + "checkedMcpNoDirectEventContentsGuard": True, + "skillPath": str(skill_path.relative_to(repo_root)), + "requiredSnippetCount": len(REQUIRED_SNIPPETS), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +VERIFY_README_HANDOFF_SCRIPT = """#!/usr/bin/env python3 + +import json +import pathlib +import sys + + +REQUIRED_SNIPPETS = [ + "npm i -g open-computer-use", + "open-computer-use install-codex-record-and-replay-mcp", + "Python 3 must be available as `python3`", + "Set `OPEN_COMPUTER_USE_CLI=/path/to/open-computer-use`", + "The Record & Replay MCP server must expose only these official-compatible tools", + "event_stream_start", + "event_stream_status", + "event_stream_stop", + "OCU extension commands such as `event-stream wait`, `validate`, `summarize`, and", + "record-and-replay-event-stream-surface-1.0.857.json", + "record-and-replay-official-no-active-status-stop-1.0.857.json", + "record-and-replay-official-raw-start-timeout-1.0.857.json", + "hostless official raw `event_stream_start/status/stop`", + "must not be treated as a successful recording", + "official fixture set gate also requires same-scenario", + "AX diff evidence, AX diff marker alignment", + "suppressed event sequence, and suppressed schema comparison", + "make record-and-replay-baseline-audit", + "make record-and-replay-official-golden-gate-audit", + "scripts/check-record-and-replay-baseline-summary.py", + "--allow-strict-official-golden-missing", + "dist/record-and-replay-baseline-summary.json", + "evidence/source-baseline-summary.json", + "dist/record-and-replay-official-golden-gate-summary.json", + "The minimum required official successful recording scenario is", + "`simple-action-stop`. The recommended calibration set is", + "`simple-action-stop`, `keyboard-input-stop`, `drag-stop`, `cancel`, and `timeout`.", + "officialEvidence.scenarioRecipes", + "Successful recording fixtures are still required before claiming official", + "event schema, AX diff, screenshot, or timeout endReason equivalence", + "open-computer-use event-stream validate --json --strict-ocu --require-skill-draft ", + "open-computer-use event-stream validate --json --require-skill-draft ", + "open-computer-use event-stream scaffold-skill --json ", + "Cancelled, incomplete, or malformed recordings must not be used to", + "create or update a skill.", + "open-computer-use event-stream wait --json --session-id --notify-command", + "Keep this listener path outside the official-compatible MCP tool", + "./scripts/recording-lifecycle-smoke.py", + "./scripts/verify-runtime.py", + "scripts/verify-source-baseline-summary.py", + "./scripts/verify-skill-workflow.py", +] + + +def main(): + repo_root = pathlib.Path(__file__).resolve().parents[1] + readme_path = repo_root / "README.md" + text = readme_path.read_text() + + missing = [snippet for snippet in REQUIRED_SNIPPETS if snippet not in text] + if missing: + raise AssertionError( + json.dumps( + { + "missingRequiredSnippets": missing, + "readmePath": str(readme_path.relative_to(repo_root)), + }, + sort_keys=True, + ) + ) + + print( + json.dumps( + { + "ok": True, + "checkedReadmeHandoffContract": True, + "checkedReadmeOfficialEvidenceHandoff": True, + "checkedReadmeOfficialGoldenGap": True, + "checkedReadmeWaitNotifyBoundary": True, + "requiredSnippetCount": len(REQUIRED_SNIPPETS), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +RECORDING_TO_SKILL_SMOKE_SCRIPT = """#!/usr/bin/env python3 + +import json +import os +import pathlib +import subprocess +import sys +import tempfile + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\\n") + + +def write_jsonl(path, records): + path.write_text("".join(json.dumps(record, sort_keys=True) + "\\n" for record in records)) + + +def main(): + cli = os.environ.get("OPEN_COMPUTER_USE_CLI", "open-computer-use") + env = {**os.environ, "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1"} + with tempfile.TemporaryDirectory(prefix="record-and-replay-skill-smoke-") as tmp: + root = pathlib.Path(tmp) + session_dir = root / "session" + session_dir.mkdir() + events = [ + { + "type": "session.started", + "sessionId": "synthetic-skill-smoke", + "timestamp": "2026-06-27T00:00:00Z", + }, + { + "type": "mouse.click", + "timestamp": "2026-06-27T00:00:01Z", + "app": {"name": "Example"}, + "window": {"title": "Example Window"}, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Create", + "label": "Create", + }, + }, + { + "type": "session.ended", + "sessionId": "synthetic-skill-smoke", + "timestamp": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_stopped", + }, + ] + metadata = { + "sessionId": "synthetic-skill-smoke", + "state": "stopped", + "active": False, + "startTime": "2026-06-27T00:00:00Z", + "endTime": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": 0, + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "eventsPath": "events.jsonl", + "suppressedEventsPath": "suppressed.jsonl", + } + write_jsonl(session_dir / "events.jsonl", events) + write_json(session_dir / "metadata.json", metadata) + write_json(session_dir / "session.json", metadata) + (session_dir / "suppressed.jsonl").write_text("") + + validation = subprocess.run( + [ + cli, + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-skill-draft", + str(session_dir / "metadata.json"), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + validation_payload = json.loads(validation.stdout) + if validation_payload.get("ok") is not True or validation_payload.get("skillDraftReady") is not True: + raise AssertionError(validation_payload) + declared_paths = validation_payload.get("declaredPaths") + expected_handoff_paths = [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + ] + if not isinstance(declared_paths, dict): + raise AssertionError(validation_payload) + for key in expected_handoff_paths: + evidence = declared_paths.get(key) + if not isinstance(evidence, dict) or evidence.get("exists") is not True: + raise AssertionError(validation_payload) + + external_screenshot_session_dir = root / "external-screenshot-session" + external_screenshot_session_dir.mkdir() + external_screenshot = root / "outside-session-screenshot.png" + external_screenshot.write_bytes(b"\\x89PNG") + external_screenshot_events = [ + { + "type": "session.started", + "sessionId": "synthetic-external-screenshot-smoke", + "timestamp": "2026-06-27T00:00:00Z", + }, + { + "type": "AX.focusedWindowChanged", + "sessionId": "synthetic-external-screenshot-smoke", + "timestamp": "2026-06-27T00:00:00.500Z", + "accessibilityInspectorPayload": { + "screenshotPath": str(external_screenshot), + }, + }, + { + "type": "mouse.click", + "sessionId": "synthetic-external-screenshot-smoke", + "timestamp": "2026-06-27T00:00:01Z", + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Create", + }, + }, + { + "type": "session.ended", + "sessionId": "synthetic-external-screenshot-smoke", + "timestamp": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_stopped", + }, + ] + external_screenshot_metadata = { + "sessionId": "synthetic-external-screenshot-smoke", + "state": "stopped", + "active": False, + "startTime": "2026-06-27T00:00:00Z", + "endTime": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_stopped", + "eventCount": len(external_screenshot_events), + "suppressedEventCount": 0, + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "eventsPath": "events.jsonl", + "suppressedEventsPath": "suppressed.jsonl", + } + write_jsonl(external_screenshot_session_dir / "events.jsonl", external_screenshot_events) + write_json(external_screenshot_session_dir / "metadata.json", external_screenshot_metadata) + write_json(external_screenshot_session_dir / "session.json", external_screenshot_metadata) + (external_screenshot_session_dir / "suppressed.jsonl").write_text("") + external_screenshot_validation = subprocess.run( + [ + cli, + "event-stream", + "validate", + "--json", + "--strict-ocu", + str(external_screenshot_session_dir / "metadata.json"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + external_screenshot_payload = json.loads( + external_screenshot_validation.stdout or external_screenshot_validation.stderr + ) + if external_screenshot_validation.returncode == 0: + raise AssertionError("external screenshotPath validation should fail") + expected_screenshot_error = ( + "screenshotPath from event line 2 must stay inside session directory: " + f"{external_screenshot}" + ) + if expected_screenshot_error not in external_screenshot_payload.get("errors", []): + raise AssertionError(external_screenshot_payload) + + events_only_validation = subprocess.run( + [ + cli, + "event-stream", + "validate", + "--json", + "--require-skill-draft", + str(session_dir / "events.jsonl"), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + events_only_payload = json.loads(events_only_validation.stdout) + if events_only_payload.get("ok") is not True or events_only_payload.get("skillDraftReady") is not True: + raise AssertionError(events_only_payload) + if events_only_payload.get("metadataPath") is not None: + raise AssertionError(events_only_payload) + + output_dir = root / "draft-skill" + scaffold = subprocess.run( + [ + cli, + "event-stream", + "scaffold-skill", + "--json", + str(session_dir / "metadata.json"), + "--skill-name", + "synthetic-recording-workflow", + "--description", + "Replay the synthetic Record & Replay workflow.", + "--output-dir", + str(output_dir), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + scaffold_payload = json.loads(scaffold.stdout) + if scaffold_payload.get("ok") is not True: + raise AssertionError(scaffold_payload) + + skill_text = (output_dir / "SKILL.md").read_text() + summary = json.loads((output_dir / "references" / "recording-summary.json").read_text()) + if "synthetic-recording-workflow" not in skill_text or "Click role=" not in skill_text: + raise AssertionError(skill_text) + for required_snippet in [ + "skill-creator", + "Complete the `skill-creator` workflow, including validation", + "not a standalone runbook or replay plan", + ]: + if required_snippet not in skill_text: + raise AssertionError(skill_text) + if str(root) in skill_text or str(root) in json.dumps(summary, sort_keys=True): + raise AssertionError("generated scaffold leaked temporary local paths") + readiness = summary.get("skillReadiness", {}) + if readiness.get("canCreateSkillDraft") is not True: + raise AssertionError(readiness) + + invalid_session_dir = root / "invalid-session" + invalid_session_dir.mkdir() + invalid_events = [ + { + "type": "session.started", + "sessionId": "synthetic-invalid-skill-smoke", + "timestamp": "2026-06-27T00:00:00Z", + }, + { + "type": "session.ended", + "sessionId": "synthetic-invalid-skill-smoke", + "timestamp": "2026-06-27T00:00:01Z", + "endReason": "recording_controls_stopped", + }, + ] + invalid_metadata = { + "sessionId": "synthetic-invalid-skill-smoke", + "state": "stopped", + "active": False, + "startTime": "2026-06-27T00:00:00Z", + "endTime": "2026-06-27T00:00:01Z", + "endReason": "recording_controls_stopped", + "eventCount": len(invalid_events), + "suppressedEventCount": 0, + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "eventsPath": "events.jsonl", + "suppressedEventsPath": "suppressed.jsonl", + } + write_jsonl(invalid_session_dir / "events.jsonl", invalid_events) + write_json(invalid_session_dir / "metadata.json", invalid_metadata) + write_json(invalid_session_dir / "session.json", invalid_metadata) + (invalid_session_dir / "suppressed.jsonl").write_text("") + invalid_scaffold = subprocess.run( + [ + cli, + "event-stream", + "scaffold-skill", + "--json", + str(invalid_session_dir / "metadata.json"), + "--skill-name", + "invalid-recording-workflow", + "--description", + "This should not be generated.", + "--output-dir", + str(root / "invalid-draft-skill"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + invalid_payload = json.loads(invalid_scaffold.stdout or invalid_scaffold.stderr) + if invalid_scaffold.returncode == 0: + raise AssertionError("failed scaffold-skill should make the CLI command fail") + if invalid_payload.get("ok") is not False: + raise AssertionError(invalid_payload) + + cancelled_session_dir = root / "cancelled-session" + cancelled_session_dir.mkdir() + cancelled_events = [ + { + "type": "session.started", + "sessionId": "synthetic-cancelled-skill-smoke", + "timestamp": "2026-06-27T00:00:00Z", + }, + { + "type": "mouse.click", + "timestamp": "2026-06-27T00:00:01Z", + "app": {"name": "Example"}, + "window": {"title": "Example Window"}, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Discard", + "label": "Discard", + }, + }, + { + "type": "session.ended", + "sessionId": "synthetic-cancelled-skill-smoke", + "timestamp": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_cancelled", + }, + ] + cancelled_metadata = { + "sessionId": "synthetic-cancelled-skill-smoke", + "state": "cancelled", + "active": False, + "startTime": "2026-06-27T00:00:00Z", + "endTime": "2026-06-27T00:00:02Z", + "endReason": "recording_controls_cancelled", + "eventCount": len(cancelled_events), + "suppressedEventCount": 0, + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "eventsPath": "events.jsonl", + "suppressedEventsPath": "suppressed.jsonl", + } + write_jsonl(cancelled_session_dir / "events.jsonl", cancelled_events) + write_json(cancelled_session_dir / "metadata.json", cancelled_metadata) + write_json(cancelled_session_dir / "session.json", cancelled_metadata) + (cancelled_session_dir / "suppressed.jsonl").write_text("") + cancelled_reason = "recording was cancelled; do not create or update a skill from this event stream" + + for cancelled_input, extra_args in [ + (cancelled_session_dir / "metadata.json", ["--strict-ocu"]), + (cancelled_session_dir / "events.jsonl", []), + ]: + cancelled_validation = subprocess.run( + [ + cli, + "event-stream", + "validate", + "--json", + *extra_args, + "--require-skill-draft", + str(cancelled_input), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + cancelled_validation_payload = json.loads( + cancelled_validation.stdout or cancelled_validation.stderr + ) + if cancelled_validation.returncode == 0: + raise AssertionError("cancelled validation should fail with --require-skill-draft") + if cancelled_reason not in json.dumps(cancelled_validation_payload, sort_keys=True): + raise AssertionError(cancelled_validation_payload) + + cancelled_scaffold = subprocess.run( + [ + cli, + "event-stream", + "scaffold-skill", + "--json", + str(cancelled_session_dir / "metadata.json"), + "--skill-name", + "cancelled-recording-workflow", + "--description", + "This should not be generated.", + "--output-dir", + str(root / "cancelled-draft-skill"), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + cancelled_payload = json.loads(cancelled_scaffold.stdout or cancelled_scaffold.stderr) + if cancelled_scaffold.returncode == 0: + raise AssertionError("cancelled scaffold-skill should make the CLI command fail") + if cancelled_payload.get("ok") is not False: + raise AssertionError(cancelled_payload) + if cancelled_reason not in json.dumps(cancelled_payload, sort_keys=True): + raise AssertionError(cancelled_payload) + + print( + json.dumps( + { + "ok": True, + "skillName": scaffold_payload.get("skillName"), + "checkedStrictValidation": True, + "checkedDeclaredHandoffPaths": True, + "checkedScreenshotPathContainment": True, + "checkedEventsOnlyValidation": True, + "checkedScaffoldSkill": True, + "checkedScaffoldSkillFailureExit": True, + "checkedCancelledRecordingRejected": True, + "checkedSkillCreatorHandoff": True, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +WAIT_NOTIFY_CONTRACT_SMOKE_SCRIPT = """#!/usr/bin/env python3 + +import json +import os +import pathlib +import subprocess +import sys +import tempfile + + +def main(): + cli = os.environ.get("OPEN_COMPUTER_USE_CLI", "open-computer-use") + with tempfile.TemporaryDirectory(prefix="record-and-replay-wait-notify-") as tmp: + root = pathlib.Path(tmp) + recordings = root / "recordings" + callback_marker = root / "callback-ran.json" + callback_script = root / "callback.py" + callback_script.write_text( + "import pathlib\\n" + "import sys\\n" + f"pathlib.Path({str(callback_marker)!r}).write_text(sys.stdin.read())\\n" + ) + env = { + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + result = subprocess.run( + [ + cli, + "event-stream", + "wait", + "--json", + "--session-id", + "missing-session-for-contract-smoke", + "--timeout", + "0.05", + "--notify-command", + json.dumps([sys.executable, str(callback_script)]), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + payload = json.loads(result.stdout) + if payload.get("waitTimedOut") is not True: + raise AssertionError(payload) + if payload.get("waitSessionMatched") is not False: + raise AssertionError(payload) + notification = payload.get("notification", {}) + if notification.get("skipped") is not True or notification.get("reason") != "waitTimedOut": + raise AssertionError(payload) + if callback_marker.exists(): + raise AssertionError("wait notify callback ran for an unmatched session") + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("wait on an unmatched session should not create recording session files") + + completed_session_id = "session-wait-notify-contract" + completed_session_dir = recordings / completed_session_id + completed_session_dir.mkdir(parents=True, exist_ok=True) + events_path = completed_session_dir / "events.jsonl" + metadata_path = completed_session_dir / "metadata.json" + suppressed_path = completed_session_dir / "suppressed.jsonl" + session_path = completed_session_dir / "session.json" + events_path.write_text( + json.dumps( + { + "type": "session.started", + "timestamp": "2026-06-27T00:00:00.000Z", + "sessionId": completed_session_id, + } + ) + + "\\n" + + json.dumps( + { + "type": "session.ended", + "timestamp": "2026-06-27T00:00:01.000Z", + "sessionId": completed_session_id, + "endReason": "recording_controls_stopped", + } + ) + + "\\n" + ) + suppressed_path.write_text("") + metadata = { + "sessionID": completed_session_id, + "sessionId": completed_session_id, + "state": "stopped", + "active": False, + "startedAt": "2026-06-27T00:00:00.000Z", + "endedAt": "2026-06-27T00:00:01.000Z", + "endReason": "recording_controls_stopped", + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "suppressedEventsPath": str(suppressed_path), + "eventCount": 2, + "suppressedEventCount": 0, + } + metadata_path.write_text(json.dumps(metadata, sort_keys=True)) + session_path.write_text(json.dumps(metadata, sort_keys=True)) + + success_status_path = root / "notify-success-status.json" + success_suppressed_path = root / "notify-success-suppressed-path.txt" + success_script = root / "notify-success.py" + success_script.write_text( + "import os\\n" + "import pathlib\\n" + "import sys\\n" + "pathlib.Path(sys.argv[1]).write_text(sys.stdin.read())\\n" + "pathlib.Path(sys.argv[2]).write_text(" + "os.environ.get('OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH', '')" + ")\\n" + ) + success_result = subprocess.run( + [ + cli, + "event-stream", + "wait", + "--json", + "--session-id", + completed_session_id, + "--timeout", + "1", + "--notify-command", + json.dumps([ + sys.executable, + str(success_script), + str(success_status_path), + str(success_suppressed_path), + ]), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + success_payload = json.loads(success_result.stdout) + success_notification = success_payload.get("notification", {}) + if success_payload.get("waitTimedOut") is not False: + raise AssertionError(success_payload) + if success_payload.get("waitSessionMatched") is not True: + raise AssertionError(success_payload) + if success_notification.get("ok") is not True: + raise AssertionError(success_payload) + success_callback_status = json.loads(success_status_path.read_text()) + if success_callback_status.get("sessionId") != completed_session_id: + raise AssertionError(success_callback_status) + if pathlib.Path(success_suppressed_path.read_text()) != suppressed_path: + raise AssertionError("notify callback did not receive suppressed events path") + + failure_script = root / "notify-fail.py" + failure_script.write_text("import sys\\nsys.exit(7)\\n") + failure_result = subprocess.run( + [ + cli, + "event-stream", + "wait", + "--json", + "--session-id", + completed_session_id, + "--timeout", + "1", + "--notify-command", + json.dumps([sys.executable, str(failure_script)]), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + failure_payload = json.loads(failure_result.stdout) + failure_notification = failure_payload.get("notification", {}) + if failure_payload.get("waitTimedOut") is not False: + raise AssertionError(failure_payload) + if failure_payload.get("waitSessionMatched") is not True: + raise AssertionError(failure_payload) + if failure_notification.get("ok") is not False: + raise AssertionError(failure_payload) + if failure_notification.get("reason") != "nonZeroExit": + raise AssertionError(failure_payload) + if failure_notification.get("exitCode") != 7: + raise AssertionError(failure_payload) + if failure_result.returncode == 0: + raise AssertionError("failed notify callback should make the CLI command fail") + + timeout_script = root / "notify-sleep.py" + timeout_script.write_text("import time\\ntime.sleep(2)\\n") + timeout_env = { + **env, + "OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS": "0.1", + } + timeout_result = subprocess.run( + [ + cli, + "event-stream", + "wait", + "--json", + "--session-id", + completed_session_id, + "--timeout", + "1", + "--notify-command", + json.dumps([sys.executable, str(timeout_script)]), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=timeout_env, + ) + timeout_payload = json.loads(timeout_result.stdout) + timeout_notification = timeout_payload.get("notification", {}) + if timeout_payload.get("waitTimedOut") is not False: + raise AssertionError(timeout_payload) + if timeout_payload.get("waitSessionMatched") is not True: + raise AssertionError(timeout_payload) + if timeout_notification.get("ok") is not False: + raise AssertionError(timeout_payload) + if timeout_notification.get("timedOut") is not True: + raise AssertionError(timeout_payload) + if timeout_notification.get("reason") != "timeout": + raise AssertionError(timeout_payload) + if timeout_result.returncode == 0: + raise AssertionError("timed out notify callback should make the CLI command fail") + + print( + json.dumps( + { + "ok": True, + "checkedWaitNotifyContract": True, + "checkedNotifySuppressedEventsPathEnv": True, + "checkedNotifyCallbackFailureExit": True, + "checkedNotifyCallbackFailureReason": True, + "checkedNotifyCallbackTimeoutFailureExit": True, + "checkedNotifyCallbackTimeoutReason": True, + "waitTimedOut": payload.get("waitTimedOut"), + "waitSessionMatched": payload.get("waitSessionMatched"), + "callbackSkipped": notification.get("skipped"), + "callbackFailureDetected": True, + "callbackFailureReason": failure_notification.get("reason"), + "callbackTimeoutDetected": True, + "callbackTimeoutReason": timeout_notification.get("reason"), + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +RECORDING_LIFECYCLE_SMOKE_SCRIPT = """#!/usr/bin/env python3 + +import json +import os +import pathlib +import select +import subprocess +import sys +import tempfile + + +def read_line(process, timeout=10, timeout_message=None): + ready, _, _ = select.select([process.stdout], [], [], timeout) + if not ready: + if timeout_message is not None: + raise RuntimeError(timeout_message()) + raise RuntimeError("timed out waiting for Record & Replay MCP response") + line = process.stdout.readline() + if not line: + raise RuntimeError("Record & Replay MCP server exited before responding") + return json.loads(line) + + +def send_message(process, message): + process.stdin.write(json.dumps(message, separators=(",", ":")) + "\\n") + process.stdin.flush() + + +def request(process, message, timeout=10, timeout_message=None): + send_message(process, message) + response = read_line(process, timeout=timeout, timeout_message=timeout_message) + if "error" in response: + raise AssertionError(response["error"]) + return response + + +def tool_text(response): + content = response.get("result", {}).get("content", []) + if not content or content[0].get("type") != "text": + raise AssertionError(f"unexpected tool response: {response!r}") + return json.loads(content[0]["text"]) + + +def runtime_version(cli): + try: + completed = subprocess.run( + [cli, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=5, + ) + except Exception as error: + return {"ok": False, "error": f"{type(error).__name__}: {error}"} + return { + "ok": completed.returncode == 0, + "returncode": completed.returncode, + "stdout": completed.stdout.strip(), + "stderr": completed.stderr.strip(), + } + + +def mcp_timeout_message(cli, command, process): + parts = [ + "timed out waiting for Record & Replay MCP response", + f"runtime={cli!r}", + f"command={command!r}", + f"runtimeVersion={runtime_version(cli)!r}", + ] + poll = process.poll() + if poll is None: + parts.append("processStillRunning=true") + else: + parts.append(f"processReturnCode={poll}") + if process.stderr: + parts.append(f"stderr={process.stderr.read().strip()!r}") + parts.append( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime if this points to an older install." + ) + return "; ".join(parts) + + +def main(): + cli = os.environ.get("OPEN_COMPUTER_USE_CLI", "open-computer-use") + response_timeout = float(os.environ.get("OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS", "10")) + with tempfile.TemporaryDirectory(prefix="record-and-replay-lifecycle-") as tmp: + recordings = pathlib.Path(tmp) / "recordings" + env = { + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL": "approve", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + command = [cli, "event-stream", "mcp"] + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + request_failed = False + try: + timeout_message = lambda: mcp_timeout_message(cli, command, process) + initialize = request( + process, + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": { + "name": "record-and-replay-lifecycle-smoke", + "version": "0", + }, + }, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + send_message( + process, + {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, + ) + tools = request( + process, + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + timeout=response_timeout, + timeout_message=timeout_message, + ) + start = request( + process, + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + repeat_start = request( + process, + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + status = request( + process, + { + "jsonrpc": "2.0", + "id": 5, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + stop = request( + process, + { + "jsonrpc": "2.0", + "id": 6, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + repeat_stop = request( + process, + { + "jsonrpc": "2.0", + "id": 7, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + final_status = request( + process, + { + "jsonrpc": "2.0", + "id": 8, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=timeout_message, + ) + except BaseException: + request_failed = True + raise + finally: + if process.stdin: + process.stdin.close() + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=5) + stderr = process.stderr.read() if process.stderr else "" + if not request_failed and process.returncode != 0: + raise AssertionError(stderr) + + initialize_result = initialize["result"] + if initialize_result.get("serverInfo", {}).get("name") != "Record & Replay": + raise AssertionError(initialize_result) + tool_names = [tool.get("name") for tool in tools.get("result", {}).get("tools", [])] + if tool_names != ["event_stream_start", "event_stream_status", "event_stream_stop"]: + raise AssertionError(tool_names) + + started = tool_text(start) + repeated_start = tool_text(repeat_start) + current = tool_text(status) + stopped = tool_text(stop) + repeated = tool_text(repeat_stop) + final = tool_text(final_status) + session_id = started.get("sessionId") + if not session_id or started.get("state") != "recording": + raise AssertionError(started) + if ( + repeated_start.get("sessionId") != session_id + or repeated_start.get("state") != "recording" + ): + raise AssertionError(repeated_start) + if current.get("sessionId") != session_id or current.get("state") != "recording": + raise AssertionError(current) + if ( + stopped.get("sessionId") != session_id + or stopped.get("state") != "stopped" + or stopped.get("endReason") != "recording_controls_stopped" + ): + raise AssertionError(stopped) + if ( + repeated.get("sessionId") != session_id + or repeated.get("state") != "stopped" + or repeated.get("endReason") != "recording_controls_stopped" + ): + raise AssertionError(repeated) + if ( + final.get("sessionId") != session_id + or final.get("state") != "stopped" + or final.get("endReason") != "recording_controls_stopped" + ): + raise AssertionError(final) + + metadata_path = pathlib.Path(stopped["metadataPath"]) + events_path = pathlib.Path(stopped["eventsPath"]) + session_path = pathlib.Path(stopped["sessionPath"]) + for path in [metadata_path, events_path, session_path]: + if not path.exists(): + raise AssertionError(f"missing recording artifact: {path}") + + events = [ + json.loads(line) + for line in events_path.read_text().splitlines() + if line.strip() + ] + event_types = [event.get("type") for event in events] + for required in ["session.started", "session.ended"]: + if required not in event_types: + raise AssertionError({"missing": required, "eventTypes": event_types}) + + validation = subprocess.run( + [ + cli, + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + str(metadata_path), + ], + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={**os.environ, "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1"}, + ) + validation_payload = json.loads(validation.stdout) + if validation_payload.get("ok") is not True: + raise AssertionError(validation_payload) + + print( + json.dumps( + { + "ok": True, + "sessionId": session_id, + "eventCount": stopped.get("eventCount"), + "eventTypes": sorted(set(event_types)), + "checkedOneActive": True, + "checkedIdempotentStop": True, + "checkedFinalStatus": True, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +GITHUB_CI_WORKFLOW = """name: ci + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +jobs: + ci: + runs-on: macos-26 + + steps: + - name: Check out repository + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd + + - name: Set up Node.js + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: "24" + + - name: Install Open Computer Use runtime + run: npm i -g open-computer-use + + - name: Run standalone skill checks + run: ./scripts/check.sh +""" + + +VERIFY_RUNTIME_SCRIPT = """#!/usr/bin/env python3 + +import json +import os +import pathlib +import select +import subprocess +import sys +import tempfile +import time + + +EXPECTED_TOOLS = [ + { + "name": "event_stream_start", + "description": ( + "Start recording the user's actions for up to 30 minutes. If a recording is already " + "active, return that active session instead of starting another one." + ), + "annotations": { + "destructiveHint": False, + "idempotentHint": False, + "openWorldHint": False, + "readOnlyHint": False, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, + { + "name": "event_stream_status", + "description": ( + "Get the current or most recent Record & Replay recording status including paths to " + "metadata and events during the recording." + ), + "annotations": { + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + "readOnlyHint": True, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, + { + "name": "event_stream_stop", + "description": ( + "Stop the active event stream recording if one is running and return status including " + "paths to metadata and events during the recording." + ), + "annotations": { + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + "readOnlyHint": False, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, +] + +EXPECTED_TOOL_NAMES = [tool["name"] for tool in EXPECTED_TOOLS] +TOOL_CONTRACT_KEYS = ["name", "description", "inputSchema", "annotations"] +EXPECTED_INITIALIZE_CAPABILITIES = {"tools": {"listChanged": False}} +EXPECTED_NO_ACTIVE_RESPONSE = {"isRecording": False, "maxDurationSeconds": 1800} + + +def runtime_version(cli): + try: + completed = subprocess.run( + [cli, "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=5, + ) + except Exception as error: + return {"ok": False, "error": f"{type(error).__name__}: {error}"} + return { + "ok": completed.returncode == 0, + "returncode": completed.returncode, + "stdout": completed.stdout.strip(), + "stderr": completed.stderr.strip(), + } + + +def mcp_timeout_message(cli, command, process): + parts = [ + "timed out waiting for Record & Replay MCP response", + f"runtime={cli!r}", + f"command={command!r}", + f"runtimeVersion={runtime_version(cli)!r}", + ] + poll = process.poll() + if poll is None: + parts.append("processStillRunning=true") + else: + parts.append(f"processReturnCode={poll}") + if process.stderr: + parts.append(f"stderr={process.stderr.read().strip()!r}") + parts.append( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime if this points to an older install." + ) + return "; ".join(parts) + + +def read_line(process, timeout=10, timeout_message=None): + deadline = time.time() + timeout + while time.time() < deadline: + ready, _, _ = select.select([process.stdout], [], [], 0.1) + if ready: + line = process.stdout.readline() + if line: + return line + if process.poll() is not None: + break + if timeout_message is not None: + raise RuntimeError(timeout_message()) + raise RuntimeError("timed out waiting for Record & Replay MCP response") + + +def send_message(process, message): + process.stdin.write(json.dumps(message, separators=(",", ":")) + "\\n") + process.stdin.flush() + + +def request(process, message, timeout=10, timeout_message=None): + send_message(process, message) + response = json.loads(read_line(process, timeout=timeout, timeout_message=timeout_message)) + if "error" in response: + raise AssertionError(response["error"]) + return response + + +def tool_text(response): + content = response.get("result", {}).get("content", []) + if not content or content[0].get("type") != "text": + raise AssertionError(f"unexpected tool response: {response!r}") + return json.loads(content[0]["text"]) + + +def tool_error_text(response): + result = response.get("result", {}) + if result.get("isError") is not True: + raise AssertionError(f"expected tool error response: {response!r}") + content = result.get("content", []) + if not content or content[0].get("type") != "text": + raise AssertionError(f"unexpected tool error response: {response!r}") + return content[0]["text"] + + +def main(): + cli = os.environ.get("OPEN_COMPUTER_USE_CLI", "open-computer-use") + response_timeout = float(os.environ.get("OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS", "10")) + with tempfile.TemporaryDirectory(prefix="record-and-replay-runtime-verify-") as tmp: + recordings = pathlib.Path(tmp) / "recordings" + env = { + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": os.environ.get( + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY", + "1", + ), + "OPEN_COMPUTER_USE_EVENT_STREAM_CONTROLS": "0", + "OPEN_COMPUTER_USE_EVENT_STREAM_SCREENSHOTS": "never", + "OPEN_COMPUTER_USE_EVENT_STREAM_DIR": str(recordings), + } + command = [cli, "event-stream", "mcp"] + process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + try: + initialize = request( + process, + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-11-25", + "capabilities": {}, + "clientInfo": {"name": "record-and-replay-skill-repo-verify", "version": "0"}, + }, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + send_message( + process, + {"jsonrpc": "2.0", "method": "notifications/initialized", "params": {}}, + ) + tools_list = request( + process, + {"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}}, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + no_active_status = request( + process, + { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + no_active_stop = request( + process, + { + "jsonrpc": "2.0", + "id": 4, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + unexpected_arguments = request( + process, + { + "jsonrpc": "2.0", + "id": 5, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {"unexpected": True}}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + non_object_arguments = request( + process, + { + "jsonrpc": "2.0", + "id": 6, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": []}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + missing_tool_name = request( + process, + { + "jsonrpc": "2.0", + "id": 7, + "method": "tools/call", + "params": {"arguments": {}}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + non_string_tool_name = request( + process, + { + "jsonrpc": "2.0", + "id": 8, + "method": "tools/call", + "params": {"name": ["event_stream_start"], "arguments": {}}, + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + missing_tool_params = request( + process, + { + "jsonrpc": "2.0", + "id": 9, + "method": "tools/call", + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + non_object_tool_params = request( + process, + { + "jsonrpc": "2.0", + "id": 10, + "method": "tools/call", + "params": [], + }, + timeout=response_timeout, + timeout_message=lambda: mcp_timeout_message(cli, command, process), + ) + finally: + process.terminate() + try: + process.wait(timeout=2) + except subprocess.TimeoutExpired: + process.kill() + + result = initialize.get("result", {}) + server_info = result.get("serverInfo", {}) + if server_info.get("name") != "Record & Replay": + raise AssertionError(f"unexpected server name: {server_info!r}") + if result.get("protocolVersion") != "2025-11-25": + raise AssertionError(f"unexpected protocolVersion: {result.get('protocolVersion')!r}") + if result.get("capabilities") != EXPECTED_INITIALIZE_CAPABILITIES: + raise AssertionError( + f"unexpected capabilities: {result.get('capabilities')!r}" + ) + if "instructions" in result: + raise AssertionError("Record & Replay initialize result should not include instructions") + checked_initialize_surface_contract = True + + tools = tools_list.get("result", {}).get("tools", []) + tool_names = [tool.get("name") for tool in tools] + if tool_names != EXPECTED_TOOL_NAMES: + raise AssertionError(f"unexpected tools: {tool_names!r}") + comparable_tools = [] + for tool in tools: + missing_keys = [key for key in TOOL_CONTRACT_KEYS if key not in tool] + if missing_keys: + raise AssertionError(f"tool {tool.get('name')!r} is missing keys: {missing_keys!r}") + comparable_tools.append({key: tool[key] for key in TOOL_CONTRACT_KEYS}) + if comparable_tools != EXPECTED_TOOLS: + raise AssertionError( + "Record & Replay tool metadata drifted: " + + json.dumps( + {"expected": EXPECTED_TOOLS, "actual": comparable_tools}, + sort_keys=True, + separators=(",", ":"), + ) + ) + checked_tool_metadata_contract = True + checked_tool_input_schema_no_arguments = all( + tool.get("inputSchema") == { + "additionalProperties": False, + "properties": {}, + "type": "object", + } + for tool in comparable_tools + ) + if not checked_tool_input_schema_no_arguments: + raise AssertionError(f"unexpected no-arg input schemas: {comparable_tools!r}") + + for label, response in [ + ("event_stream_status", no_active_status), + ("event_stream_stop", no_active_stop), + ]: + payload = tool_text(response) + if payload != EXPECTED_NO_ACTIVE_RESPONSE: + raise AssertionError( + f"unexpected no-active {label} response: " + + json.dumps( + {"expected": EXPECTED_NO_ACTIVE_RESPONSE, "actual": payload}, + sort_keys=True, + separators=(",", ":"), + ) + ) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("no-active status/stop should not create recording session files") + + unexpected_arguments_error = tool_error_text(unexpected_arguments) + if "event_stream_status does not accept arguments" not in unexpected_arguments_error: + raise AssertionError(unexpected_arguments) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected no-arg contract check should not create recording session files") + + non_object_arguments_error = tool_error_text(non_object_arguments) + if "tools/call arguments must be an object" not in non_object_arguments_error: + raise AssertionError(non_object_arguments) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected non-object arguments should not create recording session files") + + missing_tool_name_error = tool_error_text(missing_tool_name) + if "tools/call params.name must be a non-empty string" not in missing_tool_name_error: + raise AssertionError(missing_tool_name) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected missing tool name should not create recording session files") + + non_string_tool_name_error = tool_error_text(non_string_tool_name) + if "tools/call params.name must be a non-empty string" not in non_string_tool_name_error: + raise AssertionError(non_string_tool_name) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected non-string tool name should not create recording session files") + + missing_tool_params_error = tool_error_text(missing_tool_params) + if "tools/call params must be an object" not in missing_tool_params_error: + raise AssertionError(missing_tool_params) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected missing tool params should not create recording session files") + + non_object_tool_params_error = tool_error_text(non_object_tool_params) + if "tools/call params must be an object" not in non_object_tool_params_error: + raise AssertionError(non_object_tool_params) + if (recordings / "latest-session.json").exists() or (recordings / "active-session.json").exists(): + raise AssertionError("rejected non-object tool params should not create recording session files") + + print(json.dumps({ + "ok": True, + "server": server_info, + "tools": tool_names, + "checkedInitializeSurfaceContract": checked_initialize_surface_contract, + "checkedToolMetadataContract": checked_tool_metadata_contract, + "checkedToolInputSchemaNoArguments": checked_tool_input_schema_no_arguments, + "checkedNoActiveStatusStop": True, + "checkedRejectsUnexpectedArguments": True, + "checkedRejectsNonObjectArguments": True, + "checkedRequiresObjectParams": True, + "checkedRequiresStringToolName": True, + "checkedRequiresObjectArguments": True, + "checkedRejectedRequestsDoNotCreateSessionFiles": True, + }, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) +""" + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Scaffold a standalone thin Record & Replay skill repository." + ) + parser.add_argument( + "--output-dir", + required=True, + type=pathlib.Path, + help="Directory to create. It must be empty unless --force is passed.", + ) + parser.add_argument( + "--repo-name", + default="open-computer-use-record-and-replay-skill", + help="Human-readable repository name used in README text.", + ) + parser.add_argument( + "--skill-name", + default=SOURCE_SKILL_NAME, + help="Skill directory name to create under skills/.", + ) + parser.add_argument("--force", action="store_true", help="Replace output-dir if it exists.") + return parser.parse_args() + + +def write_text(path: pathlib.Path, content: str, executable: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content) + if executable: + path.chmod(path.stat().st_mode | 0o111) + + +def read_skill_frontmatter(skill_path: pathlib.Path) -> dict[str, str]: + content = skill_path.read_text() + if not content.startswith("---\n"): + raise ValueError(f"skill has no frontmatter: {skill_path}") + _, frontmatter, _ = content.split("---", 2) + result: dict[str, str] = {} + for raw_line in frontmatter.splitlines(): + if ":" not in raw_line: + continue + key, value = raw_line.split(":", 1) + result[key.strip()] = value.strip() + return result + + +def markdown_code_list(values: list[str]) -> str: + formatted = [f"`{value}`" for value in values] + if not formatted: + return "" + if len(formatted) == 1: + return formatted[0] + if len(formatted) == 2: + return " and ".join(formatted) + return ", ".join(formatted[:-1]) + f", and {formatted[-1]}" + + +def source_baseline_summary_projection() -> dict[str, object]: + summary_path = REPO_ROOT / "dist" / "record-and-replay-baseline-summary.json" + if not summary_path.exists(): + raise ValueError( + "missing source baseline summary artifact: " + f"{summary_path}; run make record-and-replay-baseline-audit first" + ) + summary = json.loads(summary_path.read_text()) + status = summary.get("status") + evidence = summary.get("evidence") + if not isinstance(status, dict) or not isinstance(evidence, dict): + raise ValueError(f"invalid source baseline summary artifact: {summary_path}") + return { + "baseline": summary.get("baseline"), + "checks": summary.get("checks"), + "ok": summary.get("ok"), + "status": { + key: status.get(key) + for key in [ + "usableBaseline", + "standaloneRepoBaselineReady", + "officialGoldenRequirementSatisfied", + "officialGoldenGatePassed", + "officialSuccessfulRecordingGoldenComplete", + "officialSuccessfulRecordingEquivalenceReady", + "requiresOfficialGoldenCapture", + "missingUsableBaselineEvidence", + "missingRequiredOfficialSuccessfulRecordingScenarios", + "notReadyRequiredOfficialSuccessfulRecordingScenarios", + "missingRecommendedOfficialSuccessfulRecordingScenarios", + "officialFixtureCoverageErrors", + ] + }, + "evidence": { + "officialFixtureSetGate": { + key: (evidence.get("officialFixtureSetGate") or {}).get(key) + for key in [ + "ok", + "checkedAxDiffComparisonPolicy", + "checkedSuppressedStreamComparisonPolicy", + "checkedAxDiffComparisonFailure", + ] + }, + "fixtureIngestPipelines": { + key: (evidence.get("fixtureIngestPipelines") or {}).get(key) + for key in [ + "checkedOfficialSessionDirectoryPathHandoff", + ] + }, + "preflightPipelines": { + key: (evidence.get("preflightPipelines") or {}).get(key) + for key in [ + "checkedOfficialCapturePacketInputSemanticGuard", + "checkedOfficialCapturePacketSetContractManifest", + "checkedOfficialCapturePacketPostCaptureWorkflow", + "checkedOfficialCapturePacketWorkflowVerifier", + "checkedOfficialCapturePacketSetPostCaptureWorkflow", + "checkedOfficialCapturePacketSetWorkflowVerifier", + "checkedOfficialCapturePacketStrictAuditHandoff", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ] + }, + "standaloneSkillRepo": { + key: (evidence.get("standaloneSkillRepo") or {}).get(key) + for key in [ + "checkedGeneratedRepoSelfCheck", + "checkedManifestContract", + "checkedOfficialFixtureSetComparePolicyManifest", + ] + }, + "npmStagedSkillRepo": { + key: (evidence.get("npmStagedSkillRepo") or {}).get(key) + for key in [ + "checkedGeneratedRepoSelfCheck", + "checkedManifestContract", + "checkedOfficialFixtureSetComparePolicyManifest", + ] + }, + }, + } + + +def scaffold(args: argparse.Namespace) -> dict[str, object]: + output_dir = args.output_dir + if output_dir.exists(): + if not args.force: + raise ValueError(f"output-dir already exists: {output_dir}; pass --force to replace it") + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True) + + if not (SOURCE_SKILL_DIR / "SKILL.md").exists(): + raise ValueError(f"missing source skill: {SOURCE_SKILL_DIR}") + + skill_dir = output_dir / "skills" / args.skill_name + shutil.copytree(SOURCE_SKILL_DIR, skill_dir) + + frontmatter = read_skill_frontmatter(skill_dir / "SKILL.md") + if frontmatter.get("name") != args.skill_name: + raise ValueError( + f"source skill frontmatter name={frontmatter.get('name')} does not match {args.skill_name}" + ) + + required_scenarios = list(DEFAULT_REQUIRED_SCENARIOS) + recommended_scenarios = list(DEFAULT_RECOMMENDED_SCENARIOS) + required_scenario_list = markdown_code_list(required_scenarios) + recommended_scenario_list = markdown_code_list(recommended_scenarios) + + readme = textwrap.dedent( + f"""\ + # {args.repo_name} + + Thin standalone skill repository for Open Computer Use Record & Replay. + + This repo intentionally does not contain desktop automation runtime code. It expects + users to install `open-computer-use` and configures agents to use the official-compatible + Record & Replay MCP surface: + + ```sh + npm i -g open-computer-use + open-computer-use install-codex-record-and-replay-mcp + ``` + + ## Prerequisites + + Python 3 must be available as `python3` for the generated repository + self-check scripts. If you create this repository through the npm + launcher, set `PYTHON=/path/to/python3` when your shell cannot find + Python 3 automatically. + + Install the skill from this repository: + + ```sh + npx skills add -g -a codex --skill {args.skill_name} -y + ``` + + The skill entrypoint is: + + ```text + skills/{args.skill_name}/SKILL.md + ``` + + ## Runtime Contract + + The Record & Replay MCP server must expose only these official-compatible tools, + with matching initialize capabilities, descriptions, empty input schemas, + and MCP annotations: + + ```text + event_stream_start + event_stream_status + event_stream_stop + ``` + + OCU extension commands such as `event-stream wait`, `validate`, `summarize`, and + `scaffold-skill` remain CLI/runtime helpers and are not added to the MCP tool surface. + + ## Official Evidence + + The runtime contract is pinned to the official `record-and-replay` 1.0.857 + evidence used by the Open Computer Use baseline: + + - `record-and-replay-event-stream-surface-1.0.857.json` proves the + non-recording initialize / `tools/list` surface. + - `record-and-replay-official-no-active-status-stop-1.0.857.json` proves + no-active `event_stream_status` / `event_stream_stop` return + `isRecording=false` and `maxDurationSeconds=1800` without creating + session files. + - `record-and-replay-official-raw-start-timeout-1.0.857.json` documents + the current hostless official raw `event_stream_start/status/stop` + timeout boundary and must not be treated as a successful recording + golden. + + The Open Computer Use source repo also gates this standalone contract with + a baseline contract smoke, preflight checks for official capture packets, + same-scenario OCU candidate pairing, and audit target dry-runs that keep + baseline / strict summary artifacts and override variables isolated. Those + source checks and preflight scripts are not copied into this standalone + repo, and the default standalone self-check still does not start official + recording. + + To preserve release or standalone handoff evidence in the source repo, + run the baseline audit target before cutting over. By default this writes + `dist/record-and-replay-baseline-summary.json`; this scaffold copies a + redacted projection of that artifact to `evidence/source-baseline-summary.json` + and the generated `scripts/check.sh` audits it: + + ```sh + make record-and-replay-baseline-audit + ``` + + After official successful recording fixtures are imported, use the strict + golden gate audit instead. By default this writes a separate + `dist/record-and-replay-official-golden-gate-summary.json`, so a failing + strict gate does not overwrite the usable baseline artifact: + + ```sh + make record-and-replay-official-golden-gate-audit + ``` + + Before official successful recording fixtures are imported, that strict + audit is expected to fail. To verify that its saved summary failed only + because the required official golden is missing, run: + + ```sh + scripts/check-record-and-replay-baseline-summary.py \\ + dist/record-and-replay-official-golden-gate-summary.json \\ + --allow-strict-official-golden-missing + ``` + + Successful recording fixtures are still required before claiming official + event schema, AX diff, screenshot, or timeout endReason equivalence. + The minimum required official successful recording scenario is + {required_scenario_list}. The recommended calibration set is + {recommended_scenario_list}. The machine-readable `record-and-replay-skill-repo.json` + manifest also includes `officialEvidence.scenarioRecipes`, which records + each scenario's capture goal, expected action events, expected end reason, + evidence requirements, and OCU candidate source. + The source repo's official fixture set gate also requires same-scenario + compare policy for AX diff evidence, AX diff marker alignment, + suppressed event sequence, and suppressed schema comparison before OCU + candidates can be treated as calibrated against official recordings. + + ## Recording Handoff + + After a recording stops, treat `events.jsonl` as the primary evidence and + `metadata.json` / `session.json` as session state and path metadata. Before + creating a reusable skill from a recording, run the installed runtime validator: + + ```sh + open-computer-use event-stream validate --json --strict-ocu --require-skill-draft + ``` + + If the host only returns `eventsPath`, run the non-strict events-only gate: + + ```sh + open-computer-use event-stream validate --json --require-skill-draft + ``` + + Events-only validation can prove JSONL readability, completion, + single `session.started` opening event, single `session.ended` + final-event closure, high-level action evidence, blocking diagnostics, + and cancellation inferred from + `session.ended.endReason=recording_controls_cancelled`; it cannot prove + OCU metadata/session alias consistency or the declared `metadataPath` / + `sessionPath` / `eventsPath` / `suppressedEventsPath` handoff paths. + Cancelled, incomplete, or malformed recordings must not be used to + create or update a skill. + + Generate the first draft only after validation passes: + + ```sh + open-computer-use event-stream scaffold-skill --json \\ + --skill-name \\ + --output-dir + ``` + + The scaffold command also runs the skill-draft validation gate before writing + files. The generated draft is a starting point: the agent still needs to read + the original `events.jsonl`, resolve runtime inputs and safety signals, and + finish the normal skill creation workflow before reporting that a reusable + skill exists. + + ## Package + + ```sh + ./scripts/package-skill.sh + ``` + + The package script writes `.zip` and `.skill` artifacts under `dist/skills/`. + + ## Check + + Run the generated repository self-check before publishing or installing it: + + ```sh + ./scripts/check.sh + ``` + + The check packages the skill, verifies the runtime contract, and verifies + that the README and skill workflow still preserve the official Record & + Replay handoff semantics: start once, do not poll, stop only after the user + returns, keep official golden gaps visible, and do not stop again after + cancellation. Set + `OPEN_COMPUTER_USE_CLI=/path/to/open-computer-use` when testing a specific + runtime binary. + + `scripts/check.sh` runs `scripts/verify-readme-handoff.py` to keep the + official evidence list, baseline / strict audit handoff commands, + required and recommended successful recording scenarios, and wait/notify + boundary documented in this README. + + `scripts/check.sh` also runs `scripts/verify-source-baseline-summary.py`. + That check consumes `evidence/source-baseline-summary.json`, which is a + redacted projection of the source repo's latest default baseline audit + artifact. It verifies that the source baseline was usable, that the + standalone repo baseline evidence was ready, and that the official + successful recording golden gap was still explicit. + + The generated GitHub Actions workflow runs the same check after installing + the published `open-computer-use` runtime. + + `scripts/check.sh` also runs `scripts/wait-notify-contract-smoke.py`, which + does not start a recording. It waits on an intentionally missing session + with `--notify-command`, confirms `waitTimedOut=true`, + `waitSessionMatched=false`, confirms the callback is skipped, and confirms + no recording session files are created. + + `scripts/check.sh` then runs `scripts/recording-to-skill-smoke.py`, which + also does not start a recording. It creates a temporary synthetic completed + recording, runs `event-stream validate --strict-ocu --require-skill-draft`, + checks the declared handoff path evidence, runs the events-only validation + gate, and verifies that the installed runtime can generate a draft skill + with `event-stream scaffold-skill`. + + ## Independent Wait / Notify Integration + + Wrappers that need to resume automatically after the user clicks Done or + Discard should use the OCU CLI extension layer, not extra MCP tools: + + ```sh + open-computer-use event-stream start --json + open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' + ``` + + `wait --json` returns `waitTimedOut` and `waitSessionMatched`. + `--notify-command` receives the final status JSON on stdin and the + `OPEN_COMPUTER_USE_EVENT_STREAM_*` environment variables documented in the + skill. Keep this listener path outside the official-compatible MCP tool + surface. + + ## Optional Lifecycle Smoke + + To verify that the installed runtime can create and stop a minimal recording + session, run: + + ```sh + ./scripts/recording-lifecycle-smoke.py + ``` + + This starts a short Record & Replay session with OCU controls disabled, stops + it through the same `event-stream mcp` process, and validates the resulting + `metadata.json`, `session.json`, and `events.jsonl` with the installed runtime. + It is intentionally separate from `scripts/check.sh` because it starts a real + local recording session and may require desktop permissions on macOS. + + ## Verify Runtime Contract + + This check does not start a recording. It only verifies that the configured + `open-computer-use` runtime exposes the official-compatible Record & Replay + MCP surface, including initialize capabilities, tool descriptions, no-arg + schemas, and annotations: + + ```sh + ./scripts/verify-runtime.py + ``` + + Set `OPEN_COMPUTER_USE_CLI=/path/to/open-computer-use` to test a specific + runtime binary. + + ## Verify Skill Workflow + + This check does not start a recording. It verifies that `SKILL.md` keeps the + official handoff constraints for agents: use only the three MCP tools, + end the turn after start, avoid polling, read returned recording paths, run + validation before scaffold, and avoid calling stop again after Discard: + + ```sh + ./scripts/verify-skill-workflow.py + ``` + """ + ) + write_text(output_dir / "README.md", readme) + write_text(output_dir / ".gitignore", "dist/\n.DS_Store\n") + write_text(output_dir / ".github" / "workflows" / "ci.yml", GITHUB_CI_WORKFLOW) + write_text(output_dir / "scripts" / "package-skill.sh", PACKAGE_SCRIPT, executable=True) + write_text( + output_dir / "scripts" / "verify-package-artifact.py", + VERIFY_PACKAGE_ARTIFACT_SCRIPT, + executable=True, + ) + write_text(output_dir / "scripts" / "check.sh", CHECK_SCRIPT, executable=True) + write_text(output_dir / "scripts" / "verify-manifest.py", VERIFY_MANIFEST_SCRIPT, executable=True) + write_text( + output_dir / "scripts" / "verify-source-baseline-summary.py", + VERIFY_SOURCE_BASELINE_SUMMARY_SCRIPT, + executable=True, + ) + write_text( + output_dir / "scripts" / "verify-readme-handoff.py", + VERIFY_README_HANDOFF_SCRIPT, + executable=True, + ) + write_text(output_dir / "scripts" / "verify-runtime.py", VERIFY_RUNTIME_SCRIPT, executable=True) + write_text( + output_dir / "scripts" / "verify-skill-workflow.py", + VERIFY_SKILL_WORKFLOW_SCRIPT, + executable=True, + ) + write_text( + output_dir / "scripts" / "recording-to-skill-smoke.py", + RECORDING_TO_SKILL_SMOKE_SCRIPT, + executable=True, + ) + write_text( + output_dir / "scripts" / "wait-notify-contract-smoke.py", + WAIT_NOTIFY_CONTRACT_SMOKE_SCRIPT, + executable=True, + ) + write_text( + output_dir / "scripts" / "recording-lifecycle-smoke.py", + RECORDING_LIFECYCLE_SMOKE_SCRIPT, + executable=True, + ) + + scenario_recipes = { + scenario: scenario_recipe(scenario) for scenario in recommended_scenarios + } + source_baseline_summary = source_baseline_summary_projection() + has_successful_recording_golden = ( + (source_baseline_summary.get("status") or {}).get( + "officialSuccessfulRecordingGoldenComplete" + ) + is True + ) + write_text( + output_dir / "evidence" / "source-baseline-summary.json", + json.dumps(source_baseline_summary, indent=2, sort_keys=True) + "\n", + ) + + manifest = { + "name": args.repo_name, + "kind": "open-computer-use-record-and-replay-thin-skill-repo", + "runtimeDependency": "open-computer-use", + "skill": { + "name": args.skill_name, + "path": f"skills/{args.skill_name}/SKILL.md", + }, + "officialEvidence": { + "baselineVersion": "record-and-replay/1.0.857", + "nonRecordingSurfaceFixture": ( + "record-and-replay-event-stream-surface-1.0.857.json" + ), + "noActiveStatusStopFixture": ( + "record-and-replay-official-no-active-status-stop-1.0.857.json" + ), + "hostlessRawStartTimeoutFixture": ( + "record-and-replay-official-raw-start-timeout-1.0.857.json" + ), + "sourceRepoBaselineChecks": { + "baselineContract": "baseline-contract-smoke", + "officialRawStartTimeout": "official-raw-start-timeout-fixture-smoke", + "officialFixtureSetGate": { + "check": "official-fixture-set-smoke", + "sameScenarioComparePolicy": { + "requiresAxDiffEvidence": True, + "requiresSameAxDiffMarkers": True, + "requiresSameSuppressedEventSequence": True, + "requiresSameSuppressedSchema": True, + }, + }, + "officialFixtureIngest": { + "check": "official-fixture-ingest-smoke", + "requiredEvidence": [ + "checkedOfficialSessionDirectoryPathHandoff", + ], + }, + "officialGoldenCapturePreflight": { + "check": "official-golden-capture-preflight-smoke", + "requiredEvidence": [ + "checkedOfficialCapturePacketInputSemanticGuard", + "checkedOfficialCapturePacketSetContractManifest", + "checkedOfficialCapturePacketPostCaptureWorkflow", + "checkedOfficialCapturePacketWorkflowVerifier", + "checkedOfficialCapturePacketSetPostCaptureWorkflow", + "checkedOfficialCapturePacketSetWorkflowVerifier", + "checkedOfficialCapturePacketStrictAuditHandoff", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ], + }, + "ocuCandidatePairingPreflight": ( + "ocu-candidate-pairing-preflight-smoke" + ), + }, + "sourceRepoBaselineAudit": { + "usableBaseline": "make record-and-replay-baseline-audit", + "strictOfficialGoldenGate": ( + "make record-and-replay-official-golden-gate-audit" + ), + "auditTargetDryRunSmoke": ( + "make record-and-replay-baseline-audit-targets-smoke" + ), + "baselineSummaryArtifact": ( + "dist/record-and-replay-baseline-summary.json" + ), + "copiedBaselineSummaryEvidence": ( + "evidence/source-baseline-summary.json" + ), + "strictOfficialGoldenSummaryArtifact": ( + "dist/record-and-replay-official-golden-gate-summary.json" + ), + "baselineSummaryEnvVar": "RNR_BASELINE_SUMMARY_JSON", + "strictOfficialGoldenSummaryEnvVar": ( + "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON" + ), + "strictOfficialGoldenExpectedFailureAudit": ( + "scripts/check-record-and-replay-baseline-summary.py " + "dist/record-and-replay-official-golden-gate-summary.json " + "--allow-strict-official-golden-missing" + ), + "verifiesSummaryArtifactSeparation": True, + "verifiesSummaryEnvVarIsolation": True, + }, + "standaloneRepoBoundary": { + "defaultChecksDoNotStartOfficialRecording": True, + "preflightScriptsRemainInOpenComputerUseRepo": True, + "doesNotCopyOpenComputerUseRuntimeSource": True, + }, + "hasSuccessfulRecordingGolden": has_successful_recording_golden, + "requiredSuccessfulRecordingScenarios": required_scenarios, + "recommendedSuccessfulRecordingScenarios": recommended_scenarios, + "scenarioRecipes": scenario_recipes, + "successfulRecordingGoldenRequiredFor": [ + "session file schema equivalence", + "event field schema equivalence", + "AX compact diff algorithm equivalence", + "screenshot trigger equivalence", + "timeout endReason equivalence", + ], + }, + "mcpServer": { + "command": "open-computer-use", + "args": ["event-stream", "mcp"], + "capabilities": {"tools": {"listChanged": False}}, + "tools": ["event_stream_start", "event_stream_status", "event_stream_stop"], + "requiresObjectParams": True, + "requiresStringToolName": True, + "requiresObjectArguments": True, + "rejectsUnexpectedArguments": True, + "rejectedRequestsDoNotCreateSessionFiles": True, + "noActiveResponse": { + "event_stream_status": { + "isRecording": False, + "maxDurationSeconds": 1800, + }, + "event_stream_stop": { + "isRecording": False, + "maxDurationSeconds": 1800, + }, + }, + "toolMetadata": [ + { + "name": "event_stream_start", + "description": "Start recording the user's actions for up to 30 minutes. If a recording is already active, return that active session instead of starting another one.", + "annotations": { + "destructiveHint": False, + "idempotentHint": False, + "openWorldHint": False, + "readOnlyHint": False, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, + { + "name": "event_stream_status", + "description": "Get the current or most recent Record & Replay recording status including paths to metadata and events during the recording.", + "annotations": { + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + "readOnlyHint": True, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, + { + "name": "event_stream_stop", + "description": "Stop the active event stream recording if one is running and return status including paths to metadata and events during the recording.", + "annotations": { + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + "readOnlyHint": False, + }, + "inputSchema": { + "additionalProperties": False, + "properties": {}, + "type": "object", + }, + }, + ], + }, + "extensionLayer": { + "officialCompatibleMcpSurface": False, + "commands": [ + "event-stream start --json", + "event-stream status --json", + "event-stream stop --json", + "event-stream cancel --json", + "event-stream wait --json --session-id ", + "event-stream wait --json --session-id --notify-command ", + "event-stream validate --json --require-skill-draft ", + "event-stream summarize --json ", + "event-stream scaffold-skill --json ", + ], + "waitNotify": { + "resultFields": ["waitTimedOut", "waitSessionMatched"], + "stdin": "final status JSON", + "environmentVariables": [ + "OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON", + "OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID", + "OPEN_COMPUTER_USE_EVENT_STREAM_STATE", + "OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON", + "OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH", + "OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH", + "OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH", + "OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH", + ], + "callbackSkippedWhenWaitSessionMatchedFalse": True, + "callbackFailureMakesCliFail": True, + "callbackTimeoutMakesCliFail": True, + }, + }, + "recordingToSkill": { + "strictValidation": { + "command": "event-stream validate --json --strict-ocu --require-skill-draft ", + "requiresDeclaredHandoffPaths": [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + ], + "requiresMetadataSessionAlias": True, + "requiresScreenshotPathsInsideSession": True, + "requiresSkillDraftReady": True, + }, + "eventsOnlyValidation": { + "command": "event-stream validate --json --require-skill-draft ", + "provesMetadataSessionAlias": False, + "provesDeclaredHandoffPaths": False, + }, + "scaffoldSkill": { + "command": "event-stream scaffold-skill --json ", + "runsSkillDraftValidationGate": True, + }, + "rejectsCancelledRecordings": True, + }, + "checks": { + "packageSkill": "scripts/package-skill.sh", + "packageArtifact": "scripts/verify-package-artifact.py", + "manifestContract": "scripts/verify-manifest.py", + "sourceBaselineSummaryEvidence": "scripts/verify-source-baseline-summary.py", + "readmeHandoffContract": "scripts/verify-readme-handoff.py", + "runtimeContract": "scripts/verify-runtime.py", + "skillWorkflow": "scripts/verify-skill-workflow.py", + "waitNotifyContractSmoke": "scripts/wait-notify-contract-smoke.py", + "recordingToSkillSmoke": "scripts/recording-to-skill-smoke.py", + "selfCheck": "scripts/check.sh", + }, + "optionalChecks": { + "recordingLifecycleSmoke": "scripts/recording-lifecycle-smoke.py", + }, + } + write_text(output_dir / "record-and-replay-skill-repo.json", json.dumps(manifest, indent=2) + "\n") + + return { + "ok": True, + "outputDir": str(output_dir), + "skillPath": str(skill_dir / "SKILL.md"), + "manifestPath": str(output_dir / "record-and-replay-skill-repo.json"), + } + + +def main() -> int: + args = parse_args() + try: + result = scaffold(args) + except ValueError as error: + print(json.dumps({"ok": False, "errors": [str(error)]}, indent=2, sort_keys=True), file=sys.stderr) + return 1 + print(json.dumps(result, indent=2, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/summarize-event-stream-recording.py b/scripts/summarize-event-stream-recording.py new file mode 100755 index 00000000..36d61eec --- /dev/null +++ b/scripts/summarize-event-stream-recording.py @@ -0,0 +1,846 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import sys +from collections import Counter, OrderedDict + + +ACTION_EVENT_TYPES = { + "mouse.click", + "mouse.context_menu", + "mouse.drag", + "keyboard.text_input", + "keyboard.submit", + "keyboard.shortcut", + "terminal.value_changed", + "selection.changed", +} + +CONTEXT_EVENT_TYPES = { + "session.started", + "session.ended", + "window.changed", + "AX.focusedWindowChanged", + "debug.error", + "experimentalRawEvents", +} + +SAFETY_KEYWORDS = [ + ("send", "sendAction"), + ("delete", "deleteAction"), + ("remove", "deleteAction"), + ("trash", "deleteAction"), + ("archive", "archiveAction"), + ("purchase", "purchaseAction"), + ("buy", "purchaseAction"), + ("pay", "paymentAction"), + ("approve", "approvalAction"), + ("upload", "uploadAction"), + ("publish", "publishAction"), + ("share", "shareAction"), + ("invite", "inviteAction"), + ("submit", "submitAction"), + ("save", "saveAction"), +] + +BLOCKING_DIAGNOSTIC_REASONS = {"inputMonitorsUnavailable"} + +ACTION_SEQUENCE_LIMIT = 50 +RUNTIME_INPUTS_LIMIT = 50 +SAFETY_SIGNALS_LIMIT = 50 +ELEMENT_LIST_LIMIT = 25 +DIAGNOSTICS_LIMIT = 25 +SCREENSHOT_PATHS_LIMIT = 25 + + +def load_json(path): + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def read_jsonl(path): + records = [] + if not path.exists(): + raise ValueError(f"missing JSONL file: {path}") + for index, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSONL in {path}:{index}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}:{index}") + value["_line"] = index + records.append(value) + return records + + +def resolve_input(path): + def path_from_metadata(raw, fallback): + if not raw: + return fallback + candidate = pathlib.Path(raw) + if candidate.is_absolute(): + return candidate + return path / candidate + + if path.is_dir(): + metadata_path = path / "metadata.json" + session_path = path / "session.json" + if metadata_path.exists(): + metadata = load_json(metadata_path) + elif session_path.exists(): + metadata_path = session_path + metadata = load_json(metadata_path) + else: + raise ValueError(f"session directory has no metadata.json or session.json: {path}") + events_path = path_from_metadata(metadata.get("eventsPath"), path / "events.jsonl") + return path, metadata_path, metadata, events_path + + if not path.exists(): + raise ValueError(f"path does not exist: {path}") + + if path.name in {"metadata.json", "session.json"}: + metadata = load_json(path) + session_dir = path.parent + raw_events_path = metadata.get("eventsPath") + if raw_events_path: + candidate = pathlib.Path(raw_events_path) + events_path = candidate if candidate.is_absolute() else session_dir / candidate + else: + events_path = session_dir / "events.jsonl" + return path.parent, path, metadata, events_path + + if path.name == "events.jsonl": + return path.parent, None, {}, path + + raise ValueError("input must be a session directory, metadata.json, session.json, or events.jsonl") + + +def first_string(value, keys): + if not isinstance(value, dict): + return None + for key in keys: + raw = value.get(key) + if isinstance(raw, str) and raw: + return raw + return None + + +def nested_dict(value, keys): + if not isinstance(value, dict): + return None + for key in keys: + raw = value.get(key) + if isinstance(raw, dict): + return raw + return None + + +def window_signature(event): + window = nested_dict(event, ["window", "windowContext", "targetWindow"]) + app = nested_dict(event, ["app", "application", "appContext"]) + + app_name = first_string(event, ["appName", "applicationName", "localizedName"]) + bundle_id = first_string(event, ["bundleIdentifier", "bundleId", "appBundleIdentifier"]) + window_title = first_string(event, ["windowTitle", "title"]) + + if app: + app_name = app_name or first_string(app, ["name", "appName", "localizedName"]) + bundle_id = bundle_id or first_string(app, ["bundleIdentifier", "bundleId"]) + if window: + window_title = window_title or first_string(window, ["title", "windowTitle"]) + + if not app_name and not bundle_id and not window_title: + return None + return { + "appName": app_name, + "bundleIdentifier": bundle_id, + "windowTitle": window_title, + } + + +def element_summary(event, include_text): + element = nested_dict(event, ["targetAccessibilityElement", "focusedAccessibilityElement"]) + if not element: + return None + + summary = OrderedDict() + for key in ["role", "subrole", "title", "label", "description"]: + raw = element.get(key) + if isinstance(raw, str) and raw: + summary[key] = raw + elif isinstance(raw, bool): + summary[key] = raw + for key in ["value", "selectedText"]: + raw = element.get(key) + if isinstance(raw, str) and raw: + if include_text: + summary[key] = raw + summary[f"{key}Length"] = len(raw) + elif isinstance(raw, bool): + summary[key] = raw + actions = element.get("actions") + if isinstance(actions, list) and actions: + summary["actions"] = [action for action in actions if isinstance(action, str)][:8] + if element.get("secureInput") is True: + summary["secureInput"] = True + return summary or None + + +def append_unique(mapping, key, value): + if not key: + return + if key not in mapping: + mapping[key] = value + + +def event_kind(event): + kind = event.get("kind") + if isinstance(kind, str): + return kind + event_type = event.get("type") + if isinstance(event_type, str): + return event_type + return None + + +def summarize(path, require_action, include_text): + session_dir, metadata_path, metadata, events_path = resolve_input(path) + events = read_jsonl(events_path) + + event_types = Counter(kind for event in events if (kind := event_kind(event))) + action_events = [event for event in events if event_kind(event) in ACTION_EVENT_TYPES] + context_events = [event for event in events if event_kind(event) in CONTEXT_EVENT_TYPES] + + windows = OrderedDict() + target_elements = [] + focused_elements = [] + selected_text_events = [] + debug_errors = [] + blocking_diagnostics = [] + redaction_events = [] + screenshot_paths = [] + action_sequence = [] + runtime_inputs = [] + safety_signals = [] + + for event in events: + event_type = event_kind(event) + signature = window_signature(event) + if signature: + key = "|".join( + [ + signature.get("bundleIdentifier") or "", + signature.get("appName") or "", + signature.get("windowTitle") or "", + ] + ) + append_unique(windows, key, signature) + + action_type = action_type_for_event(event) + if action_type: + action = OrderedDict() + action["line"] = event.get("_line") + action["type"] = action_type + if isinstance(event.get("timestamp"), str): + action["timestamp"] = event["timestamp"] + if signature: + action["window"] = signature + for key in ["location", "startLocation", "endLocation", "key", "modifiers", "selectionCleared", "reason"]: + if key in event: + action[key] = event[key] + raw_events = event.get("experimentalRawEvents") + if isinstance(raw_events, list): + raw_event_types = [ + raw.get("eventType") + for raw in raw_events + if isinstance(raw, dict) and isinstance(raw.get("eventType"), str) + ] + if raw_event_types: + action["rawEventTypes"] = raw_event_types[:8] + scroll = next( + ( + raw + for raw in raw_events + if isinstance(raw, dict) and raw.get("eventType") == "scrollWheel" + ), + None, + ) + if scroll: + for key in ["scrollingDeltaX", "scrollingDeltaY", "hasPreciseScrollingDeltas"]: + if key in scroll: + action[key] = scroll[key] + if "text" in event: + text = event["text"] + if isinstance(text, str): + action["textLength"] = len(text) + else: + action["text"] = text + if "textLength" in event: + action["textLength"] = event["textLength"] + element = element_summary(event, include_text=include_text) + if element: + action["element"] = element + runtime_input = runtime_input_summary(event, action) + if runtime_input and len(runtime_inputs) < RUNTIME_INPUTS_LIMIT: + runtime_inputs.append(runtime_input) + safety_signal = safety_signal_summary(event, action) + if safety_signal and len(safety_signals) < SAFETY_SIGNALS_LIMIT: + safety_signals.append(safety_signal) + action_sequence.append(action) + + target = nested_dict(event, ["targetAccessibilityElement"]) + if target: + target_elements.append(element_summary({"targetAccessibilityElement": target}, include_text=include_text) or {}) + focused = nested_dict(event, ["focusedAccessibilityElement"]) + if focused: + focused_elements.append(element_summary({"focusedAccessibilityElement": focused}, include_text=include_text) or {}) + + if event_type == "selection.changed": + selected_text_events.append({ + "line": event.get("_line"), + "selectedTextLength": len(event.get("selectedText") or ""), + "selectionCleared": event.get("selectionCleared") is True, + }) + + if event_type == "debug.error": + if is_blocking_diagnostic(event): + blocking_diagnostics.append({ + "line": event.get("_line"), + "subsystem": event.get("subsystem"), + "reason": event.get("reason"), + "errorType": event.get("errorType"), + }) + debug_errors.append({ + "line": event.get("_line"), + "subsystem": event.get("subsystem"), + "reason": event.get("reason"), + "errorType": event.get("errorType"), + }) + + if event.get("secureInput") is True or event.get("redacted") is True: + redaction_events.append({ + "line": event.get("_line"), + "type": event_type, + "secureInput": event.get("secureInput") is True, + "redacted": event.get("redacted") is True, + }) + + payload = event.get("accessibilityInspectorPayload") + if isinstance(payload, dict): + screenshot_path = payload.get("screenshotPath") + if isinstance(screenshot_path, str) and screenshot_path: + screenshot_paths.append(screenshot_path) + + ended_reasons = { + event.get("endReason") + for event in events + if event_kind(event) == "session.ended" and isinstance(event.get("endReason"), str) + } + inferred_end_reason = metadata.get("endReason") or ( + next(iter(ended_reasons)) if len(ended_reasons) == 1 else None + ) + final_event_type = event_kind(events[-1]) if events else None + first_event_type = event_kind(events[0]) if events else None + session_started_is_first = first_event_type == "session.started" + session_started_not_first = event_types["session.started"] > 0 and not session_started_is_first + session_started_count_invalid = event_types["session.started"] != 1 + session_ended_is_final = final_event_type == "session.ended" + session_ended_not_final = event_types["session.ended"] > 0 and not session_ended_is_final + session_ended_count_invalid = event_types["session.ended"] > 1 + warnings = [] + recording_was_cancelled = ( + metadata.get("state") == "cancelled" + or inferred_end_reason == "recording_controls_cancelled" + ) + incomplete = recording_incomplete(metadata, event_types) + if not action_events: + warnings.append("recording has no high-level user action events") + if recording_was_cancelled: + warnings.append("recording was cancelled; do not create or update a skill from this event stream") + if incomplete: + warnings.append("recording is not complete; stop the recording before creating a skill") + if session_started_count_invalid: + warnings.append("recording must contain exactly one session.started event before creating a skill") + if session_started_not_first: + warnings.append("recording has events before session.started; start must be the first event before creating a skill") + if session_ended_not_final: + warnings.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + if session_ended_count_invalid: + warnings.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + if event_types["AX.focusedWindowChanged"] == 0: + warnings.append("recording has no AX focused window context") + if debug_errors: + warnings.append("recording includes debug.error events; inspect diagnostics before creating a skill") + if blocking_diagnostics: + warnings.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + if redaction_events: + warnings.append("recording includes redaction or secureInput signals; avoid copying sensitive values") + if safety_signals: + warnings.append("recording includes actions that may require explicit user confirmation") + + summary_limits = summary_limit_report( + action_event_count=len(action_events), + action_sequence_stored=min(len(action_sequence), ACTION_SEQUENCE_LIMIT), + runtime_input_count=sum( + 1 for event in events if runtime_input_summary(event, {"type": event_kind(event)}) + ), + runtime_inputs_stored=len(runtime_inputs), + safety_signal_count=sum( + 1 + for event in events + if safety_signal_summary( + event, + { + "line": event.get("_line"), + "type": event_kind(event), + "element": element_summary(event, include_text=include_text) or {}, + }, + ) + ), + safety_signals_stored=len(safety_signals), + target_element_count=len(target_elements), + target_elements_stored=min(len(target_elements), ELEMENT_LIST_LIMIT), + focused_element_count=len(focused_elements), + focused_elements_stored=min(len(focused_elements), ELEMENT_LIST_LIMIT), + selection_event_count=len(selected_text_events), + selection_events_stored=min(len(selected_text_events), ELEMENT_LIST_LIMIT), + debug_error_count=len(debug_errors), + debug_errors_stored=min(len(debug_errors), DIAGNOSTICS_LIMIT), + redaction_event_count=len(redaction_events), + redaction_events_stored=min(len(redaction_events), DIAGNOSTICS_LIMIT), + screenshot_path_count=len(screenshot_paths), + screenshot_paths_stored=min(len(screenshot_paths), SCREENSHOT_PATHS_LIMIT), + ) + if summary_limits["hasTruncatedSummary"]: + warnings.append("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill") + + skill_evidence = { + "hasActionEvents": bool(action_events), + "hasInputEvents": any( + (kind := event_kind(event)) is not None and kind.startswith("keyboard.") + for event in events + ), + "hasPointerEvents": any( + (kind := event_kind(event)) is not None and kind.startswith("mouse.") + for event in events + ), + "hasAXContext": event_types["AX.focusedWindowChanged"] > 0, + "hasTargetElements": bool(target_elements), + "hasFocusedElements": bool(focused_elements), + "hasSelectionSignals": bool(selected_text_events), + "hasTerminalSignals": event_types["terminal.value_changed"] > 0, + "hasScreenshots": bool(screenshot_paths), + "hasDebugErrors": bool(debug_errors), + "hasBlockingDiagnostics": bool(blocking_diagnostics), + "recordingIncomplete": incomplete, + "sessionStartedNotFirst": session_started_not_first, + "sessionStartedCountInvalid": session_started_count_invalid, + "sessionEndedNotFinal": session_ended_not_final, + "sessionEndedCountInvalid": session_ended_count_invalid, + "hasRedactionSignals": bool(redaction_events), + "hasSafetySignals": bool(safety_signals), + "hasTruncatedSummary": summary_limits["hasTruncatedSummary"], + } + skill_readiness = skill_readiness_from_evidence( + skill_evidence, + has_window_context=bool(windows), + recording_was_cancelled=recording_was_cancelled, + recording_incomplete=incomplete, + session_started_not_first=session_started_not_first, + session_started_count_invalid=session_started_count_invalid, + session_ended_not_final=session_ended_not_final, + session_ended_count_invalid=session_ended_count_invalid, + ) + + ok = True + errors = [] + if require_action and not action_events: + ok = False + errors.append("required at least one high-level user action event") + if require_action and recording_was_cancelled: + ok = False + errors.append("recording was cancelled; do not create or update a skill from this event stream") + if require_action and incomplete: + ok = False + errors.append("recording is not complete; stop the recording before creating a skill") + if require_action and session_started_count_invalid: + ok = False + errors.append("recording must contain exactly one session.started event before creating a skill") + if require_action and session_started_not_first: + ok = False + errors.append("recording has events before session.started; start must be the first event before creating a skill") + if require_action and session_ended_not_final: + ok = False + errors.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + if require_action and session_ended_count_invalid: + ok = False + errors.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + if require_action and blocking_diagnostics: + ok = False + errors.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + + result = { + "ok": ok, + "sessionDir": str(session_dir), + "eventsPath": str(events_path), + "sessionId": metadata.get("sessionId") or metadata.get("sessionID"), + "state": metadata.get("state"), + "endReason": inferred_end_reason, + "eventCount": len(events), + "eventTypes": dict(sorted(event_types.items())), + "sessionStartedCount": event_types["session.started"], + "firstEventType": first_event_type, + "sessionStartedIsFirst": session_started_is_first if first_event_type is not None else None, + "sessionEndedCount": event_types["session.ended"], + "finalEventType": final_event_type, + "sessionEndedIsFinal": session_ended_is_final if final_event_type is not None else None, + "actionEventCount": len(action_events), + "contextEventCount": len(context_events), + "windows": list(windows.values()), + "skillEvidence": skill_evidence, + "skillReadiness": skill_readiness, + "summaryLimits": summary_limits, + "runtimeInputs": runtime_inputs, + "safetySignals": safety_signals, + "actionSequence": action_sequence[:ACTION_SEQUENCE_LIMIT], + "targetElements": target_elements[:ELEMENT_LIST_LIMIT], + "focusedElements": focused_elements[:ELEMENT_LIST_LIMIT], + "selectionEvents": selected_text_events[:ELEMENT_LIST_LIMIT], + "debugErrors": debug_errors[:DIAGNOSTICS_LIMIT], + "blockingDiagnostics": blocking_diagnostics, + "redactionEvents": redaction_events[:DIAGNOSTICS_LIMIT], + "screenshotPaths": screenshot_paths[:SCREENSHOT_PATHS_LIMIT], + "includesRawText": include_text, + "warnings": warnings, + "errors": errors, + } + if metadata_path: + result["metadataPath"] = str(metadata_path) + return result + + +def action_type_for_event(event): + event_type = event_kind(event) + if event_type in ACTION_EVENT_TYPES: + return event_type + if event_type == "experimentalRawEvents" and event.get("reason") == "scrollWheel": + return event_type + return None + + +def runtime_input_summary(event, action): + event_type = action.get("type") + result = OrderedDict() + result["line"] = action.get("line") + result["sourceEventType"] = event_type + result["requiresUserValue"] = True + if isinstance(action.get("element"), dict) and action["element"]: + result["target"] = action["element"] + + if event_type == "keyboard.text_input": + result["kind"] = "text" + result["description"] = "Runtime text to enter into the observed target." + if "textLength" in action: + result["textLength"] = action["textLength"] + if event.get("secureInput") is True or event.get("redacted") is True: + result["sensitive"] = True + return result + + if event_type == "selection.changed": + result["kind"] = "selection" + result["description"] = ( + "Current selection or selected-content semantics if the workflow depends on it." + ) + selected_text = event.get("selectedText") + if isinstance(selected_text, str): + result["selectedTextLength"] = len(selected_text) + if "selectionCleared" in action: + result["selectionCleared"] = action["selectionCleared"] + return result + + return None + + +def summary_limit_report(**counts): + source_counts = { + "actionSequence": counts["action_event_count"], + "runtimeInputs": counts["runtime_input_count"], + "safetySignals": counts["safety_signal_count"], + "targetElements": counts["target_element_count"], + "focusedElements": counts["focused_element_count"], + "selectionEvents": counts["selection_event_count"], + "debugErrors": counts["debug_error_count"], + "redactionEvents": counts["redaction_event_count"], + "screenshotPaths": counts["screenshot_path_count"], + } + stored_counts = { + "actionSequence": counts["action_sequence_stored"], + "runtimeInputs": counts["runtime_inputs_stored"], + "safetySignals": counts["safety_signals_stored"], + "targetElements": counts["target_elements_stored"], + "focusedElements": counts["focused_elements_stored"], + "selectionEvents": counts["selection_events_stored"], + "debugErrors": counts["debug_errors_stored"], + "redactionEvents": counts["redaction_events_stored"], + "screenshotPaths": counts["screenshot_paths_stored"], + } + omitted_counts = { + key: max(0, source_counts[key] - stored_counts[key]) + for key in source_counts + } + return { + "hasTruncatedSummary": any(value > 0 for value in omitted_counts.values()), + "limits": { + "actionSequence": ACTION_SEQUENCE_LIMIT, + "runtimeInputs": RUNTIME_INPUTS_LIMIT, + "safetySignals": SAFETY_SIGNALS_LIMIT, + "targetElements": ELEMENT_LIST_LIMIT, + "focusedElements": ELEMENT_LIST_LIMIT, + "selectionEvents": ELEMENT_LIST_LIMIT, + "debugErrors": DIAGNOSTICS_LIMIT, + "redactionEvents": DIAGNOSTICS_LIMIT, + "screenshotPaths": SCREENSHOT_PATHS_LIMIT, + }, + "storedCounts": stored_counts, + "sourceCounts": source_counts, + "omittedCounts": omitted_counts, + } + + +def safety_keyword_reason(element): + parts = [] + if isinstance(element, dict): + for key in ["role", "title", "label", "description", "subrole"]: + raw = element.get(key) + if isinstance(raw, str) and raw: + parts.append(raw) + text = " ".join(parts).lower() + if not text: + return None + for keyword, reason in SAFETY_KEYWORDS: + if keyword in text: + return reason + return None + + +def is_blocking_diagnostic(event): + return ( + event_kind(event) == "debug.error" + and event.get("reason") in BLOCKING_DIAGNOSTIC_REASONS + ) + + +def recording_incomplete(metadata, event_types): + if metadata.get("state") == "recording" or metadata.get("active") is True: + return True + return event_types["session.ended"] == 0 + + +def safety_signal_summary(event, action): + event_type = action.get("type") + reason = None + if event_type == "keyboard.submit": + reason = "submitAction" + element = action.get("element") if isinstance(action.get("element"), dict) else {} + if reason is None: + reason = safety_keyword_reason(element) + if reason is None: + return None + + result = OrderedDict() + result["line"] = action.get("line") + result["sourceEventType"] = event_type + result["reason"] = reason + result["confirmationRequired"] = True + if isinstance(event.get("timestamp"), str): + result["timestamp"] = event["timestamp"] + if isinstance(action.get("window"), dict) and action["window"]: + result["window"] = action["window"] + if element: + result["target"] = element + return result + + +def skill_readiness_from_evidence( + skill_evidence, + has_window_context, + recording_was_cancelled=False, + recording_incomplete=False, + session_started_not_first=False, + session_started_count_invalid=False, + session_ended_not_final=False, + session_ended_count_invalid=False, +): + has_action_events = skill_evidence.get("hasActionEvents") is True + has_ax_context = skill_evidence.get("hasAXContext") is True + has_target_elements = skill_evidence.get("hasTargetElements") is True + has_focused_elements = skill_evidence.get("hasFocusedElements") is True + has_selection_signals = skill_evidence.get("hasSelectionSignals") is True + has_debug_errors = skill_evidence.get("hasDebugErrors") is True + has_blocking_diagnostics = skill_evidence.get("hasBlockingDiagnostics") is True + recording_incomplete = recording_incomplete or skill_evidence.get("recordingIncomplete") is True + session_started_not_first = session_started_not_first or skill_evidence.get("sessionStartedNotFirst") is True + session_started_count_invalid = ( + session_started_count_invalid or skill_evidence.get("sessionStartedCountInvalid") is True + ) + session_ended_not_final = session_ended_not_final or skill_evidence.get("sessionEndedNotFinal") is True + session_ended_count_invalid = ( + session_ended_count_invalid or skill_evidence.get("sessionEndedCountInvalid") is True + ) + has_redaction_signals = skill_evidence.get("hasRedactionSignals") is True + has_safety_signals = skill_evidence.get("hasSafetySignals") is True + has_truncated_summary = skill_evidence.get("hasTruncatedSummary") is True + + reasons = [] + if recording_was_cancelled: + reasons.append("recording was cancelled; do not create or update a skill from this event stream") + if recording_incomplete: + reasons.append("recording is not complete; stop the recording before creating a skill") + if session_started_count_invalid: + reasons.append("recording must contain exactly one session.started event before creating a skill") + if session_started_not_first: + reasons.append("recording has events before session.started; start must be the first event before creating a skill") + if session_ended_not_final: + reasons.append("recording has events after session.ended; stop or cancel must be the final event before creating a skill") + if session_ended_count_invalid: + reasons.append("recording has multiple session.ended events; stop or cancel must close the event stream exactly once") + if not has_action_events: + reasons.append("recording has no high-level user action events") + if not has_window_context: + reasons.append("recording has no stable app/window context") + if not has_ax_context: + reasons.append("recording has no AX focused window context") + if not (has_target_elements or has_focused_elements or has_selection_signals): + reasons.append("recording has no semantic target, focused element, or selection evidence") + if has_debug_errors: + reasons.append("recording includes debug.error diagnostics") + if has_blocking_diagnostics: + reasons.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + if has_redaction_signals: + reasons.append("recording includes redaction or secureInput signals") + if has_safety_signals: + reasons.append("recording includes actions that may require explicit user confirmation") + if has_truncated_summary: + reasons.append("recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill") + + if recording_was_cancelled: + status = "insufficient" + recommended_next_step = ( + "Acknowledge the cancellation and ask the user to re-record when they want to " + "create a skill." + ) + elif recording_incomplete: + status = "insufficient" + recommended_next_step = ( + "Stop the recording, then inspect the completed events before creating a skill." + ) + elif session_started_count_invalid: + status = "insufficient" + recommended_next_step = ( + "Discard this malformed recording and re-record so exactly one start event opens " + "the event stream." + ) + elif session_started_not_first: + status = "insufficient" + recommended_next_step = "Discard this malformed recording and re-record so session.started is the first event." + elif session_ended_not_final: + status = "insufficient" + recommended_next_step = ( + "Discard this malformed recording and re-record so the stop or cancel event closes " + "the event stream." + ) + elif session_ended_count_invalid: + status = "insufficient" + recommended_next_step = ( + "Discard this malformed recording and re-record so the stop or cancel event closes " + "the event stream exactly once." + ) + elif has_blocking_diagnostics: + status = "insufficient" + recommended_next_step = ( + "Fix the recording permissions or input monitoring issue, then ask the user to " + "re-record the workflow." + ) + elif not has_action_events: + status = "insufficient" + recommended_next_step = ( + "Ask the user to re-record after confirming permissions and demonstrating at least " + "one workflow action." + ) + elif reasons: + status = "needsReview" + recommended_next_step = ( + "Inspect events.jsonl, replace user-specific values with inputs, and refine the " + "generated skill before packaging." + ) + else: + status = "ready" + recommended_next_step = ( + "Create or refine a reusable skill from the recording, then validate the final " + "skill package." + ) + + return { + "status": status, + "canCreateSkillDraft": ( + has_action_events + and not recording_was_cancelled + and not recording_incomplete + and not session_started_count_invalid + and not session_started_not_first + and not session_ended_not_final + and not session_ended_count_invalid + and not has_blocking_diagnostics + ), + "requiresHumanReview": status != "ready", + "reasons": reasons, + "recommendedNextStep": recommended_next_step, + } + + +def main(): + parser = argparse.ArgumentParser(description="Summarize a Record & Replay event stream for replay/skill creation.") + parser.add_argument("path", type=pathlib.Path, help="Session directory, metadata.json, session.json, or events.jsonl") + parser.add_argument( + "--require-action", + action="store_true", + help="Exit non-zero when no high-level user action events are present.", + ) + parser.add_argument( + "--include-text", + action="store_true", + help="Include raw AX value/selectedText fields in summaries. By default only lengths are emitted.", + ) + args = parser.parse_args() + + try: + result = summarize(args.path, args.require_action, args.include_text) + except ValueError as error: + result = { + "ok": False, + "errors": [str(error)], + "warnings": [], + } + + output = json.dumps(result, indent=2, sort_keys=True) + if result["ok"]: + print(output) + return 0 + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-codex-record-and-replay-installer.mjs b/scripts/test-codex-record-and-replay-installer.mjs new file mode 100755 index 00000000..587b3e09 --- /dev/null +++ b/scripts/test-codex-record-and-replay-installer.mjs @@ -0,0 +1,57 @@ +#!/usr/bin/env node + +import { mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const scriptPath = path.join(repoRoot, "scripts", "install-codex-record-and-replay-mcp.sh"); + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function runInstaller(codexHome) { + const result = spawnSync(scriptPath, { + cwd: repoRoot, + env: { + ...process.env, + CODEX_HOME: codexHome, + }, + encoding: "utf8", + }); + + if (result.status !== 0) { + throw new Error(`installer failed\nstdout:\n${result.stdout}\nstderr:\n${result.stderr}`); + } + + return result.stdout; +} + +const tempRoot = mkdtempSync(path.join(tmpdir(), "ocu-rnr-installer-")); + +try { + const codexHome = path.join(tempRoot, "codex-home"); + const configPath = path.join(codexHome, "config.toml"); + const firstOutput = runInstaller(codexHome); + const firstConfig = readFileSync(configPath, "utf8"); + + assert(firstOutput.includes('Installed Codex MCP server "record_and_replay"'), "first install should report installation"); + assert(firstConfig.includes('[mcp_servers."record_and_replay"]'), "config should include record_and_replay MCP server"); + assert(firstConfig.includes('command = "open-computer-use"'), "config should launch open-computer-use"); + assert(firstConfig.includes('args = ["event-stream","mcp"]'), "config should launch event-stream mcp"); + + const secondOutput = runInstaller(codexHome); + const secondConfig = readFileSync(configPath, "utf8"); + + assert(secondOutput.includes('Codex MCP server "record_and_replay" is already installed'), "second install should be idempotent"); + assert(secondConfig === firstConfig, "second install should leave config unchanged"); + + process.stdout.write(JSON.stringify({ ok: true, installer: "codex-record-and-replay-mcp" }) + "\n"); +} finally { + rmSync(tempRoot, { recursive: true, force: true }); +} diff --git a/scripts/test-event-stream-fixture-import.py b/scripts/test-event-stream-fixture-import.py new file mode 100755 index 00000000..086322fd --- /dev/null +++ b/scripts/test-event-stream-fixture-import.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text( + "\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n" + ) + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + recording = root / "recording" + recording.mkdir() + events_path = recording / "events.jsonl" + metadata_path = recording / "metadata.json" + session_path = recording / "session.json" + suppressed_path = recording / "suppressed.jsonl" + + events = [ + { + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.000Z", + "sessionId": "secret-session-id", + "appName": "Secret App", + "bundleIdentifier": "com.example.secret", + }, + { + "type": "mouse.click", + "timestamp": "2026-06-26T00:00:01.000Z", + "sessionId": "secret-session-id", + "windowTitle": "Private Window Title", + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Sensitive Button", + "value": "secret-value", + "selectedText": "selected-secret", + "actions": ["AXPress"], + }, + "accessibilityInspectorPayload": { + "renderedText": "Visible private content", + "fullTree": [ + "0 Private full tree root", + "1 Private full tree child", + ], + "treeLines": [ + "+ Visible private tree line", + "~ Old private value -> New private value", + ], + "cumulativeTreeLines": [ + "- Removed private tree line", + ], + "screenshotPath": str(recording / "screenshots" / "private.png"), + }, + }, + { + "type": "session.ended", + "timestamp": "2026-06-26T00:00:02.000Z", + "sessionId": "secret-session-id", + "endReason": "recording_controls_stopped", + }, + ] + write_jsonl(events_path, events) + suppressed = [ + { + "type": "AX.snapshot.suppressed", + "timestamp": "2026-06-26T00:00:01.500Z", + "sessionId": "secret-session-id", + "reason": "snapshotTooLarge", + "windowTitle": "Suppressed Private Window", + "value": "suppressed-secret-value", + "path": str(recording / "screenshots" / "suppressed-sensitive.png"), + } + ] + write_jsonl(suppressed_path, suppressed) + + metadata = { + "sessionId": "secret-session-id", + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": len(suppressed), + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "suppressedEventsPath": str(suppressed_path), + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + transcript_path = recording / "probe.json" + transcript = { + "target": "official", + "command": ["/private/path/SkyComputerUseClient", "event-stream", "mcp"], + "cwd": "/private/path/plugin", + "recordingsDir": str(recording), + "repeatStartResponseShape": { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": "secret-session-id", + "currentSegmentEventsPath": str(events_path), + "currentSegmentMetadataPath": str(metadata_path), + } + ), + } + ] + } + }, + "repeatStopResponseShape": { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": "secret-session-id", + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "state": "stopped", + } + ), + } + ] + } + }, + "finalStatusResponseShape": { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": "secret-session-id", + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "state": "stopped", + } + ), + } + ] + } + }, + "transcript": [ + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }, + }, + { + "direction": "receive", + "message": { + "jsonrpc": "2.0", + "id": 1, + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": "secret-session-id", + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "windowTitle": "Private Window Title", + } + ), + } + ] + }, + }, + }, + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }, + }, + {"direction": "receive", "message": {"jsonrpc": "2.0", "id": 2, "result": {"content": []}}}, + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }, + }, + {"direction": "receive", "message": {"jsonrpc": "2.0", "id": 3, "result": {"content": []}}}, + ], + } + write_json(transcript_path, transcript) + + output_dir = root / "fixtures" + subprocess.run( + [ + sys.executable, + str(repo / "scripts/import-event-stream-fixture.py"), + str(recording), + "--name", + "sample-official-recording", + "--output-dir", + str(output_dir), + "--source", + "official", + "--scenario", + "simple-action-stop", + "--official-plugin-version", + "record-and-replay 1.0.857", + "--captured-at", + "2026-06-26", + "--mcp-transcript", + str(transcript_path), + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + fixture = output_dir / "sample-official-recording" + combined = "\n".join(path.read_text() for path in fixture.glob("*") if path.is_file()) + for secret in [ + "secret-session-id", + "Secret App", + "com.example.secret", + "Private Window Title", + "Sensitive Button", + "secret-value", + "selected-secret", + "Visible private content", + "Private full tree root", + "Private full tree child", + "Visible private tree line", + "Old private value", + "New private value", + "Removed private tree line", + "private.png", + "Suppressed Private Window", + "suppressed-secret-value", + "suppressed-sensitive.png", + "/private/path", + ]: + assert secret not in combined, secret + assert "+ ", + "fullTree": [""], + "treeLines": [""], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + }, + }, + { + "type": "mouse.click", + "sessionId": "ready", + "targetAccessibilityElement": {"role": "AXButton", "title": ""}, + }, + { + "type": "AX.focusedWindowChanged", + "sessionId": "ready", + "accessibilityInspectorPayload": { + "kind": "diff", + "diffFromPrevious": True, + "renderedText": "", + "treeLines": ["~ "], + "cumulativeDiffFromInitial": True, + "cumulativeRenderedText": "", + "cumulativeTreeLines": ["+ "], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + }, + }, + {"type": "session.ended", "sessionId": "ready", "endReason": "recording_controls_stopped"}, + ] + ready_suppressed_events = [ + { + "type": "AX.snapshot.suppressed", + "reason": "snapshotTooLarge", + "subsystem": "accessibility", + } + ] + mcp_transcript = { + "startResponseShape": {"result": {"content": []}}, + "repeatStartResponseShape": {"result": {"content": []}}, + "statusResponseShape": {"result": {"content": []}}, + "stopResponseShape": {"result": {"content": []}}, + "repeatStopResponseShape": {"result": {"content": []}}, + "finalStatusResponseShape": {"result": {"content": []}}, + "transcript": [ + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 1, + "method": "tools/call", + "params": {"name": "event_stream_start", "arguments": {}}, + }, + }, + {"direction": "receive", "message": {"jsonrpc": "2.0", "id": 1, "result": {"content": []}}}, + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "event_stream_status", "arguments": {}}, + }, + }, + {"direction": "receive", "message": {"jsonrpc": "2.0", "id": 2, "result": {"content": []}}}, + { + "direction": "send", + "message": { + "jsonrpc": "2.0", + "id": 3, + "method": "tools/call", + "params": {"name": "event_stream_stop", "arguments": {}}, + }, + }, + {"direction": "receive", "message": {"jsonrpc": "2.0", "id": 3, "result": {"content": []}}}, + ] + } + ready = make_recording( + root, + "ready", + ready_events, + fixture_manifest( + "ready", + len(ready_events), + mcp_transcript=True, + suppressed_event_count=len(ready_suppressed_events), + ), + mcp_transcript=mcp_transcript, + suppressed_events=ready_suppressed_events, + ) + + ready_result = subprocess.run( + [ + sys.executable, + str(script), + str(ready), + "--require-fixture-manifest", + "--require-source", + "official", + "--require-official-plugin-version", + "record-and-replay 1.0.857", + "--require-session-alias", + "--require-metadata-counts", + "--require-handoff-paths", + "--require-full-ax-payload", + "--require-diff-payload", + "--require-cumulative-diff", + "--require-event-type", + "mouse.click", + "--require-end-reason", + "recording_controls_stopped", + "--require-suppressed-events", + "--require-suppressed-event-type", + "AX.snapshot.suppressed", + "--require-mcp-transcript", + "--require-mcp-tool-response", + "event_stream_start", + "--require-mcp-tool-response", + "event_stream_status", + "--require-mcp-tool-response", + "event_stream_stop", + "--require-mcp-response-shape", + "startResponseShape", + "--require-mcp-response-shape", + "statusResponseShape", + "--require-mcp-response-shape", + "stopResponseShape", + "--require-mcp-response-shape", + "finalStatusResponseShape", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + ready_json = json.loads(ready_result.stdout) + assert ready_json["ok"] is True + assert ready_json["recordings"][0]["metadataSessionAliasComplete"] is True + assert ready_json["recordings"][0]["metadataSessionAliasMatches"] is True + assert ready_json["recordings"][0]["metadataEventCount"] == len(ready_events) + assert ready_json["recordings"][0]["metadataEventCountMatches"] is True + assert ready_json["recordings"][0]["metadataSuppressedEventCount"] == 1 + assert ready_json["recordings"][0]["metadataSuppressedEventCountMatches"] is True + assert ready_json["recordings"][0]["declaredPaths"]["metadataPath"]["exists"] is True + assert ready_json["recordings"][0]["declaredPaths"]["eventsPath"]["exists"] is True + assert ready_json["recordings"][0]["declaredPaths"]["sessionPath"]["exists"] is True + assert ready_json["recordings"][0]["declaredPaths"]["suppressedEventsPath"]["exists"] is True + assert ready_json["recordings"][0]["sessionStartedCount"] == 1 + assert ready_json["recordings"][0]["sessionStartedIsFirst"] is True + assert ready_json["recordings"][0]["endReasons"]["recording_controls_stopped"] >= 1 + assert ready_json["recordings"][0]["hasDiffAXPayload"] is True + assert ready_json["recordings"][0]["hasFullTreePayload"] is True + assert ready_json["recordings"][0]["hasCumulativeAXDiff"] is True + assert ready_json["recordings"][0]["mcpToolResponses"]["event_stream_start"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpToolResponses"]["event_stream_status"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpToolResponses"]["event_stream_stop"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpResponseShapes"]["startResponseShape"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpResponseShapes"]["statusResponseShape"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpResponseShapes"]["stopResponseShape"]["hasResult"] is True + assert ready_json["recordings"][0]["mcpResponseShapes"]["finalStatusResponseShape"]["hasResult"] is True + assert ready_json["recordings"][0]["suppressedEventCount"] == 1 + assert ready_json["recordings"][0]["suppressedEventTypes"]["AX.snapshot.suppressed"] == 1 + + official_ax_events = [ + {"kind": "session.started", "sessionId": "official-ax"}, + { + "kind": "window.changed", + "sessionId": "official-ax", + "ax": { + "mode": "fullTree", + "textRedacted": True, + "textLength": 42, + }, + }, + { + "kind": "mouse.click", + "sessionId": "official-ax", + "ax": { + "mode": "fullTree", + "textRedacted": True, + "textLength": 42, + }, + }, + { + "kind": "session.ended", + "sessionId": "official-ax", + "endReason": "recording_controls_stopped", + }, + ] + official_ax = make_recording( + root, + "official-ax", + official_ax_events, + fixture_manifest("official-ax", len(official_ax_events)), + ) + official_ax_result = subprocess.run( + [ + sys.executable, + str(script), + str(official_ax), + "--require-full-ax-payload", + "--require-event-type", + "mouse.click", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + official_ax_json = json.loads(official_ax_result.stdout) + assert official_ax_json["recordings"][0]["hasAccessibilityPayload"] is True + assert official_ax_json["recordings"][0]["hasFullAXPayload"] is True + assert official_ax_json["recordings"][0]["hasFullTreePayload"] is True + + official_session_alias = make_recording( + root, + "official-session-alias", + ready_events, + fixture_manifest("official-session-alias", len(ready_events)), + ) + official_metadata = json.loads((official_session_alias / "metadata.json").read_text()) + official_metadata["startedAt"] = "2026-06-26T00:00:00.000Z" + official_metadata["endedAt"] = "2026-06-26T00:00:05.000Z" + write_json(official_session_alias / "metadata.json", official_metadata) + write_json( + official_session_alias / "session.json", + { + "id": "official-session-alias", + "startedAt": "2026-06-26T00:00:00.000Z", + "endedAt": "2026-06-26T00:00:05.000Z", + "endReason": "recording_controls_stopped", + "eventsPath": "events.jsonl", + }, + ) + official_session_alias_result = subprocess.run( + [ + sys.executable, + str(script), + str(official_session_alias), + "--require-fixture-manifest", + "--require-session-alias", + "--allow-no-ax-payload", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + official_session_alias_json = json.loads(official_session_alias_result.stdout) + assert official_session_alias_json["ok"] is True + assert official_session_alias_json["recordings"][0]["metadataSessionAliasComplete"] is True + assert official_session_alias_json["recordings"][0]["metadataSessionAliasMatches"] is True + + def top_level_kind_only(event): + result = dict(event) + if "type" in result: + result["kind"] = result.pop("type") + return result + + kind_only_events = [top_level_kind_only(event) for event in ready_events] + kind_only_suppressed = [top_level_kind_only(event) for event in ready_suppressed_events] + kind_only = make_recording( + root, + "kind-only", + kind_only_events, + fixture_manifest( + "kind-only", + len(kind_only_events), + suppressed_event_count=len(kind_only_suppressed), + ), + suppressed_events=kind_only_suppressed, + ) + kind_only_result = subprocess.run( + [ + sys.executable, + str(script), + str(kind_only), + "--require-fixture-manifest", + "--require-session-alias", + "--require-metadata-counts", + "--require-handoff-paths", + "--require-full-ax-payload", + "--require-diff-payload", + "--require-cumulative-diff", + "--require-event-type", + "mouse.click", + "--require-end-reason", + "recording_controls_stopped", + "--require-suppressed-events", + "--require-suppressed-event-type", + "AX.snapshot.suppressed", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + kind_only_json = json.loads(kind_only_result.stdout) + assert kind_only_json["ok"] is True + assert kind_only_json["recordings"][0]["eventTypes"]["mouse.click"] == 1 + assert kind_only_json["recordings"][0]["sessionStartedIsFirst"] is True + assert kind_only_json["recordings"][0]["sessionEndedIsFinal"] is True + assert kind_only_json["recordings"][0]["suppressedEventTypes"]["AX.snapshot.suppressed"] == 1 + + lifecycle_events = [ + {"type": "session.started", "sessionId": "lifecycle"}, + {"type": "session.ended", "sessionId": "lifecycle", "endReason": "recording_controls_stopped"}, + ] + lifecycle = make_recording(root, "lifecycle", lifecycle_events, fixture_manifest("lifecycle", len(lifecycle_events))) + missing_session_alias = make_recording( + root, + "missing-session-alias", + lifecycle_events, + fixture_manifest("missing-session-alias", len(lifecycle_events)), + ) + (missing_session_alias / "session.json").unlink() + missing_session_alias_failure = subprocess.run( + [ + sys.executable, + str(script), + str(missing_session_alias), + "--require-fixture-manifest", + "--require-session-alias", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_session_alias_failure.returncode == 1 + missing_session_alias_json = json.loads(missing_session_alias_failure.stderr) + assert "missing session.json alias" in missing_session_alias_json["recordings"][0]["errors"] + assert missing_session_alias_json["recordings"][0]["metadataSessionAliasComplete"] is False + assert missing_session_alias_json["recordings"][0]["metadataSessionAliasMatches"] is None + + mismatched_session_alias = make_recording( + root, + "mismatched-session-alias", + lifecycle_events, + fixture_manifest("mismatched-session-alias", len(lifecycle_events)), + ) + mismatched_session_json = json.loads((mismatched_session_alias / "session.json").read_text()) + mismatched_session_json["endReason"] = "recording_controls_cancelled" + write_json(mismatched_session_alias / "session.json", mismatched_session_json) + mismatched_session_alias_failure = subprocess.run( + [ + sys.executable, + str(script), + str(mismatched_session_alias), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert mismatched_session_alias_failure.returncode == 1 + mismatched_session_alias_json = json.loads(mismatched_session_alias_failure.stderr) + assert "metadata.json and session.json differ" in mismatched_session_alias_json["recordings"][0]["errors"] + assert mismatched_session_alias_json["recordings"][0]["metadataSessionAliasComplete"] is True + assert mismatched_session_alias_json["recordings"][0]["metadataSessionAliasMatches"] is False + + bad_metadata_count = make_recording( + root, + "bad-metadata-count", + lifecycle_events, + fixture_manifest("bad-metadata-count", len(lifecycle_events)), + ) + bad_metadata = json.loads((bad_metadata_count / "metadata.json").read_text()) + bad_metadata["eventCount"] = 99 + write_json(bad_metadata_count / "metadata.json", bad_metadata) + write_json(bad_metadata_count / "session.json", bad_metadata) + bad_metadata_count_failure = subprocess.run( + [ + sys.executable, + str(script), + str(bad_metadata_count), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert bad_metadata_count_failure.returncode == 1 + bad_metadata_count_json = json.loads(bad_metadata_count_failure.stderr) + assert ( + "eventCount=99 does not match events.jsonl lines=2" + in bad_metadata_count_json["recordings"][0]["errors"] + ) + assert bad_metadata_count_json["recordings"][0]["metadataEventCountMatches"] is False + + missing_metadata_counts = make_recording( + root, + "missing-metadata-counts", + lifecycle_events, + fixture_manifest("missing-metadata-counts", len(lifecycle_events)), + ) + missing_counts_metadata = json.loads((missing_metadata_counts / "metadata.json").read_text()) + del missing_counts_metadata["eventCount"] + del missing_counts_metadata["suppressedEventCount"] + write_json(missing_metadata_counts / "metadata.json", missing_counts_metadata) + write_json(missing_metadata_counts / "session.json", missing_counts_metadata) + missing_metadata_counts_failure = subprocess.run( + [ + sys.executable, + str(script), + str(missing_metadata_counts), + "--require-fixture-manifest", + "--require-metadata-counts", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_metadata_counts_failure.returncode == 1 + missing_metadata_counts_json = json.loads(missing_metadata_counts_failure.stderr) + missing_metadata_count_errors = missing_metadata_counts_json["recordings"][0]["errors"] + assert "missing metadata eventCount" in missing_metadata_count_errors + assert "missing metadata suppressedEventCount" in missing_metadata_count_errors + assert missing_metadata_counts_json["recordings"][0]["metadataEventCountMatches"] is None + + missing_handoff_paths = make_recording( + root, + "missing-handoff-paths", + lifecycle_events, + fixture_manifest("missing-handoff-paths", len(lifecycle_events)), + ) + missing_handoff_metadata = json.loads((missing_handoff_paths / "metadata.json").read_text()) + del missing_handoff_metadata["metadataPath"] + del missing_handoff_metadata["sessionPath"] + del missing_handoff_metadata["eventsPath"] + del missing_handoff_metadata["suppressedEventsPath"] + write_json(missing_handoff_paths / "metadata.json", missing_handoff_metadata) + write_json(missing_handoff_paths / "session.json", missing_handoff_metadata) + missing_handoff_paths_failure = subprocess.run( + [ + sys.executable, + str(script), + str(missing_handoff_paths), + "--require-fixture-manifest", + "--require-handoff-paths", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_handoff_paths_failure.returncode == 1 + missing_handoff_paths_json = json.loads(missing_handoff_paths_failure.stderr) + missing_handoff_path_errors = missing_handoff_paths_json["recordings"][0]["errors"] + assert "missing metadataPath" in missing_handoff_path_errors + assert "missing eventsPath" in missing_handoff_path_errors + assert "missing sessionPath" in missing_handoff_path_errors + assert "missing suppressedEventsPath" in missing_handoff_path_errors + + bad_session_path = make_recording( + root, + "bad-session-path", + lifecycle_events, + fixture_manifest("bad-session-path", len(lifecycle_events)), + ) + bad_session_path_metadata = json.loads((bad_session_path / "metadata.json").read_text()) + bad_session_path_metadata["sessionPath"] = "missing-session.json" + write_json(bad_session_path / "metadata.json", bad_session_path_metadata) + write_json(bad_session_path / "session.json", bad_session_path_metadata) + bad_session_path_failure = subprocess.run( + [ + sys.executable, + str(script), + str(bad_session_path), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert bad_session_path_failure.returncode == 1 + bad_session_path_json = json.loads(bad_session_path_failure.stderr) + assert "sessionPath does not exist: missing-session.json" in bad_session_path_json["recordings"][0]["errors"] + assert bad_session_path_json["recordings"][0]["declaredPaths"]["sessionPath"]["exists"] is False + + bad_events_path = make_recording( + root, + "bad-events-path", + lifecycle_events, + fixture_manifest("bad-events-path", len(lifecycle_events)), + ) + bad_events_path_metadata = json.loads((bad_events_path / "metadata.json").read_text()) + bad_events_path_metadata["eventsPath"] = "missing-events.jsonl" + write_json(bad_events_path / "metadata.json", bad_events_path_metadata) + write_json(bad_events_path / "session.json", bad_events_path_metadata) + bad_events_path_failure = subprocess.run( + [ + sys.executable, + str(script), + str(bad_events_path), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert bad_events_path_failure.returncode == 1 + bad_events_path_json = json.loads(bad_events_path_failure.stderr) + bad_events_path_errors = bad_events_path_json["recordings"][0]["errors"] + assert "eventsPath does not exist: missing-events.jsonl" in bad_events_path_errors + assert any(error.startswith("missing JSONL file: ") for error in bad_events_path_errors) + assert bad_events_path_json["recordings"][0]["declaredPaths"]["eventsPath"]["exists"] is False + + bad_manifest_suppressed_count = make_recording( + root, + "bad-manifest-suppressed-count", + lifecycle_events, + fixture_manifest("bad-manifest-suppressed-count", len(lifecycle_events), suppressed_event_count=7), + ) + bad_manifest_suppressed_count_failure = subprocess.run( + [ + sys.executable, + str(script), + str(bad_manifest_suppressed_count), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert bad_manifest_suppressed_count_failure.returncode == 1 + bad_manifest_suppressed_count_json = json.loads(bad_manifest_suppressed_count_failure.stderr) + assert ( + "fixture manifest suppressedEventCount=7 does not match suppressed.jsonl lines=0" + in bad_manifest_suppressed_count_json["recordings"][0]["errors"] + ) + + missing_suppressed_failure = subprocess.run( + [ + sys.executable, + str(script), + str(lifecycle), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + "--require-suppressed-events", + "--require-suppressed-event-type", + "AX.snapshot.suppressed", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_suppressed_failure.returncode == 1 + missing_suppressed_json = json.loads(missing_suppressed_failure.stderr) + missing_suppressed_errors = missing_suppressed_json["recordings"][0]["errors"] + assert "missing suppressed events" in missing_suppressed_errors + assert "missing required suppressed event type: AX.snapshot.suppressed" in missing_suppressed_errors + + missing_end_reason_failure = subprocess.run( + [ + sys.executable, + str(script), + str(lifecycle), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + "--require-end-reason", + "recording_controls_cancelled", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_end_reason_failure.returncode == 1 + missing_end_reason_json = json.loads(missing_end_reason_failure.stderr) + assert ( + "missing required endReason: recording_controls_cancelled" + in missing_end_reason_json["recordings"][0]["errors"] + ) + + lifecycle_failure = subprocess.run( + [sys.executable, str(script), str(lifecycle), "--require-fixture-manifest"], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert lifecycle_failure.returncode == 1 + failure_json = json.loads(lifecycle_failure.stderr) + errors = failure_json["recordings"][0]["errors"] + assert any("missing action event" in error for error in errors) + assert any("missing accessibilityInspectorPayload" in error for error in errors) + + missing_start_events = [ + {"type": "mouse.click", "sessionId": "missing-start"}, + { + "type": "session.ended", + "sessionId": "missing-start", + "endReason": "recording_controls_stopped", + }, + ] + missing_start = make_recording( + root, + "missing-start", + missing_start_events, + fixture_manifest("missing-start", len(missing_start_events)), + ) + missing_start_failure = subprocess.run( + [ + sys.executable, + str(script), + str(missing_start), + "--require-fixture-manifest", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_start_failure.returncode == 1 + missing_start_json = json.loads(missing_start_failure.stderr) + assert "missing session.started event" in missing_start_json["recordings"][0]["errors"] + assert missing_start_json["recordings"][0]["sessionStartedCount"] == 0 + + lifecycle_pass = subprocess.run( + [ + sys.executable, + str(script), + str(lifecycle), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert json.loads(lifecycle_pass.stdout)["ok"] is True + + duplicate_end_events = [ + {"type": "session.started", "sessionId": "duplicate-end"}, + { + "type": "session.ended", + "sessionId": "duplicate-end", + "endReason": "recording_controls_stopped", + }, + { + "type": "session.ended", + "sessionId": "duplicate-end", + "endReason": "recording_controls_stopped", + }, + ] + duplicate_end = make_recording( + root, + "duplicate-end", + duplicate_end_events, + fixture_manifest("duplicate-end", len(duplicate_end_events)), + ) + duplicate_end_failure = subprocess.run( + [ + sys.executable, + str(script), + str(duplicate_end), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert duplicate_end_failure.returncode == 1 + duplicate_end_json = json.loads(duplicate_end_failure.stderr) + assert "multiple session.ended events" in duplicate_end_json["recordings"][0]["errors"] + assert duplicate_end_json["recordings"][0]["sessionEndedCount"] == 2 + assert duplicate_end_json["recordings"][0]["sessionEndedIsFinal"] is True + + end_not_final_events = [ + {"type": "session.started", "sessionId": "end-not-final"}, + { + "type": "session.ended", + "sessionId": "end-not-final", + "endReason": "recording_controls_stopped", + }, + {"type": "debug.error", "sessionId": "end-not-final", "reason": "lateEvent"}, + ] + end_not_final = make_recording( + root, + "end-not-final", + end_not_final_events, + fixture_manifest("end-not-final", len(end_not_final_events)), + ) + end_not_final_failure = subprocess.run( + [ + sys.executable, + str(script), + str(end_not_final), + "--require-fixture-manifest", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert end_not_final_failure.returncode == 1 + end_not_final_json = json.loads(end_not_final_failure.stderr) + assert "session.ended is not the final event" in end_not_final_json["recordings"][0]["errors"] + assert end_not_final_json["recordings"][0]["sessionEndedCount"] == 1 + assert end_not_final_json["recordings"][0]["sessionEndedIsFinal"] is False + + timeout_transcript = { + "startResponseShape": {"timeout": 5.0}, + "repeatStartResponseShape": {"timeout": 5.0}, + "transcriptShape": [ + { + "direction": "send", + "id": 3, + "method": "tools/call", + "timeout": None, + }, + { + "direction": "receive", + "id": None, + "method": None, + "timeout": 5.0, + }, + ], + } + timeout_fixture = make_recording( + root, + "timeout", + lifecycle_events, + fixture_manifest("timeout", len(lifecycle_events), mcp_transcript=True), + mcp_transcript=timeout_transcript, + ) + timeout_failure = subprocess.run( + [ + sys.executable, + str(script), + str(timeout_fixture), + "--require-fixture-manifest", + "--require-mcp-transcript", + "--require-mcp-tool-response", + "event_stream_start", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert timeout_failure.returncode == 1 + timeout_json = json.loads(timeout_failure.stderr) + timeout_errors = timeout_json["recordings"][0]["errors"] + assert "MCP tool timed out: event_stream_start" in timeout_errors + assert timeout_json["recordings"][0]["mcpResponseShapes"]["startResponseShape"]["timedOut"] is True + assert timeout_json["recordings"][0]["mcpToolResponses"]["event_stream_start"]["timedOut"] is True + + repeat_timeout_failure = subprocess.run( + [ + sys.executable, + str(script), + str(timeout_fixture), + "--require-fixture-manifest", + "--require-mcp-transcript", + "--require-mcp-response-shape", + "repeatStartResponseShape", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert repeat_timeout_failure.returncode == 1 + repeat_timeout_json = json.loads(repeat_timeout_failure.stderr) + repeat_timeout_errors = repeat_timeout_json["recordings"][0]["errors"] + assert "MCP response shape timed out: repeatStartResponseShape" in repeat_timeout_errors + assert ( + repeat_timeout_json["recordings"][0]["mcpResponseShapes"]["repeatStartResponseShape"]["timedOut"] + is True + ) + + mixed_transcript = { + "startResponseShape": {"result": {"content": []}}, + "repeatStartResponseShape": {"timeout": 5.0}, + } + mixed_fixture = make_recording( + root, + "mixed", + lifecycle_events, + fixture_manifest("mixed", len(lifecycle_events), mcp_transcript=True), + mcp_transcript=mixed_transcript, + ) + mixed_tool_pass = subprocess.run( + [ + sys.executable, + str(script), + str(mixed_fixture), + "--require-fixture-manifest", + "--require-mcp-transcript", + "--require-mcp-tool-response", + "event_stream_start", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + mixed_tool_json = json.loads(mixed_tool_pass.stdout) + assert mixed_tool_json["recordings"][0]["mcpToolResponses"]["event_stream_start"]["hasResult"] is True + assert mixed_tool_json["recordings"][0]["mcpToolResponses"]["event_stream_start"]["timedOut"] is False + mixed_shape_failure = subprocess.run( + [ + sys.executable, + str(script), + str(mixed_fixture), + "--require-fixture-manifest", + "--require-mcp-transcript", + "--require-mcp-response-shape", + "repeatStartResponseShape", + "--allow-no-action", + "--allow-no-ax-payload", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert mixed_shape_failure.returncode == 1 + mixed_shape_json = json.loads(mixed_shape_failure.stderr) + assert ( + "MCP response shape timed out: repeatStartResponseShape" + in mixed_shape_json["recordings"][0]["errors"] + ) + + print(json.dumps({"ok": True}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-event-stream-local-probe.py b/scripts/test-event-stream-local-probe.py new file mode 100755 index 00000000..dd8856e3 --- /dev/null +++ b/scripts/test-event-stream-local-probe.py @@ -0,0 +1,196 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import re +import subprocess +import sys +import tempfile + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +PROBE = REPO_ROOT / "scripts/probe-event-stream-recording.py" +VALIDATOR = REPO_ROOT / "scripts/validate-event-stream-recording.py" +LOCAL_CLIENT = REPO_ROOT / ".build/debug/OpenComputerUse" +ABSOLUTE_PATH_PATTERN = re.compile(r"(/Users/|/var/folders/|/private/var/|/tmp/)") +EXPECTED_TOOL_NAMES = [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", +] + + +def run(command: list[str], **kwargs) -> subprocess.CompletedProcess: + return subprocess.run( + command, + cwd=REPO_ROOT, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + **kwargs, + ) + + +def content_text_json(shape: dict) -> dict: + content = shape["result"]["content"] + assert len(content) == 1 + item = content[0] + assert item["type"] == "text" + assert "text" not in item + decoded = item["textJSON"] + assert isinstance(decoded, dict) + return decoded + + +def assert_redacted_path(payload: dict, key: str) -> None: + assert payload[key] == f"", payload + + +def assert_active_payload(payload: dict) -> None: + assert payload["state"] == "recording" + assert payload["active"] is True + assert payload["isRecording"] is True + assert payload["maxDurationSeconds"] == 1800 + assert payload["eventCount"] >= 3 + assert payload["suppressedEventCount"] == 0 + for key in ( + "eventsPath", + "metadataPath", + "sessionPath", + "suppressedEventsPath", + "currentSegmentEventsPath", + "currentSegmentMetadataPath", + ): + assert_redacted_path(payload, key) + + +def assert_stopped_payload(payload: dict) -> None: + assert payload["state"] == "stopped" + assert payload["active"] is False + assert payload["isRecording"] is False + assert payload["maxDurationSeconds"] == 1800 + assert payload["endReason"] == "recording_controls_stopped" + assert payload["eventCount"] >= 4 + assert "currentSegmentEventsPath" not in payload + assert "currentSegmentMetadataPath" not in payload + for key in ("eventsPath", "metadataPath", "sessionPath", "suppressedEventsPath"): + assert_redacted_path(payload, key) + + +def main() -> int: + run(["swift", "build", "--product", "OpenComputerUse"]) + + with tempfile.TemporaryDirectory(prefix="event-stream-local-probe-") as tmp: + tmpdir = pathlib.Path(tmp) + output_path = tmpdir / "probe-output.json" + fixture_path = tmpdir / "probe-fixture.json" + recordings_dir = tmpdir / "recordings" + + run( + [ + sys.executable, + str(PROBE), + "--target", + "local", + "--client", + str(LOCAL_CLIENT), + "--recordings-dir", + str(recordings_dir), + "--start-stop", + "--timeout", + "5", + "--output", + str(output_path), + "--fixture-output", + str(fixture_path), + ] + ) + + output = json.loads(output_path.read_text(encoding="utf-8")) + fixture_text = fixture_path.read_text(encoding="utf-8") + if ABSOLUTE_PATH_PATTERN.search(fixture_text): + raise AssertionError("local probe fixture contains a local absolute path") + fixture = json.loads(fixture_text) + + assert output["ok"] is True + assert output["startStopRequested"] is True + assert output["startStopCompleted"] is True + assert output["startTimedOut"] is False + assert output["process"]["returncode"] == 0 + assert fixture["fixtureFormatVersion"] == 1 + assert fixture["kind"] == "record-and-replay-event-stream-probe" + assert fixture["source"] == "local" + assert fixture["officialPluginVersion"] is None + assert fixture["protocolVersion"] == "2025-11-25" + assert fixture["serverInfo"]["name"] == "Record & Replay" + assert fixture["toolNames"] == EXPECTED_TOOL_NAMES + assert fixture["elicitationRequests"] == [] + + start_payload = content_text_json(fixture["startResponseShape"]) + repeat_start_payload = content_text_json(fixture["repeatStartResponseShape"]) + status_payload = content_text_json(fixture["statusResponseShape"]) + stop_payload = content_text_json(fixture["stopResponseShape"]) + repeat_stop_payload = content_text_json(fixture["repeatStopResponseShape"]) + final_status_payload = content_text_json(fixture["finalStatusResponseShape"]) + assert_active_payload(start_payload) + assert_active_payload(repeat_start_payload) + assert_active_payload(status_payload) + assert_stopped_payload(stop_payload) + assert_stopped_payload(repeat_stop_payload) + assert_stopped_payload(final_status_payload) + + assert output["startStatus"]["sessionId"] == output["repeatStartStatus"]["sessionId"] + assert output["startStatus"]["eventsPath"] == output["repeatStartStatus"]["eventsPath"] + assert output["repeatStartStatus"]["state"] == "recording" + assert output["repeatStartStatus"]["active"] is True + assert output["stopStatus"]["sessionId"] == output["repeatStopStatus"]["sessionId"] + assert output["stopStatus"]["eventsPath"] == output["repeatStopStatus"]["eventsPath"] + assert output["stopStatus"]["sessionId"] == output["finalStatusStatus"]["sessionId"] + assert output["repeatStopStatus"]["state"] == "stopped" + assert output["repeatStopStatus"]["active"] is False + assert output["finalStatusStatus"]["state"] == "stopped" + assert output["finalStatusStatus"]["active"] is False + + metadata_path = pathlib.Path(output["stopStatus"]["metadataPath"]) + session_path = pathlib.Path(output["stopStatus"]["sessionPath"]) + events_path = pathlib.Path(output["stopStatus"]["eventsPath"]) + assert metadata_path.exists() + assert session_path.exists() + assert events_path.exists() + + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + assert "currentSegmentEventsPath" not in metadata + assert "currentSegmentMetadataPath" not in metadata + + run( + [ + sys.executable, + str(VALIDATOR), + str(metadata_path), + "--strict-ocu", + "--require-event-type", + "session.started", + "--require-event-type", + "session.ended", + ] + ) + runtime_validation = run( + [ + str(LOCAL_CLIENT), + "event-stream", + "validate", + "--json", + "--strict-ocu", + str(session_path), + ] + ) + runtime_payload = json.loads(runtime_validation.stdout) + assert runtime_payload["ok"] is True, runtime_payload + + print(json.dumps({"ok": True, "probe": "event-stream-local"})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-event-stream-official-fixture-coverage.py b/scripts/test-event-stream-official-fixture-coverage.py new file mode 100755 index 00000000..0f7806a3 --- /dev/null +++ b/scripts/test-event-stream-official-fixture-coverage.py @@ -0,0 +1,370 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import scenario_recipe + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def make_fixture(root, name, source="official", scenario="simple-action-stop"): + fixture = root / name + fixture.mkdir(parents=True) + write_json( + fixture / "fixture-manifest.json", + { + "fixtureFormatVersion": 1, + "name": name, + "source": source, + "scenario": scenario, + "scenarioRecipe": scenario_recipe(scenario), + "officialPluginVersion": "record-and-replay 1.0.857" if source == "official" else None, + "eventCount": 4, + }, + ) + return fixture + + +def write_jsonl(path, records): + path.write_text("\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n") + + +def response_shape(paths): + return { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": str(paths["metadata"]), + "sessionPath": str(paths["session"]), + "eventsPath": str(paths["events"]), + "suppressedEventsPath": str(paths["suppressed"]), + } + ), + } + ] + } + } + + +def make_ready_fixture(root, name, scenario="simple-action-stop"): + fixture = make_fixture(root, name, source="official", scenario=scenario) + metadata_path = fixture / "metadata.json" + session_path = fixture / "session.json" + events_path = fixture / "events.jsonl" + suppressed_path = fixture / "suppressed.jsonl" + events = [ + {"type": "session.started", "sessionId": "ready-session"}, + { + "type": "AX.focusedWindowChanged", + "sessionId": "ready-session", + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "treeLines": ["AXWindow Ready"], + }, + }, + { + "type": "mouse.click", + "sessionId": "ready-session", + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": {"role": "AXButton", "title": "Ready"}, + }, + { + "type": "session.ended", + "sessionId": "ready-session", + "endReason": "recording_controls_stopped", + }, + ] + write_jsonl(events_path, events) + write_jsonl(suppressed_path, []) + metadata = { + "sessionId": "ready-session", + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": 0, + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "eventsPath": str(events_path), + "suppressedEventsPath": str(suppressed_path), + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + shape = response_shape( + { + "metadata": metadata_path, + "session": session_path, + "events": events_path, + "suppressed": suppressed_path, + } + ) + write_json( + fixture / "mcp-transcript.json", + { + "startResponseShape": shape, + "repeatStartResponseShape": shape, + "statusResponseShape": shape, + "stopResponseShape": shape, + "repeatStopResponseShape": shape, + "finalStatusResponseShape": shape, + }, + ) + return fixture + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + script = repo / "scripts/check-event-stream-official-fixture-coverage.py" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + fixture_root = root / "fixtures" + + missing = run([sys.executable, str(script), "--fixture-root", str(fixture_root)], repo) + assert missing.returncode == 1 + missing_json = json.loads(missing.stderr) + assert missing_json["coverageOk"] is False + assert missing_json["hasRequiredOfficialSuccessfulFixture"] is False + assert missing_json["missingOfficialScenarios"] == ["simple-action-stop"] + assert missing_json["recommendedOfficialScenarios"] == [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", + ] + assert missing_json["missingRecommendedOfficialScenarios"] == [ + "cancel", + "drag-stop", + "keyboard-input-stop", + "simple-action-stop", + "timeout", + ] + assert missing_json["hasRecommendedOfficialScenarioCoverage"] is False + + allow_missing = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--allow-missing", + ], + repo, + ) + assert allow_missing.returncode == 0 + allow_json = json.loads(allow_missing.stdout) + assert allow_json["ok"] is True + assert allow_json["coverageOk"] is False + assert allow_json["scenarioCoverageOk"] is False + + allow_missing_readiness = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--allow-missing", + "--check-readiness", + ], + repo, + ) + assert allow_missing_readiness.returncode == 0 + allow_missing_readiness_json = json.loads(allow_missing_readiness.stdout) + assert allow_missing_readiness_json["ok"] is True + assert allow_missing_readiness_json["coverageOk"] is False + assert allow_missing_readiness_json["requiredOfficialReadinessChecked"] is True + assert allow_missing_readiness_json["requiredOfficialReadinessOk"] is False + assert allow_missing_readiness_json["notReadyOfficialScenarios"] == [] + + make_fixture(fixture_root, "candidate-action", source="ocu") + ignored_candidate = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--allow-missing", + ], + repo, + ) + assert ignored_candidate.returncode == 0 + ignored_json = json.loads(ignored_candidate.stdout) + assert ignored_json["availableOfficialScenarios"] == [] + + make_fixture(fixture_root, "official-action") + covered = run([sys.executable, str(script), "--fixture-root", str(fixture_root)], repo) + assert covered.returncode == 0, covered.stderr + covered_json = json.loads(covered.stdout) + assert covered_json["ok"] is True + assert covered_json["coverageOk"] is True + assert covered_json["scenarioCoverageOk"] is True + assert covered_json["hasRequiredOfficialSuccessfulFixture"] is True + assert covered_json["requiredOfficialReadinessChecked"] is False + assert covered_json["requiredOfficialReadinessOk"] is None + assert covered_json["hasRecommendedOfficialScenarioCoverage"] is False + assert covered_json["availableOfficialScenarios"] == ["simple-action-stop"] + assert covered_json["missingOfficialScenarios"] == [] + assert covered_json["missingRecommendedOfficialScenarios"] == [ + "cancel", + "drag-stop", + "keyboard-input-stop", + "timeout", + ] + assert covered_json["officialFixtures"][0]["path"].startswith("/tmp/") is False + + custom_recommended = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--recommended-scenario", + "simple-action-stop", + ], + repo, + ) + assert custom_recommended.returncode == 0, custom_recommended.stderr + custom_recommended_json = json.loads(custom_recommended.stdout) + assert custom_recommended_json["recommendedOfficialScenarios"] == ["simple-action-stop"] + assert custom_recommended_json["missingRecommendedOfficialScenarios"] == [] + assert custom_recommended_json["hasRecommendedOfficialScenarioCoverage"] is True + + covered_not_ready = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--check-readiness", + ], + repo, + ) + assert covered_not_ready.returncode == 1 + covered_not_ready_json = json.loads(covered_not_ready.stderr) + assert covered_not_ready_json["scenarioCoverageOk"] is True + assert covered_not_ready_json["coverageOk"] is False + assert covered_not_ready_json["hasRequiredOfficialSuccessfulFixture"] is False + assert covered_not_ready_json["requiredOfficialReadinessChecked"] is True + assert covered_not_ready_json["requiredOfficialReadinessOk"] is False + assert covered_not_ready_json["notReadyOfficialScenarios"] == ["simple-action-stop"] + assert any("failed readiness" in error for error in covered_not_ready_json["errors"]) + + ready_root = root / "ready" + make_ready_fixture(ready_root, "official-ready-action") + ready = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(ready_root), + "--require-readiness", + ], + repo, + ) + assert ready.returncode == 0, ready.stderr + ready_json = json.loads(ready.stdout) + assert ready_json["ok"] is True + assert ready_json["scenarioCoverageOk"] is True + assert ready_json["coverageOk"] is True + assert ready_json["hasRequiredOfficialSuccessfulFixture"] is True + assert ready_json["requiredOfficialReadinessChecked"] is True + assert ready_json["requiredOfficialReadinessOk"] is True + assert ready_json["notReadyOfficialScenarios"] == [] + assert ready_json["officialFixtureSetReadiness"]["ok"] is True + + duplicate_root = root / "duplicates" + make_fixture(duplicate_root, "official-action-a") + make_fixture(duplicate_root, "official-action-b") + duplicate = run( + [sys.executable, str(script), "--fixture-root", str(duplicate_root), "--allow-missing"], + repo, + ) + assert duplicate.returncode == 1 + duplicate_json = json.loads(duplicate.stderr) + assert any("duplicate official scenario" in error for error in duplicate_json["errors"]) + + missing_scenario_root = root / "missing-scenario" + fixture = make_fixture(missing_scenario_root, "official-action") + manifest = json.loads((fixture / "fixture-manifest.json").read_text()) + del manifest["scenario"] + write_json(fixture / "fixture-manifest.json", manifest) + missing_scenario = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(missing_scenario_root), + "--allow-missing", + ], + repo, + ) + assert missing_scenario.returncode == 1 + missing_scenario_json = json.loads(missing_scenario.stderr) + assert any("official fixture missing scenario" in error for error in missing_scenario_json["errors"]) + + recipe_drift_root = root / "recipe-drift" + recipe_drift_fixture = make_fixture(recipe_drift_root, "official-action") + recipe_drift_manifest_path = recipe_drift_fixture / "fixture-manifest.json" + recipe_drift_manifest = json.loads(recipe_drift_manifest_path.read_text()) + recipe_drift_manifest["scenarioRecipe"]["expectedActionEvents"] = ["keyboard.text_input"] + write_json(recipe_drift_manifest_path, recipe_drift_manifest) + recipe_drift = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(recipe_drift_root), + ], + repo, + ) + assert recipe_drift.returncode == 1 + recipe_drift_json = json.loads(recipe_drift.stderr) + assert recipe_drift_json["availableOfficialScenarios"] == ["simple-action-stop"] + assert recipe_drift_json["officialFixtures"][0]["scenarioRecipeMatches"] is False + assert any("scenarioRecipe does not match scenario" in error for error in recipe_drift_json["errors"]) + + print( + json.dumps( + { + "ok": True, + "checkedMissingCoverageFailure": True, + "checkedAllowMissingReport": True, + "checkedCandidateIgnored": True, + "checkedCoverageSuccess": True, + "checkedReadinessReport": True, + "checkedReadinessFailure": True, + "checkedReadinessSuccess": True, + "checkedRecommendedCoverageReport": True, + "checkedDuplicateScenarioFailure": True, + "checkedMissingScenarioFailure": True, + "checkedScenarioRecipeDriftFailure": True, + "checkedScenarioRecipeManifest": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-event-stream-official-fixture-set.py b/scripts/test-event-stream-official-fixture-set.py new file mode 100755 index 00000000..e717bf06 --- /dev/null +++ b/scripts/test-event-stream-official-fixture-set.py @@ -0,0 +1,531 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import scenario_recipe + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text("\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n") + + +def fixture_manifest(name, source, scenario, event_count): + return { + "fixtureFormatVersion": 1, + "name": name, + "scenario": scenario, + "scenarioRecipe": scenario_recipe(scenario), + "source": source, + "officialPluginVersion": "record-and-replay 1.0.857" if source == "official" else None, + "capturedAt": "2026-06-27", + "importedAt": "2026-06-27", + "eventCount": event_count, + "suppressedEventCount": 0, + "eventTypes": {}, + "files": { + "metadata": "metadata.json", + "session": "session.json", + "events": "events.jsonl", + "suppressed": "suppressed.jsonl", + "mcpTranscript": "mcp-transcript.json", + }, + "redaction": { + "screenshotsCopied": False, + "mcpTranscriptSanitized": True, + "preserveAppAttribution": False, + "textKeys": ["title"], + "timestampKeys": ["timestamp"], + "sessionIdKeys": ["sessionId"], + "pathKeysRewrittenRelative": ["eventsPath"], + "pathKeysRedacted": ["path", "*Path"], + }, + } + + +def mcp_transcript(): + return { + "startResponseShape": {"result": {"content": []}}, + "repeatStartResponseShape": {"result": {"content": []}}, + "statusResponseShape": {"result": {"content": []}}, + "stopResponseShape": {"result": {"content": []}}, + "repeatStopResponseShape": {"result": {"content": []}}, + "finalStatusResponseShape": {"result": {"content": []}}, + } + + +def action_event_for_scenario(session_id, scenario): + if scenario == "keyboard-input-stop": + return { + "type": "keyboard.text_input", + "sessionId": session_id, + "textLength": 4, + "focusedAccessibilityElement": { + "role": "AXTextField", + "title": "", + "actions": ["AXConfirm"], + }, + } + if scenario == "drag-stop": + return { + "type": "mouse.drag", + "sessionId": session_id, + "button": "left", + "startLocation": {"x": 10, "y": 20}, + "endLocation": {"x": 80, "y": 120}, + "distance": 122.07, + "durationMs": 400, + "targetAccessibilityElement": { + "role": "AXSlider", + "title": "", + "actions": ["AXPress"], + }, + } + return { + "type": "mouse.click", + "sessionId": session_id, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "", + "actions": ["AXPress"], + }, + } + + +def action_events(session_id, scenario="simple-action-stop", end_reason="recording_controls_stopped"): + if scenario == "timeout": + return [ + {"type": "session.started", "sessionId": session_id}, + {"type": "session.ended", "sessionId": session_id, "endReason": end_reason}, + ] + return [ + {"type": "session.started", "sessionId": session_id}, + { + "type": "AX.focusedWindowChanged", + "sessionId": session_id, + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "renderedText": "", + "treeLines": [""], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + }, + }, + action_event_for_scenario(session_id, scenario), + {"type": "session.ended", "sessionId": session_id, "endReason": end_reason}, + ] + + +def make_fixture(root, name, source="official", scenario="simple-action-stop", end_reason="recording_controls_stopped"): + fixture = root / name + fixture.mkdir(parents=True) + events = action_events(name, scenario, end_reason) + write_jsonl(fixture / "events.jsonl", events) + write_jsonl(fixture / "suppressed.jsonl", []) + metadata = { + "sessionId": name, + "state": "stopped", + "active": False, + "endReason": end_reason, + "eventCount": len(events), + "suppressedEventCount": 0, + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "suppressedEventsPath": "suppressed.jsonl", + } + write_json(fixture / "metadata.json", metadata) + write_json(fixture / "session.json", metadata) + write_json(fixture / "mcp-transcript.json", mcp_transcript()) + write_json(fixture / "fixture-manifest.json", fixture_manifest(name, source, scenario, len(events))) + return fixture + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + script = repo / "scripts/check-event-stream-official-fixture-set.py" + import_script = repo / "scripts/import-event-stream-fixture.py" + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + official_root = root / "official" + candidate_root = root / "candidate" + official = make_fixture(official_root, "official-action") + candidate = make_fixture(candidate_root, "candidate-action", source="ocu") + + success = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert success.returncode == 0, success.stderr + success_json = json.loads(success.stdout) + assert success_json["ok"] is True + assert success_json["requiredScenarios"] == ["simple-action-stop"] + assert success_json["availableOfficialScenarios"] == ["simple-action-stop"] + assert success_json["availableCandidateScenarios"] == ["simple-action-stop"] + assert success_json["comparePolicy"] == { + "requiresAxDiffEvidence": True, + "requiresSameAxDiffMarkers": True, + "requiresSuppressedEventSequence": True, + "requiresSuppressedSchema": True, + } + assert success_json["scenarioResults"][0]["officialReadiness"]["ok"] is True + assert success_json["scenarioResults"][0]["candidateCompare"]["ok"] is True + + missing_candidate = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(root / "empty-candidate"), + ], + repo, + ) + assert missing_candidate.returncode == 1 + missing_candidate_json = json.loads(missing_candidate.stderr) + assert "missing OCU candidate fixture scenario: simple-action-stop" in missing_candidate_json["errors"] + + ax_diff_official_root = root / "ax-diff-official" + ax_diff_candidate_root = root / "ax-diff-candidate" + ax_diff_official = make_fixture(ax_diff_official_root, "official-action") + make_fixture(ax_diff_candidate_root, "candidate-action", source="ocu") + ax_diff_events_path = ax_diff_official / "events.jsonl" + ax_diff_events = [ + json.loads(line) + for line in ax_diff_events_path.read_text().splitlines() + if line.strip() + ] + for event in ax_diff_events: + if event.get("type") == "AX.focusedWindowChanged": + event["accessibilityInspectorPayload"] = { + "kind": "diff", + "diffFromPrevious": True, + "renderedText": "~ 1 button Old -> 1 button New\n+ 2 button Done", + "treeLines": [ + "~ 1 button Old -> 1 button New", + "+ 2 button Done", + ], + "cumulativeDiffFromInitial": True, + "cumulativeRenderedText": "+ 2 button Done", + "cumulativeTreeLines": [ + "+ 2 button Done", + ], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + } + write_jsonl(ax_diff_events_path, ax_diff_events) + ax_diff_mismatch = run( + [ + sys.executable, + str(script), + "--official-root", + str(ax_diff_official_root), + "--candidate-root", + str(ax_diff_candidate_root), + ], + repo, + ) + assert ax_diff_mismatch.returncode == 1 + ax_diff_mismatch_json = json.loads(ax_diff_mismatch.stderr) + assert "OCU candidate scenario failed official comparison: simple-action-stop" in ( + ax_diff_mismatch_json["errors"] + ) + candidate_compare_errors = ( + ax_diff_mismatch_json["scenarioResults"][0]["candidateCompare"]["errors"] + ) + assert "candidate missing AX diff payload" in candidate_compare_errors + assert "candidate missing cumulative AX diff payload" in candidate_compare_errors + + bad_candidate_root = root / "bad-candidate" + make_fixture(bad_candidate_root, "candidate-action", source="ocu") + bad_candidate_metadata_path = bad_candidate_root / "candidate-action" / "metadata.json" + bad_candidate_metadata = json.loads(bad_candidate_metadata_path.read_text()) + bad_candidate_metadata["state"] = "recording" + bad_candidate_metadata["active"] = True + write_json(bad_candidate_metadata_path, bad_candidate_metadata) + bad_candidate = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(bad_candidate_root), + ], + repo, + ) + assert bad_candidate.returncode == 1 + bad_candidate_json = json.loads(bad_candidate.stderr) + assert "OCU candidate scenario failed readiness: simple-action-stop" in bad_candidate_json["errors"] + candidate_readiness_errors = ( + bad_candidate_json["scenarioResults"][0]["candidateReadiness"]["recordings"][0]["errors"] + ) + assert "metadata state is not final: recording" in candidate_readiness_errors + + bad_end_reason_root = root / "bad-end-reason" + make_fixture(bad_end_reason_root, "official-action", end_reason="recording_controls_cancelled") + bad_end_reason = run( + [ + sys.executable, + str(script), + "--official-root", + str(bad_end_reason_root), + ], + repo, + ) + assert bad_end_reason.returncode == 1 + bad_end_reason_json = json.loads(bad_end_reason.stderr) + assert "official fixture scenario failed readiness: simple-action-stop" in bad_end_reason_json["errors"] + readiness_errors = bad_end_reason_json["scenarioResults"][0]["officialReadiness"]["recordings"][0]["errors"] + assert "missing required endReason: recording_controls_stopped" in readiness_errors + + cancel_root = root / "cancel-official" + make_fixture( + cancel_root, + "official-cancel", + scenario="cancel", + end_reason="recording_controls_cancelled", + ) + cancel_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(cancel_root), + "--require-scenario", + "cancel", + ], + repo, + ) + assert cancel_result.returncode == 0, cancel_result.stderr + cancel_json = json.loads(cancel_result.stdout) + assert cancel_json["scenarioResults"][0]["officialReadiness"]["ok"] is True + + keyboard_root = root / "keyboard-official" + make_fixture(keyboard_root, "official-keyboard", scenario="keyboard-input-stop") + keyboard_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(keyboard_root), + "--require-scenario", + "keyboard-input-stop", + ], + repo, + ) + assert keyboard_result.returncode == 0, keyboard_result.stderr + keyboard_json = json.loads(keyboard_result.stdout) + assert keyboard_json["scenarioResults"][0]["officialReadiness"]["ok"] is True + + drag_root = root / "drag-official" + make_fixture(drag_root, "official-drag", scenario="drag-stop") + drag_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(drag_root), + "--require-scenario", + "drag-stop", + ], + repo, + ) + assert drag_result.returncode == 0, drag_result.stderr + drag_json = json.loads(drag_result.stdout) + assert drag_json["scenarioResults"][0]["officialReadiness"]["ok"] is True + + timeout_root = root / "timeout-official" + make_fixture( + timeout_root, + "official-timeout", + scenario="timeout", + end_reason="recording_time_limit_reached", + ) + timeout_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(timeout_root), + "--require-scenario", + "timeout", + ], + repo, + ) + assert timeout_result.returncode == 0, timeout_result.stderr + timeout_json = json.loads(timeout_result.stdout) + timeout_readiness = timeout_json["scenarioResults"][0]["officialReadiness"] + assert timeout_readiness["ok"] is True + timeout_recording = timeout_readiness["recordings"][0] + assert timeout_recording["actionEventCount"] == 0 + assert timeout_recording["hasAccessibilityPayload"] is False + assert timeout_recording["endReasons"] == {"recording_time_limit_reached": 2} + + keyboard_missing_event_root = root / "keyboard-missing-event" + make_fixture(keyboard_missing_event_root, "official-keyboard", scenario="keyboard-input-stop") + keyboard_events_path = keyboard_missing_event_root / "official-keyboard" / "events.jsonl" + keyboard_events = [ + json.loads(line) + for line in keyboard_events_path.read_text().splitlines() + if line.strip() + ] + keyboard_events = [ + {**event, "type": "mouse.click"} + if event.get("type") == "keyboard.text_input" + else event + for event in keyboard_events + ] + write_jsonl(keyboard_events_path, keyboard_events) + keyboard_missing_event = run( + [ + sys.executable, + str(script), + "--official-root", + str(keyboard_missing_event_root), + "--require-scenario", + "keyboard-input-stop", + ], + repo, + ) + assert keyboard_missing_event.returncode == 1 + keyboard_missing_event_json = json.loads(keyboard_missing_event.stderr) + assert "official fixture scenario failed readiness: keyboard-input-stop" in keyboard_missing_event_json["errors"] + keyboard_readiness_errors = ( + keyboard_missing_event_json["scenarioResults"][0]["officialReadiness"]["recordings"][0]["errors"] + ) + assert "missing required event type: keyboard.text_input" in keyboard_readiness_errors + + no_scenario_root = root / "no-scenario" + no_scenario = make_fixture(no_scenario_root, "official-action") + manifest = json.loads((no_scenario / "fixture-manifest.json").read_text()) + del manifest["scenario"] + write_json(no_scenario / "fixture-manifest.json", manifest) + no_scenario_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(no_scenario_root), + ], + repo, + ) + assert no_scenario_result.returncode == 1 + no_scenario_json = json.loads(no_scenario_result.stderr) + assert any("fixture-manifest.json missing scenario" in error for error in no_scenario_json["errors"]) + + recipe_drift_root = root / "recipe-drift" + recipe_drift = make_fixture(recipe_drift_root, "official-action") + recipe_manifest_path = recipe_drift / "fixture-manifest.json" + recipe_manifest = json.loads(recipe_manifest_path.read_text()) + recipe_manifest["scenarioRecipe"]["expectedActionEvents"] = ["keyboard.text_input"] + write_json(recipe_manifest_path, recipe_manifest) + recipe_drift_result = run( + [ + sys.executable, + str(script), + "--official-root", + str(recipe_drift_root), + ], + repo, + ) + assert recipe_drift_result.returncode == 1 + recipe_drift_json = json.loads(recipe_drift_result.stderr) + assert any("scenarioRecipe does not match scenario" in error for error in recipe_drift_json["errors"]) + + imported_root = root / "imported" + import_result = run( + [ + sys.executable, + str(import_script), + str(official), + "--name", + "imported-action", + "--output-dir", + str(imported_root), + "--scenario", + "simple-action-stop", + "--mcp-transcript", + str(official / "mcp-transcript.json"), + ], + repo, + ) + assert import_result.returncode == 0, import_result.stderr + imported_manifest = json.loads((imported_root / "imported-action" / "fixture-manifest.json").read_text()) + assert imported_manifest["scenario"] == "simple-action-stop" + + shutil.rmtree(candidate) + make_fixture(candidate_root, "candidate-action", source="ocu", scenario="different-scenario") + mismatched_candidate = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert mismatched_candidate.returncode == 1 + mismatched_candidate_json = json.loads(mismatched_candidate.stderr) + assert "missing OCU candidate fixture scenario: simple-action-stop" in mismatched_candidate_json["errors"] + + print( + json.dumps( + { + "ok": True, + "checkedFixtureSetGate": True, + "checkedRequiredSimpleActionStopScenario": True, + "checkedCandidatePairing": True, + "checkedCandidateReadinessFailure": True, + "checkedMissingCandidateFailure": True, + "checkedAxDiffComparisonPolicy": True, + "checkedSuppressedStreamComparisonPolicy": True, + "checkedAxDiffComparisonFailure": True, + "checkedStopEndReasonPolicy": True, + "checkedCancelScenarioPolicy": True, + "checkedKeyboardInputScenarioPolicy": True, + "checkedDragScenarioPolicy": True, + "checkedTimeoutScenarioPolicy": True, + "checkedMissingScenarioFailure": True, + "checkedScenarioRecipeDriftFailure": True, + "checkedImportScenarioManifest": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-event-stream-probe-fixtures.py b/scripts/test-event-stream-probe-fixtures.py new file mode 100755 index 00000000..feb84d81 --- /dev/null +++ b/scripts/test-event-stream-probe-fixtures.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import re +import sys + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +FIXTURE = ( + REPO_ROOT + / "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json" +) +NO_ACTIVE_FIXTURE = ( + REPO_ROOT + / "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-no-active-status-stop-1.0.857.json" +) +ABSOLUTE_PATH_PATTERN = re.compile(r"(/Users/|/var/folders/|/private/var/|/tmp/)") +EXPECTED_NO_ACTIVE_RESPONSE = {"isRecording": False, "maxDurationSeconds": 1800} +RECORDING_PATH_KEYS = { + "currentSegmentEventsPath", + "currentSegmentMetadataPath", + "eventsPath", + "metadataPath", + "sessionPath", + "suppressedEventsPath", +} + + +def contains_recording_path(value) -> bool: + if isinstance(value, dict): + if any(key in RECORDING_PATH_KEYS for key in value): + return True + return any(contains_recording_path(item) for item in value.values()) + if isinstance(value, list): + return any(contains_recording_path(item) for item in value) + return False + + +def main() -> int: + text = FIXTURE.read_text(encoding="utf-8") + if ABSOLUTE_PATH_PATTERN.search(text): + raise AssertionError(f"probe fixture contains a local absolute path: {FIXTURE}") + data = json.loads(text) + assert data["fixtureFormatVersion"] == 1 + assert data["kind"] == "record-and-replay-event-stream-probe" + assert data["source"] == "official" + assert data["officialPluginVersion"] == "1.0.857" + assert data["ok"] is True + assert data["startStopRequested"] is True + assert data["startStopCompleted"] is False + assert data["startTimedOut"] is True + assert data["startResponseShape"] == {"timeout": 10.0} + assert data["statusResponseShape"] == {"timeout": 10.0} + assert data["stopResponseShape"] == {"timeout": 10.0} + assert data["protocolVersion"] == "2025-11-25" + assert data["serverInfo"]["name"] == "Record & Replay" + assert data["toolNames"] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ] + assert data.get("elicitationRequests") == [] + transcript = data["transcriptShape"] + has_tool_call = any( + item.get("direction") == "send" + and item.get("method") == "tools/call" + for item in transcript + ) + assert has_tool_call + tool_call_count = sum( + 1 + for item in transcript + if item.get("direction") == "send" and item.get("method") == "tools/call" + ) + timeout_count = sum(1 for item in transcript if item.get("timeout") is not None) + assert tool_call_count == 3 + assert timeout_count == 3 + assert not contains_recording_path(data) + + no_active_text = NO_ACTIVE_FIXTURE.read_text(encoding="utf-8") + if ABSOLUTE_PATH_PATTERN.search(no_active_text): + raise AssertionError( + f"no-active fixture contains a local absolute path: {NO_ACTIVE_FIXTURE}" + ) + no_active = json.loads(no_active_text) + assert no_active["officialPluginVersion"] == "record-and-replay 1.0.857" + assert "no active recording" in no_active["captureScope"] + responses = no_active["toolResponses"] + for tool_name in ("event_stream_status", "event_stream_stop"): + content = responses[tool_name]["content"] + assert content == [{"type": "text", "textJSON": EXPECTED_NO_ACTIVE_RESPONSE}] + + print( + json.dumps( + { + "ok": True, + "checkedOfficialRawStartTimeoutBoundary": True, + "checkedOfficialRawStartStatusStopTimeout": True, + "checkedOfficialRawStartDoesNotReturnRecordingPaths": True, + "checkedOfficialRawSurface": True, + "checkedRawProbeFixtureRedaction": True, + "checkedNoActiveFixtureRedaction": True, + "officialRawToolCallCount": tool_call_count, + "officialRawTimeoutCount": timeout_count, + "fixtures": [ + str(FIXTURE.relative_to(REPO_ROOT)), + str(NO_ACTIVE_FIXTURE.relative_to(REPO_ROOT)), + ], + } + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-event-stream-recording-compare.py b/scripts/test-event-stream-recording-compare.py new file mode 100755 index 00000000..056dfc55 --- /dev/null +++ b/scripts/test-event-stream-recording-compare.py @@ -0,0 +1,726 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text( + "\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n" + ) + + +def make_recording( + root, + name, + events, + metadata_overrides=None, + remove_metadata_keys=None, + mcp_transcript=None, + suppressed_events=None, +): + recording = root / name + recording.mkdir() + events_path = recording / "events.jsonl" + metadata_path = recording / "metadata.json" + suppressed_path = recording / "suppressed.jsonl" + write_jsonl(events_path, events) + suppressed_events = suppressed_events or [] + write_jsonl(suppressed_path, suppressed_events) + metadata = { + "sessionId": name, + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": len(suppressed_events), + "eventsPath": "events.jsonl", + "metadataPath": "metadata.json", + "sessionPath": "session.json", + "suppressedEventsPath": "suppressed.jsonl", + } + if metadata_overrides: + metadata.update(metadata_overrides) + for key in remove_metadata_keys or []: + metadata.pop(key, None) + write_json(metadata_path, metadata) + write_json(recording / "session.json", metadata) + if mcp_transcript is not None: + write_json(recording / "mcp-transcript.json", mcp_transcript) + return recording + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + baseline_events = [ + {"type": "session.started", "sessionId": "baseline"}, + { + "type": "AX.focusedWindowChanged", + "sessionId": "baseline", + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "fullTree": [ + "0 window Demo", + "1 search text field Alpha", + ], + "treeLines": [ + "0 window Demo", + "1 search text field Alpha", + ], + }, + }, + { + "type": "AX.focusedWindowChanged", + "sessionId": "baseline", + "accessibilityInspectorPayload": { + "kind": "diff", + "diffFromPrevious": True, + "treeLines": [ + "~ 1 search text field Alpha -> 1 search text field Beta", + "+ 2 button Done", + ], + "cumulativeDiffFromInitial": True, + "cumulativeTreeLines": [ + "+ 2 button Done", + ], + }, + }, + { + "type": "mouse.click", + "sessionId": "baseline", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Save", + "actions": ["AXPress"], + }, + }, + {"type": "session.ended", "sessionId": "baseline", "endReason": "recording_controls_stopped"}, + ] + baseline_suppressed_events = [ + { + "type": "AX.snapshot.suppressed", + "reason": "snapshotTooLarge", + "subsystem": "accessibility", + "lineCount": 250, + } + ] + mcp_transcript = { + "startResponseShape": {"result": {"content": [{"type": "text", "textJSON": {"state": "recording"}}]}}, + "repeatStartResponseShape": { + "result": {"content": [{"type": "text", "textJSON": {"state": "recording"}}]} + }, + "statusResponseShape": {"result": {"content": [{"type": "text", "textJSON": {"state": "recording"}}]}}, + "stopResponseShape": {"result": {"content": [{"type": "text", "textJSON": {"state": "stopped"}}]}}, + "repeatStopResponseShape": {"result": {"content": [{"type": "text", "textJSON": {"state": "stopped"}}]}}, + "finalStatusResponseShape": {"result": {"content": [{"type": "text", "textJSON": {"state": "stopped"}}]}}, + } + baseline = make_recording( + root, + "baseline", + baseline_events, + mcp_transcript=mcp_transcript, + suppressed_events=baseline_suppressed_events, + ) + matching = make_recording( + root, + "matching", + baseline_events, + mcp_transcript=mcp_transcript, + suppressed_events=baseline_suppressed_events, + ) + missing_mcp_shape = make_recording( + root, + "missing-mcp-shape", + baseline_events, + mcp_transcript={ + key: value + for key, value in mcp_transcript.items() + if key != "repeatStartResponseShape" + }, + suppressed_events=baseline_suppressed_events, + ) + timeout_mcp_shape = make_recording( + root, + "timeout-mcp-shape", + baseline_events, + mcp_transcript={ + **mcp_transcript, + "repeatStartResponseShape": {"timeout": 5.0}, + }, + suppressed_events=baseline_suppressed_events, + ) + schema_mcp_shape = make_recording( + root, + "schema-mcp-shape", + baseline_events, + mcp_transcript={ + **mcp_transcript, + "repeatStartResponseShape": {"result": {"content": [{"type": "text"}]}}, + }, + suppressed_events=baseline_suppressed_events, + ) + metadata_drift = make_recording( + root, + "metadata-drift", + baseline_events, + remove_metadata_keys=["suppressedEventCount"], + suppressed_events=baseline_suppressed_events, + ) + metadata_value_drift = make_recording( + root, + "metadata-value-drift", + baseline_events, + metadata_overrides={"endReason": "recording_controls_cancelled"}, + suppressed_events=baseline_suppressed_events, + ) + missing_target = make_recording(root, "missing-target", [ + baseline_events[0], + baseline_events[1], + baseline_events[2], + { + "type": "mouse.click", + "sessionId": "candidate", + "location": {"x": 10, "y": 20}, + }, + baseline_events[4], + ], suppressed_events=baseline_suppressed_events) + reordered = make_recording(root, "reordered", [ + baseline_events[0], + baseline_events[1], + baseline_events[3], + baseline_events[2], + ], suppressed_events=baseline_suppressed_events) + start_not_first = make_recording(root, "start-not-first", [ + baseline_events[1], + baseline_events[0], + baseline_events[2], + baseline_events[3], + ], suppressed_events=baseline_suppressed_events) + missing_ax_diff = make_recording(root, "missing-ax-diff", [ + baseline_events[0], + { + "type": "AX.focusedWindowChanged", + "sessionId": "candidate", + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "treeLines": [ + "0 standard window Example", + "1 search text field Beta", + ], + }, + }, + baseline_events[3], + baseline_events[4], + ], suppressed_events=baseline_suppressed_events) + marker_mismatch = make_recording(root, "marker-mismatch", [ + baseline_events[0], + baseline_events[1], + { + "type": "AX.focusedWindowChanged", + "sessionId": "candidate", + "accessibilityInspectorPayload": { + "kind": "diff", + "diffFromPrevious": True, + "treeLines": [ + "+ 2 button Done", + ], + "cumulativeDiffFromInitial": True, + "cumulativeTreeLines": [ + "+ 2 button Done", + ], + }, + }, + baseline_events[3], + baseline_events[4], + ], suppressed_events=baseline_suppressed_events) + missing_suppressed = make_recording(root, "missing-suppressed", baseline_events) + suppressed_schema_drift = make_recording( + root, + "suppressed-schema-drift", + baseline_events, + suppressed_events=[ + { + "type": "AX.snapshot.suppressed", + "reason": "snapshotTooLarge", + } + ], + ) + + pass_result = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(matching), + "--require-same-event-sequence", + "--require-same-schema", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + pass_json = json.loads(pass_result.stdout) + assert pass_json["ok"] is True + assert pass_json["eventSequenceEqual"] is True + assert pass_json["schemaEqual"] is True + assert pass_json["metadataKeysEqual"] is True + assert pass_json["metadataKeyDiff"] == {"extraKeys": [], "missingKeys": []} + assert pass_json["metadataStableValuesEqual"] is True + assert pass_json["metadataValueDiff"]["changedValues"] == [] + assert pass_json["metadataValueDiff"]["missingKeys"] == [] + assert pass_json["handoffPathEvidence"]["baseline"]["metadataPath"]["exists"] is True + assert pass_json["handoffPathEvidence"]["baseline"]["sessionPath"]["exists"] is True + assert pass_json["handoffPathEvidence"]["baseline"]["eventsPath"]["exists"] is True + assert pass_json["handoffPathEvidence"]["baseline"]["suppressedEventsPath"]["exists"] is True + assert pass_json["suppressedEventSequenceEqual"] is True + assert pass_json["suppressedSchemaEqual"] is True + assert pass_json["baseline"]["suppressedEventCount"] == 1 + assert pass_json["finalSessionEvidence"]["baseline"]["sessionStartedCount"] == 1 + assert pass_json["finalSessionEvidence"]["baseline"]["hasInitialSessionStarted"] is True + assert pass_json["finalSessionEvidence"]["baseline"]["sessionEndedCount"] == 1 + assert pass_json["finalSessionEvidence"]["baseline"]["hasFinalSessionEnded"] is True + assert pass_json["finalSessionEvidence"]["baseline"]["endReasons"] == ["recording_controls_stopped"] + assert pass_json["mcpResponseShapeEvidence"]["baseline"]["repeatStartResponseShape"]["hasResult"] is True + assert pass_json["mcpResponseShapeEvidence"]["candidate"]["repeatStartResponseShape"]["hasResult"] is True + assert pass_json["axDiffEvidence"]["baseline"]["diffMarkers"] == ["+", "~"] + assert pass_json["axDiffEvidence"]["baseline"]["hasFullTree"] is True + assert pass_json["axDiffEvidence"]["baseline"]["cumulativeDiffMarkers"] == ["+"] + assert "targetAccessibilityElement.role" in pass_json["semanticFieldEvidence"]["baseline"]["mouse.click"]["presentPaths"] + assert "accessibilityInspectorPayload.fullTree[]" in pass_json["semanticFieldEvidence"]["baseline"]["AX.focusedWindowChanged"]["presentPaths"] + assert "accessibilityInspectorPayload.treeLines[]" in pass_json["semanticFieldEvidence"]["baseline"]["AX.focusedWindowChanged"]["presentPaths"] + + official_ax_events = [ + {"kind": "session.started", "sessionId": "official-ax"}, + { + "kind": "window.changed", + "sessionId": "official-ax", + "ax": { + "mode": "fullTree", + "textRedacted": True, + "textLength": 42, + }, + }, + { + "kind": "mouse.click", + "sessionId": "official-ax", + "ax": { + "mode": "fullTree", + "textRedacted": True, + "textLength": 42, + }, + }, + { + "kind": "session.ended", + "sessionId": "official-ax", + "endReason": "recording_controls_stopped", + }, + ] + official_ax_baseline = make_recording(root, "official-ax-baseline", official_ax_events) + official_ax_candidate = make_recording(root, "official-ax-candidate", official_ax_events) + official_ax_result = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(official_ax_baseline), + str(official_ax_candidate), + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + official_ax_json = json.loads(official_ax_result.stdout) + assert official_ax_json["axDiffEvidence"]["baseline"]["payloadCount"] == 2 + assert official_ax_json["axDiffEvidence"]["baseline"]["hasFullPayload"] is True + assert official_ax_json["axDiffEvidence"]["baseline"]["hasFullTree"] is True + + suppressed_sequence_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_suppressed), + "--require-same-suppressed-event-sequence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert suppressed_sequence_failure.returncode == 1 + suppressed_sequence_report = json.loads(suppressed_sequence_failure.stderr) + assert "suppressed event type sequence differs" in suppressed_sequence_report["errors"] + assert suppressed_sequence_report["firstSuppressedEventSequenceDifference"] == { + "index": 0, + "baseline": "AX.snapshot.suppressed", + "candidate": None, + } + + suppressed_schema_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(suppressed_schema_drift), + "--require-same-suppressed-schema", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert suppressed_schema_failure.returncode == 1 + suppressed_schema_report = json.loads(suppressed_schema_failure.stderr) + assert "suppressed event schema differs" in suppressed_schema_report["errors"] + assert "AX.snapshot.suppressed" in suppressed_schema_report["suppressedSchemaDiff"] + assert "subsystem" in suppressed_schema_report["suppressedSchemaDiff"]["AX.snapshot.suppressed"]["missingPaths"] + + handoff_path_pass = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(matching), + "--require-handoff-paths", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert json.loads(handoff_path_pass.stdout)["ok"] is True + + missing_handoff_path = make_recording( + root, + "missing-handoff-path", + baseline_events, + remove_metadata_keys=["sessionPath"], + suppressed_events=baseline_suppressed_events, + ) + missing_handoff_path_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_handoff_path), + "--require-handoff-paths", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_handoff_path_failure.returncode == 1 + missing_handoff_path_report = json.loads(missing_handoff_path_failure.stderr) + assert "candidate missing declared handoff path: sessionPath" in missing_handoff_path_report["errors"] + assert missing_handoff_path_report["handoffPathEvidence"]["candidate"]["sessionPath"]["exists"] is None + + bad_handoff_path = make_recording( + root, + "bad-handoff-path", + baseline_events, + metadata_overrides={"suppressedEventsPath": "missing-suppressed.jsonl"}, + suppressed_events=baseline_suppressed_events, + ) + bad_handoff_path_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(bad_handoff_path), + "--require-handoff-paths", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert bad_handoff_path_failure.returncode == 1 + bad_handoff_path_report = json.loads(bad_handoff_path_failure.stderr) + assert "candidate declared handoff path does not exist: suppressedEventsPath" in bad_handoff_path_report["errors"] + assert bad_handoff_path_report["handoffPathEvidence"]["candidate"]["suppressedEventsPath"]["exists"] is False + + mcp_shape_pass = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(matching), + "--require-mcp-response-shapes", + "--require-same-mcp-response-schema", + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert json.loads(mcp_shape_pass.stdout)["ok"] is True + + missing_mcp_shape_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_mcp_shape), + "--require-mcp-response-shapes", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing_mcp_shape_failure.returncode == 1 + missing_mcp_shape_report = json.loads(missing_mcp_shape_failure.stderr) + assert "candidate missing MCP response shape: repeatStartResponseShape" in missing_mcp_shape_report["errors"] + + timeout_mcp_shape_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(timeout_mcp_shape), + "--require-mcp-response-shapes", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert timeout_mcp_shape_failure.returncode == 1 + timeout_mcp_shape_report = json.loads(timeout_mcp_shape_failure.stderr) + assert any( + error.startswith("candidate MCP response shape repeatStartResponseShape hasResult differs") + for error in timeout_mcp_shape_report["errors"] + ) + assert any( + error.startswith("candidate MCP response shape repeatStartResponseShape timedOut differs") + for error in timeout_mcp_shape_report["errors"] + ) + + schema_mcp_shape_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(schema_mcp_shape), + "--require-mcp-response-shapes", + "--require-same-mcp-response-schema", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert schema_mcp_shape_failure.returncode == 1 + schema_mcp_shape_report = json.loads(schema_mcp_shape_failure.stderr) + assert any( + error.startswith("candidate MCP response shape repeatStartResponseShape missing schema paths:") + for error in schema_mcp_shape_report["errors"] + ) + + schema_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_target), + "--require-same-schema", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert schema_failure.returncode == 1 + schema_report = json.loads(schema_failure.stderr) + assert schema_report["ok"] is False + assert "event schema differs" in schema_report["errors"] + missing_paths = schema_report["schemaDiff"]["mouse.click"]["missingPaths"] + assert "targetAccessibilityElement" in missing_paths + + metadata_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(metadata_drift), + "--require-same-metadata-keys", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert metadata_failure.returncode == 1 + metadata_report = json.loads(metadata_failure.stderr) + assert metadata_report["ok"] is False + assert "metadata keys differ" in metadata_report["errors"] + assert metadata_report["metadataKeyDiff"]["missingKeys"] == ["suppressedEventCount"] + assert metadata_report["metadataKeyDiff"]["extraKeys"] == [] + + metadata_value_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(metadata_value_drift), + "--require-same-metadata-values", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert metadata_value_failure.returncode == 1 + metadata_value_report = json.loads(metadata_value_failure.stderr) + assert metadata_value_report["ok"] is False + assert "stable metadata values differ" in metadata_value_report["errors"] + assert metadata_value_report["metadataValueDiff"]["changedValues"] == [ + { + "key": "endReason", + "baseline": "recording_controls_stopped", + "candidate": "recording_controls_cancelled", + } + ] + assert metadata_value_report["metadataValueDiff"]["missingKeys"] == [] + + semantic_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_target), + "--require-semantic-fields", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert semantic_failure.returncode == 1 + semantic_report = json.loads(semantic_failure.stderr) + assert semantic_report["ok"] is False + assert any( + error.startswith("candidate missing semantic fields for mouse.click:") + for error in semantic_report["errors"] + ) + assert "targetAccessibilityElement.role" in semantic_report["errors"][0] + + sequence_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(reordered), + "--require-same-event-sequence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert sequence_failure.returncode == 1 + sequence_report = json.loads(sequence_failure.stderr) + assert sequence_report["ok"] is False + assert "event type sequence differs" in sequence_report["errors"] + assert sequence_report["firstEventSequenceDifference"]["index"] == 2 + + final_session_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(reordered), + "--require-final-session-evidence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert final_session_failure.returncode == 1 + final_session_report = json.loads(final_session_failure.stderr) + assert final_session_report["ok"] is False + assert "candidate final event is not session.ended" in final_session_report["errors"] + assert final_session_report["finalSessionEvidence"]["candidate"]["hasFinalSessionEnded"] is False + assert final_session_report["finalSessionEvidence"]["candidate"]["lastEventType"] == "AX.focusedWindowChanged" + + start_session_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(start_not_first), + "--require-final-session-evidence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert start_session_failure.returncode == 1 + start_session_report = json.loads(start_session_failure.stderr) + assert start_session_report["ok"] is False + assert "candidate first event is not session.started" in start_session_report["errors"] + assert start_session_report["finalSessionEvidence"]["candidate"]["hasInitialSessionStarted"] is False + assert start_session_report["finalSessionEvidence"]["candidate"]["firstEventType"] == "AX.focusedWindowChanged" + + ax_missing_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(missing_ax_diff), + "--require-ax-diff-evidence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert ax_missing_failure.returncode == 1 + ax_missing_report = json.loads(ax_missing_failure.stderr) + assert "candidate missing AX diff payload" in ax_missing_report["errors"] + assert "candidate missing cumulative AX diff payload" in ax_missing_report["errors"] + + marker_failure = subprocess.run( + [ + sys.executable, + str(repo / "scripts/compare-event-stream-recordings.py"), + str(baseline), + str(marker_mismatch), + "--require-ax-diff-evidence", + "--require-same-ax-diff-markers", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert marker_failure.returncode == 1 + marker_report = json.loads(marker_failure.stderr) + assert "candidate missing AX diff markers: ~" in marker_report["errors"] + + print(json.dumps({"ok": True}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-event-stream-recording-probe.py b/scripts/test-event-stream-recording-probe.py new file mode 100755 index 00000000..142604c2 --- /dev/null +++ b/scripts/test-event-stream-recording-probe.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile +import textwrap + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +PROBE = REPO_ROOT / "scripts/probe-event-stream-recording.py" + + +FAKE_SERVER = r''' +#!/usr/bin/env python3 +import json +import os +import pathlib +import sys + +record_file = pathlib.Path(os.environ["FAKE_ELICITATION_RESPONSE_FILE"]) +active_started = False +completed = False + +def send(message): + sys.stdout.write(json.dumps(message, separators=(",", ":")) + "\n") + sys.stdout.flush() + +pending_start_id = None +for line in sys.stdin: + message = json.loads(line) + method = message.get("method") + if method == "initialize": + send({ + "jsonrpc": "2.0", + "id": message["id"], + "result": { + "protocolVersion": "2025-11-25", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "Record & Replay", "version": "fake"}, + }, + }) + elif method == "tools/list": + send({ + "jsonrpc": "2.0", + "id": message["id"], + "result": { + "tools": [ + {"name": "event_stream_start", "inputSchema": {"type": "object", "properties": {}}}, + {"name": "event_stream_status", "inputSchema": {"type": "object", "properties": {}}}, + {"name": "event_stream_stop", "inputSchema": {"type": "object", "properties": {}}}, + ] + }, + }) + elif method == "tools/call": + params = message.get("params", {}) + name = params.get("name") + if name == "event_stream_start": + if active_started: + send({ + "jsonrpc": "2.0", + "id": message["id"], + "result": { + "content": [{"type": "text", "text": json.dumps({ + "state": "recording", + "active": True, + "sessionId": "fake-session", + "eventsPath": "/tmp/fake-session/events.jsonl", + "metadataPath": "/tmp/fake-session/metadata.json", + "currentSegmentEventsPath": "/tmp/fake-session/events.jsonl", + "currentSegmentMetadataPath": "/tmp/fake-session/metadata.json", + })}], + "isError": False, + }, + }) + continue + pending_start_id = message["id"] + send({ + "jsonrpc": "2.0", + "id": "fake-approval-1", + "method": "elicitation/create", + "params": { + "mode": "form", + "message": "Fake approval request", + "requestedSchema": { + "type": "object", + "properties": {"note": {"type": "string"}}, + "required": [], + "additionalProperties": False, + }, + }, + }) + elif name == "event_stream_status": + state_payload = { + "state": "stopped" if completed else "recording", + "active": False if completed else True, + "sessionId": "fake-session", + "eventsPath": "/tmp/fake-session/events.jsonl", + "metadataPath": "/tmp/fake-session/metadata.json", + } + if completed: + state_payload["endReason"] = "recording_controls_stopped" + else: + state_payload["currentSegmentEventsPath"] = "/tmp/fake-session/events.jsonl" + state_payload["currentSegmentMetadataPath"] = "/tmp/fake-session/metadata.json" + send({ + "jsonrpc": "2.0", + "id": message["id"], + "result": { + "content": [{"type": "text", "text": json.dumps(state_payload)}], + "isError": False, + }, + }) + elif name == "event_stream_stop": + completed = True + send({ + "jsonrpc": "2.0", + "id": message["id"], + "result": { + "content": [{"type": "text", "text": json.dumps({ + "state": "stopped", + "active": False, + "sessionId": "fake-session", + "eventsPath": "/tmp/fake-session/events.jsonl", + "metadataPath": "/tmp/fake-session/metadata.json", + "endReason": "recording_controls_stopped", + })}], + "isError": False, + }, + }) + elif message.get("id") == "fake-approval-1" and "result" in message: + active_started = True + record_file.write_text(json.dumps(message["result"], sort_keys=True) + "\n", encoding="utf-8") + send({ + "jsonrpc": "2.0", + "id": pending_start_id, + "result": { + "content": [{"type": "text", "text": json.dumps({"state": "recording", "active": True, "sessionId": "fake-session"})}], + "isError": False, + }, + }) +''' + + +def write_fake_server(directory: pathlib.Path) -> pathlib.Path: + path = directory / "fake-event-stream-server.py" + path.write_text(textwrap.dedent(FAKE_SERVER).lstrip(), encoding="utf-8") + path.chmod(0o755) + return path + + +def run_probe( + fake_server: pathlib.Path, + tmpdir: pathlib.Path, + action: str = "accept", + legacy_decline: bool = False, +) -> tuple[dict, dict, dict]: + label = "legacy-decline" if legacy_decline else action + response_file = tmpdir / f"{label}-response.json" + output_file = tmpdir / f"{label}-output.json" + fixture_file = tmpdir / f"{label}-fixture.json" + command = [ + sys.executable, + str(PROBE), + "--target", + "local", + "--client", + str(fake_server), + "--start-stop", + "--timeout", + "2", + "--output", + str(output_file), + "--fixture-output", + str(fixture_file), + ] + if legacy_decline: + command.append("--decline-elicitation") + else: + command.extend(["--elicitation-action", action]) + env = dict(**os_environ(), FAKE_ELICITATION_RESPONSE_FILE=str(response_file)) + subprocess.run(command, cwd=REPO_ROOT, env=env, check=True, stdout=subprocess.PIPE, text=True) + return ( + json.loads(output_file.read_text(encoding="utf-8")), + json.loads(fixture_file.read_text(encoding="utf-8")), + json.loads(response_file.read_text(encoding="utf-8")), + ) + + +def os_environ() -> dict[str, str]: + import os + + return os.environ.copy() + + +def assert_probe_result(output: dict, fixture: dict) -> None: + assert output["ok"] is True + assert output["startStopCompleted"] is True + assert output["startTimedOut"] is False + assert fixture["kind"] == "record-and-replay-event-stream-probe" + assert fixture["source"] == "local" + assert fixture["toolNames"] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ] + assert fixture["elicitationRequests"] == [ + { + "message": "Fake approval request", + "mode": "form", + "requestedSchemaHasAdditionalProperties": True, + "requestedSchemaPropertyNames": ["note"], + "requestedSchemaRequired": [], + "requestedSchemaType": "object", + } + ] + fixture_text = json.dumps(fixture, sort_keys=True) + assert "/tmp/fake-session" not in fixture_text + status_text = fixture["statusResponseShape"]["result"]["content"][0] + assert status_text["type"] == "text" + assert "text" not in status_text + assert status_text["textJSON"]["eventsPath"] == "" + assert status_text["textJSON"]["metadataPath"] == "" + assert ( + status_text["textJSON"]["currentSegmentEventsPath"] + == "" + ) + repeat_start_text = fixture["repeatStartResponseShape"]["result"]["content"][0] + assert repeat_start_text["type"] == "text" + assert "text" not in repeat_start_text + assert repeat_start_text["textJSON"]["sessionId"] == "" + assert ( + repeat_start_text["textJSON"]["currentSegmentMetadataPath"] + == "" + ) + stop_text = fixture["stopResponseShape"]["result"]["content"][0] + assert "text" not in stop_text + assert "currentSegmentEventsPath" not in stop_text["textJSON"] + assert stop_text["textJSON"]["endReason"] == "recording_controls_stopped" + repeat_stop_text = fixture["repeatStopResponseShape"]["result"]["content"][0] + assert "text" not in repeat_stop_text + assert repeat_stop_text["textJSON"]["state"] == "stopped" + assert repeat_stop_text["textJSON"]["sessionId"] == "" + final_status_text = fixture["finalStatusResponseShape"]["result"]["content"][0] + assert "text" not in final_status_text + assert final_status_text["textJSON"]["state"] == "stopped" + assert "currentSegmentEventsPath" not in final_status_text["textJSON"] + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="event-stream-probe-test-") as tmp: + tmpdir = pathlib.Path(tmp) + fake_server = write_fake_server(tmpdir) + + accept_output, accept_fixture, accept_response = run_probe(fake_server, tmpdir) + assert_probe_result(accept_output, accept_fixture) + assert accept_response == {"action": "accept", "content": {}} + + decline_output, decline_fixture, decline_response = run_probe(fake_server, tmpdir, "decline") + assert_probe_result(decline_output, decline_fixture) + assert decline_response == {"action": "decline", "content": None} + + cancel_output, cancel_fixture, cancel_response = run_probe(fake_server, tmpdir, "cancel") + assert_probe_result(cancel_output, cancel_fixture) + assert cancel_response == {"action": "cancel", "content": None} + + legacy_decline_output, legacy_decline_fixture, legacy_decline_response = run_probe( + fake_server, + tmpdir, + legacy_decline=True, + ) + assert_probe_result(legacy_decline_output, legacy_decline_fixture) + assert legacy_decline_response == {"action": "decline", "content": None} + + print(json.dumps({"ok": True, "probe": "event-stream-recording"})) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-event-stream-skill-scaffold.py b/scripts/test-event-stream-skill-scaffold.py new file mode 100755 index 00000000..2f502103 --- /dev/null +++ b/scripts/test-event-stream-skill-scaffold.py @@ -0,0 +1,1212 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile + + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +ABSOLUTE_PATH_PATTERN = re.compile(r"(/Users/|/var/folders/|/private/var/|/tmp/)") + + +def write_json(path: pathlib.Path, value: object) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def assert_generated_skill( + output_dir: pathlib.Path, + expected_quote: str, + expect_metadata_path: bool = True, +) -> None: + skill_text = (output_dir / "SKILL.md").read_text(encoding="utf-8") + summary_text = (output_dir / "references/recording-summary.json").read_text(encoding="utf-8") + if ABSOLUTE_PATH_PATTERN.search(skill_text + summary_text): + raise AssertionError("generated skill scaffold contains local absolute paths") + assert "name: recorded-example-workflow" in skill_text + assert f"Click role={expected_quote}AXButton{expected_quote}, title={expected_quote}New Item{expected_quote}" in skill_text + assert "Scroll in Example using the recorded wheel direction" in skill_text + assert "Enter user-provided text" in skill_text + assert "## Runtime Inputs" in skill_text + assert "Runtime text for role=" in skill_text + assert "use a fresh runtime value" in skill_text + assert "Runtime selection or selected-content meaning" in skill_text + assert "## Workflow Readiness" in skill_text + assert "Status: needsReview" in skill_text + assert "Can create skill draft: true" in skill_text + assert "Requires human review: true" in skill_text + assert "recording includes actions that may require explicit user confirmation" in skill_text + assert "## Summary Limits" in skill_text + assert "No high-volume summary fields were truncated." in skill_text + assert "## Agent Replay Procedure" in skill_text + assert "connector, API, or dedicated tool" in skill_text + assert "visually dependent verification" in skill_text + assert "get_app_state" in skill_text + assert "`element_index` actions" in skill_text + assert "## Verification" in skill_text + assert "### Confirmation Signals" in skill_text + assert "`keyboard.submit` matched submitAction" in skill_text + assert "ask for explicit confirmation before replaying this step" in skill_text + assert "open-computer-use event-stream validate --json --strict-ocu --require-skill-draft " in skill_text + assert "## Finalizing The Skill" in skill_text + assert "skill-creator" in skill_text + assert "Read and follow the `skill-creator` skill" in skill_text + assert "Complete the `skill-creator` workflow, including validation" in skill_text + assert "actual discoverable skill directory" in skill_text + assert "not a standalone runbook or replay plan" in skill_text + summary = json.loads(summary_text) + assert summary["eventsPath"] == "" + if expect_metadata_path: + assert summary["metadataPath"] == "" + else: + assert "metadataPath" not in summary + assert summary["sessionDir"] == "" + assert summary["runtimeInputs"][0]["kind"] == "text" + assert summary["runtimeInputs"][0]["sourceEventType"] == "keyboard.text_input" + assert summary["runtimeInputs"][0]["textLength"] == 12 + assert summary["runtimeInputs"][1]["kind"] == "selection" + assert summary["runtimeInputs"][1]["sourceEventType"] == "selection.changed" + assert summary["safetySignals"][0]["sourceEventType"] == "keyboard.submit" + assert summary["safetySignals"][0]["reason"] == "submitAction" + assert summary["safetySignals"][0]["confirmationRequired"] is True + assert summary["skillEvidence"]["hasSafetySignals"] is True + assert summary["summaryLimits"]["hasTruncatedSummary"] is False + assert summary["summaryLimits"]["omittedCounts"]["actionSequence"] == 0 + assert summary["skillReadiness"]["status"] == "needsReview" + assert summary["skillReadiness"]["canCreateSkillDraft"] is True + + +def run_source_scaffold( + metadata_path: pathlib.Path, + output_dir: pathlib.Path, + expect_metadata_path: bool = True, +) -> None: + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/scaffold-event-stream-skill.py"), + str(metadata_path), + "--skill-name", + "recorded-example-workflow", + "--description", + "Replay the recorded Example app workflow.", + "--output-dir", + str(output_dir), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr or completed.stdout) + assert_generated_skill(output_dir, expected_quote="'", expect_metadata_path=expect_metadata_path) + + +def run_source_summary_reports_truncation(metadata_path: pathlib.Path) -> None: + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr or completed.stdout) + summary = json.loads(completed.stdout) + assert summary["summaryLimits"]["hasTruncatedSummary"] is True + assert summary["summaryLimits"]["omittedCounts"]["actionSequence"] == 5 + assert summary["summaryLimits"]["omittedCounts"]["targetElements"] == 30 + assert len(summary["actionSequence"]) == 50 + assert len(summary["targetElements"]) == 25 + assert "recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill" in summary["warnings"] + assert summary["skillEvidence"]["hasTruncatedSummary"] is True + assert ( + "recording summary truncated high-volume fields; inspect events.jsonl before finalizing a skill" + in summary["skillReadiness"]["reasons"] + ) + + +def runtime_cli_path() -> pathlib.Path: + override = os.environ.get("OPEN_COMPUTER_USE_CLI") + if override: + return pathlib.Path(override) + + subprocess.run( + ["swift", "build", "--product", "OpenComputerUse"], + cwd=str(REPO_ROOT), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + check=True, + ) + return REPO_ROOT / ".build/debug/OpenComputerUse" + + +def run_runtime_scaffold( + metadata_path: pathlib.Path, + output_dir: pathlib.Path, + expect_metadata_path: bool = True, +) -> None: + cli = runtime_cli_path() + completed = subprocess.run( + [ + str(cli), + "event-stream", + "scaffold-skill", + "--json", + str(metadata_path), + "--skill-name", + "recorded-example-workflow", + "--description", + "Replay the recorded Example app workflow.", + "--output-dir", + str(output_dir), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + if completed.returncode != 0: + raise AssertionError(completed.stderr or completed.stdout) + payload = json.loads(completed.stdout) + assert payload["ok"] is True + assert payload["skillName"] == "recorded-example-workflow" + assert pathlib.Path(payload["skillPath"]).exists() + assert pathlib.Path(payload["summaryPath"]).exists() + assert_generated_skill(output_dir, expected_quote='"', expect_metadata_path=expect_metadata_path) + + +def run_validator_skill_draft_gate(metadata_path: pathlib.Path) -> None: + source_completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(metadata_path), + "--require-skill-draft", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if source_completed.returncode != 0: + raise AssertionError(source_completed.stderr or source_completed.stdout) + source_payload = json.loads(source_completed.stdout) + assert source_payload["skillDraftReady"] is True + + cli = runtime_cli_path() + runtime_completed = subprocess.run( + [ + str(cli), + "event-stream", + "validate", + "--json", + "--require-skill-draft", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + if runtime_completed.returncode != 0: + raise AssertionError(runtime_completed.stderr or runtime_completed.stdout) + runtime_payload = json.loads(runtime_completed.stdout) + assert runtime_payload["skillDraftReady"] is True + + +def run_external_screenshot_path_validation_gate(root: pathlib.Path) -> None: + metadata_path = write_recording(root / "external-screenshot-session") + external_screenshot = root / "outside-session-screenshot.png" + external_screenshot.write_bytes(b"\x89PNG") + + events_path = metadata_path.parent / "events.jsonl" + events = [json.loads(line) for line in events_path.read_text(encoding="utf-8").splitlines() if line.strip()] + events[2]["accessibilityInspectorPayload"] = {"screenshotPath": str(external_screenshot)} + events_path.write_text( + "".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events), + encoding="utf-8", + ) + + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(metadata_path), + "--strict-ocu", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source validator accepted an external screenshotPath") + payload = json.loads(completed.stderr or completed.stdout) + expected = f"screenshotPath from event line 3 must stay inside session directory: {external_screenshot}" + assert expected in payload["errors"] + assert f"screenshotPath from event line 3 does not exist: {external_screenshot}" not in payload["errors"] + + +def run_events_jsonl_input_gate(metadata_path: pathlib.Path, root: pathlib.Path) -> None: + events_path = metadata_path.parent / "events.jsonl" + source_completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(events_path), + "--require-skill-draft", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if source_completed.returncode != 0: + raise AssertionError(source_completed.stderr or source_completed.stdout) + source_payload = json.loads(source_completed.stdout) + assert source_payload["skillDraftReady"] is True + assert source_payload["metadataPath"] is None + assert "metadata/session files not available; validating events.jsonl only" in source_payload["warnings"] + + source_output_dir = root / "events-jsonl-source-skill" + run_source_scaffold(events_path, source_output_dir, expect_metadata_path=False) + + cli = runtime_cli_path() + runtime_completed = subprocess.run( + [ + str(cli), + "event-stream", + "validate", + "--json", + "--require-skill-draft", + "--events-path", + str(events_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + if runtime_completed.returncode != 0: + raise AssertionError(runtime_completed.stderr or runtime_completed.stdout) + runtime_payload = json.loads(runtime_completed.stdout) + assert runtime_payload["skillDraftReady"] is True + assert "metadataPath" not in runtime_payload + assert "metadata/session files not available; validating events.jsonl only" in runtime_payload["warnings"] + + runtime_output_dir = root / "events-jsonl-runtime-skill" + run_runtime_scaffold(events_path, runtime_output_dir, expect_metadata_path=False) + + +def write_recording( + session_dir: pathlib.Path, + state: str = "stopped", + end_reason: str = "recording_controls_stopped", + include_blocking_diagnostic: bool = False, + include_session_ended: bool = True, +) -> pathlib.Path: + session_dir.mkdir() + events_path = session_dir / "events.jsonl" + metadata_path = session_dir / "metadata.json" + session_path = session_dir / "session.json" + suppressed_path = session_dir / "suppressed.jsonl" + events = [ + { + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.000Z", + }, + { + "type": "window.changed", + "timestamp": "2026-06-26T00:00:01.000Z", + "appName": "Example", + "bundleIdentifier": "com.example.app", + "windowTitle": "Example Window", + }, + { + "type": "AX.focusedWindowChanged", + "timestamp": "2026-06-26T00:00:02.000Z", + "appName": "Example", + "windowTitle": "Example Window", + }, + { + "type": "mouse.click", + "timestamp": "2026-06-26T00:00:03.000Z", + "appName": "Example", + "windowTitle": "Example Window", + "targetAccessibilityElement": { + "role": "AXButton", + "title": "New Item", + "actions": ["AXPress"], + }, + }, + { + "type": "experimentalRawEvents", + "timestamp": "2026-06-26T00:00:03.500Z", + "appName": "Example", + "windowTitle": "Example Window", + "reason": "scrollWheel", + "experimentalRawEvents": [ + { + "eventType": "scrollWheel", + "scrollingDeltaX": 0, + "scrollingDeltaY": -4, + "hasPreciseScrollingDeltas": False, + }, + ], + }, + { + "type": "keyboard.text_input", + "timestamp": "2026-06-26T00:00:04.000Z", + "appName": "Example", + "windowTitle": "Example Window", + "textLength": 12, + "focusedAccessibilityElement": { + "role": "AXTextField", + "title": "Name", + }, + }, + { + "type": "keyboard.submit", + "timestamp": "2026-06-26T00:00:04.250Z", + "appName": "Example", + "windowTitle": "Example Window", + "focusedAccessibilityElement": { + "role": "AXButton", + "title": "Submit", + }, + }, + { + "type": "selection.changed", + "timestamp": "2026-06-26T00:00:04.500Z", + "appName": "Example", + "windowTitle": "Example Window", + "selectedText": "example selected value", + "focusedAccessibilityElement": { + "role": "AXTextArea", + "title": "Body", + }, + }, + ] + if include_blocking_diagnostic: + events.append( + { + "type": "debug.error", + "timestamp": "2026-06-26T00:00:04.750Z", + "subsystem": "inputMonitoring", + "reason": "inputMonitorsUnavailable", + "errorType": "permission", + } + ) + if include_session_ended: + events.append( + { + "type": "session.ended", + "timestamp": "2026-06-26T00:00:05.000Z", + "endReason": end_reason, + } + ) + events_path.write_text( + "".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events), + encoding="utf-8", + ) + suppressed_path.write_text("", encoding="utf-8") + metadata = { + "sessionId": f"session-test-{state}", + "state": state, + "active": state == "recording", + "endReason": end_reason, + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "suppressedEventsPath": str(suppressed_path), + "eventCount": len(events), + "suppressedEventCount": 0, + } + if state == "recording": + metadata["currentSegmentEventsPath"] = str(events_path) + metadata["currentSegmentMetadataPath"] = str(metadata_path) + write_json(metadata_path, metadata) + write_json(session_path, metadata) + return metadata_path + + +def write_invalid_event_count_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metadata["eventCount"] = 999 + write_json(metadata_path, metadata) + write_json(metadata_path.parent / "session.json", metadata) + return metadata_path + + +def write_events(metadata_path: pathlib.Path, events: list[dict]) -> None: + events_path = metadata_path.parent / "events.jsonl" + events_path.write_text( + "".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events), + encoding="utf-8", + ) + metadata = json.loads(metadata_path.read_text(encoding="utf-8")) + metadata["eventCount"] = len(events) + write_json(metadata_path, metadata) + write_json(metadata_path.parent / "session.json", metadata) + + +def read_events(metadata_path: pathlib.Path) -> list[dict]: + events_path = metadata_path.parent / "events.jsonl" + return [ + json.loads(line) + for line in events_path.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + + +def write_missing_session_started_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + events = [event for event in read_events(metadata_path) if event.get("type") != "session.started"] + write_events(metadata_path, events) + return metadata_path + + +def write_duplicate_session_started_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + events = read_events(metadata_path) + events.insert( + 1, + { + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.500Z", + }, + ) + write_events(metadata_path, events) + return metadata_path + + +def write_session_started_not_first_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + events = read_events(metadata_path) + first = events.pop(0) + events.insert(1, first) + write_events(metadata_path, events) + return metadata_path + + +def write_session_ended_not_final_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + events = read_events(metadata_path) + events.append( + { + "type": "window.changed", + "timestamp": "2026-06-26T00:00:06.000Z", + "appName": "Example", + "windowTitle": "Late Window", + } + ) + write_events(metadata_path, events) + return metadata_path + + +def write_duplicate_session_ended_recording(session_dir: pathlib.Path) -> pathlib.Path: + metadata_path = write_recording(session_dir) + events = read_events(metadata_path) + events.append( + { + "type": "session.ended", + "timestamp": "2026-06-26T00:00:06.000Z", + "endReason": "recording_controls_stopped", + } + ) + write_events(metadata_path, events) + return metadata_path + + +def write_long_recording(session_dir: pathlib.Path) -> pathlib.Path: + session_dir.mkdir() + events_path = session_dir / "events.jsonl" + metadata_path = session_dir / "metadata.json" + session_path = session_dir / "session.json" + suppressed_path = session_dir / "suppressed.jsonl" + events = [ + { + "type": "session.started", + "timestamp": "2026-06-26T00:00:00.000Z", + }, + { + "type": "window.changed", + "timestamp": "2026-06-26T00:00:01.000Z", + "appName": "Example", + "windowTitle": "Long Example Window", + }, + ] + for index in range(55): + events.append( + { + "type": "mouse.click", + "timestamp": f"2026-06-26T00:00:{index + 2:02d}.000Z", + "appName": "Example", + "windowTitle": "Long Example Window", + "targetAccessibilityElement": { + "role": "AXButton", + "title": f"Step {index}", + }, + } + ) + events.append( + { + "type": "session.ended", + "timestamp": "2026-06-26T00:01:00.000Z", + "endReason": "recording_controls_stopped", + } + ) + events_path.write_text( + "".join(json.dumps(event, separators=(",", ":")) + "\n" for event in events), + encoding="utf-8", + ) + suppressed_path.write_text("", encoding="utf-8") + metadata = { + "sessionId": "long-session-test", + "state": "stopped", + "endReason": "recording_controls_stopped", + "eventsPath": str(events_path), + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "suppressedEventsPath": str(suppressed_path), + "eventCount": len(events), + "suppressedEventCount": 0, + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + return metadata_path + + +def run_source_scaffold_rejected(metadata_path: pathlib.Path, output_dir: pathlib.Path, expected_reason: str) -> None: + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/scaffold-event-stream-skill.py"), + str(metadata_path), + "--skill-name", + "cancelled-recording-workflow", + "--description", + "Replay the cancelled recording.", + "--output-dir", + str(output_dir), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + combined = completed.stdout + completed.stderr + if completed.returncode == 0: + raise AssertionError("source scaffold accepted a recording that should be rejected") + assert expected_reason in combined + assert not output_dir.exists() + + +def run_runtime_scaffold_rejected(metadata_path: pathlib.Path, output_dir: pathlib.Path, expected_reason: str) -> None: + cli = runtime_cli_path() + completed = subprocess.run( + [ + str(cli), + "event-stream", + "scaffold-skill", + "--json", + str(metadata_path), + "--skill-name", + "cancelled-recording-workflow", + "--description", + "Replay the cancelled recording.", + "--output-dir", + str(output_dir), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + combined = completed.stdout + completed.stderr + if completed.returncode == 0: + payload = json.loads(completed.stdout) + assert payload["ok"] is False + assert expected_reason in combined + assert not output_dir.exists() + + +def run_validator_skill_draft_gate_rejected(metadata_path: pathlib.Path, expected_reason: str) -> None: + source_completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(metadata_path), + "--require-skill-draft", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + source_combined = source_completed.stdout + source_completed.stderr + if source_completed.returncode == 0: + raise AssertionError("source validator accepted a skill draft recording that should be rejected") + assert expected_reason in source_combined + source_payload = json.loads(source_completed.stderr or source_completed.stdout) + assert source_payload["skillDraftReady"] is False + assert expected_reason in source_payload["skillDraftReasons"] + + cli = runtime_cli_path() + runtime_completed = subprocess.run( + [ + str(cli), + "event-stream", + "validate", + "--json", + "--require-skill-draft", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + runtime_combined = runtime_completed.stdout + runtime_completed.stderr + if runtime_completed.returncode == 0: + payload = json.loads(runtime_completed.stdout) + assert payload["ok"] is False + assert payload["skillDraftReady"] is False + assert expected_reason in runtime_combined + runtime_payload = json.loads(runtime_completed.stderr or runtime_completed.stdout) + assert runtime_payload["skillDraftReady"] is False + assert expected_reason in runtime_payload["skillDraftReasons"] + + +def run_current_segment_strict_validation_gate(root: pathlib.Path) -> None: + cli = runtime_cli_path() + + active_metadata_path = write_recording( + root / "strict-current-segment-active", + state="recording", + end_reason="", + include_session_ended=False, + ) + active_metadata = json.loads(active_metadata_path.read_text(encoding="utf-8")) + active_metadata.pop("currentSegmentEventsPath") + active_metadata.pop("currentSegmentMetadataPath") + write_json(active_metadata_path, active_metadata) + write_json(active_metadata_path.parent / "session.json", active_metadata) + + final_metadata_path = write_recording(root / "strict-current-segment-final") + final_metadata = json.loads(final_metadata_path.read_text(encoding="utf-8")) + final_metadata["currentSegmentEventsPath"] = final_metadata["eventsPath"] + final_metadata["currentSegmentMetadataPath"] = final_metadata["metadataPath"] + write_json(final_metadata_path, final_metadata) + write_json(final_metadata_path.parent / "session.json", final_metadata) + + cases = [ + ( + active_metadata_path, + [ + "recording state requires currentSegmentEventsPath", + "recording state requires currentSegmentMetadataPath", + ], + ), + ( + final_metadata_path, + [ + "final state must not include currentSegmentEventsPath", + "final state must not include currentSegmentMetadataPath", + ], + ), + ] + for metadata_path, expected_errors in cases: + source = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(metadata_path), + "--strict-ocu", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert source.returncode != 0, source.stdout + source_payload = json.loads(source.stderr or source.stdout) + for expected_error in expected_errors: + assert expected_error in source_payload["errors"], source_payload + + runtime = subprocess.run( + [ + str(cli), + "event-stream", + "validate", + "--json", + "--strict-ocu", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + runtime_payload = json.loads(runtime.stdout or runtime.stderr) + assert runtime_payload["ok"] is False, runtime_payload + for expected_error in expected_errors: + assert expected_error in runtime_payload["errors"], runtime_payload + + +def run_declared_handoff_path_strict_validation_gate(root: pathlib.Path) -> None: + cli = runtime_cli_path() + + missing_metadata_path = write_recording(root / "strict-declared-paths-missing") + missing_metadata = json.loads(missing_metadata_path.read_text(encoding="utf-8")) + for key in ["metadataPath", "sessionPath", "eventsPath", "suppressedEventsPath"]: + missing_metadata.pop(key) + write_json(missing_metadata_path, missing_metadata) + write_json(missing_metadata_path.parent / "session.json", missing_metadata) + + bad_metadata_path = write_recording(root / "strict-declared-paths-bad-session") + bad_metadata = json.loads(bad_metadata_path.read_text(encoding="utf-8")) + bad_metadata["sessionPath"] = "missing-session.json" + write_json(bad_metadata_path, bad_metadata) + write_json(bad_metadata_path.parent / "session.json", bad_metadata) + + cases = [ + ( + missing_metadata_path, + [ + "strict OCU validation requires metadataPath", + "strict OCU validation requires sessionPath", + "strict OCU validation requires eventsPath", + "strict OCU validation requires suppressedEventsPath", + ], + ), + ( + bad_metadata_path, + [ + "sessionPath does not exist: missing-session.json", + ], + ), + ] + for metadata_path, expected_errors in cases: + source = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/validate-event-stream-recording.py"), + str(metadata_path), + "--strict-ocu", + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + assert source.returncode != 0, source.stdout + source_payload = json.loads(source.stderr or source.stdout) + assert set(source_payload["declaredPaths"].keys()) == { + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + } + for expected_error in expected_errors: + assert expected_error in source_payload["errors"], source_payload + + runtime = subprocess.run( + [ + str(cli), + "event-stream", + "validate", + "--json", + "--strict-ocu", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + env={ + **os.environ, + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + runtime_payload = json.loads(runtime.stdout or runtime.stderr) + assert runtime_payload["ok"] is False, runtime_payload + assert set(runtime_payload["declaredPaths"].keys()) == { + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + } + for expected_error in expected_errors: + assert expected_error in runtime_payload["errors"], runtime_payload + + +def run_source_summary_rejects_blocking_diagnostics(metadata_path: pathlib.Path) -> None: + expected_reason = "recording has blocking diagnostics; fix recording permissions and re-record before creating a skill" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a blocking diagnostic recording") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["skillEvidence"]["hasBlockingDiagnostics"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert payload["blockingDiagnostics"][0]["reason"] == "inputMonitorsUnavailable" + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_incomplete_recording(metadata_path: pathlib.Path) -> None: + expected_reason = "recording is not complete; stop the recording before creating a skill" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted an incomplete recording") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["state"] == "recording" + assert payload["skillEvidence"]["recordingIncomplete"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_session_started_count(metadata_path: pathlib.Path, expected_count: int) -> None: + expected_reason = "recording must contain exactly one session.started event before creating a skill" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a recording with invalid session.started count") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["sessionStartedCount"] == expected_count + assert payload["skillEvidence"]["sessionStartedCountInvalid"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_session_started_not_first(metadata_path: pathlib.Path) -> None: + expected_reason = "recording has events before session.started; start must be the first event before creating a skill" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a recording with events before session.started") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["firstEventType"] == "window.changed" + assert payload["sessionStartedCount"] == 1 + assert payload["sessionStartedIsFirst"] is False + assert payload["skillEvidence"]["sessionStartedNotFirst"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_session_ended_not_final(metadata_path: pathlib.Path) -> None: + expected_reason = "recording has events after session.ended; stop or cancel must be the final event before creating a skill" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a recording with events after session.ended") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["finalEventType"] == "window.changed" + assert payload["sessionEndedIsFinal"] is False + assert payload["skillEvidence"]["sessionEndedNotFinal"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_duplicate_session_ended(metadata_path: pathlib.Path) -> None: + expected_reason = "recording has multiple session.ended events; stop or cancel must close the event stream exactly once" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(metadata_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a recording with duplicate session.ended events") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["sessionEndedCount"] == 2 + assert payload["sessionEndedIsFinal"] is True + assert payload["skillEvidence"]["sessionEndedCountInvalid"] is True + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def run_source_summary_rejects_cancelled_events_jsonl(events_path: pathlib.Path) -> None: + expected_reason = "recording was cancelled; do not create or update a skill from this event stream" + completed = subprocess.run( + [ + sys.executable, + str(REPO_ROOT / "scripts/summarize-event-stream-recording.py"), + "--require-action", + str(events_path), + ], + cwd=str(REPO_ROOT), + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + if completed.returncode == 0: + raise AssertionError("source summary accepted a cancelled events.jsonl recording") + payload = json.loads(completed.stderr or completed.stdout) + assert payload["ok"] is False + assert payload["endReason"] == "recording_controls_cancelled" + assert payload["skillReadiness"]["status"] == "insufficient" + assert payload["skillReadiness"]["canCreateSkillDraft"] is False + assert expected_reason in payload["errors"] + assert expected_reason in payload["warnings"] + assert expected_reason in payload["skillReadiness"]["reasons"] + + +def main() -> int: + with tempfile.TemporaryDirectory(prefix="event-stream-skill-scaffold-") as raw_root: + root = pathlib.Path(raw_root) + session_dir = root / "session" + source_output_dir = root / "generated-source-skill" + runtime_output_dir = root / "generated-runtime-skill" + + metadata_path = write_recording(session_dir) + long_metadata_path = write_long_recording(root / "long-session") + + run_source_summary_reports_truncation(long_metadata_path) + run_validator_skill_draft_gate(metadata_path) + run_external_screenshot_path_validation_gate(root) + run_current_segment_strict_validation_gate(root) + run_declared_handoff_path_strict_validation_gate(root) + run_events_jsonl_input_gate(metadata_path, root) + run_source_scaffold(metadata_path, source_output_dir) + run_runtime_scaffold(metadata_path, runtime_output_dir) + + cancelled_metadata_path = write_recording( + root / "cancelled-session", + state="cancelled", + end_reason="recording_controls_cancelled", + ) + cancelled_reason = "recording was cancelled; do not create or update a skill from this event stream" + run_validator_skill_draft_gate_rejected(cancelled_metadata_path, cancelled_reason) + run_source_scaffold_rejected(cancelled_metadata_path, root / "cancelled-source-skill", cancelled_reason) + run_runtime_scaffold_rejected(cancelled_metadata_path, root / "cancelled-runtime-skill", cancelled_reason) + cancelled_events_path = cancelled_metadata_path.parent / "events.jsonl" + run_source_summary_rejects_cancelled_events_jsonl(cancelled_events_path) + run_validator_skill_draft_gate_rejected(cancelled_events_path, cancelled_reason) + run_source_scaffold_rejected(cancelled_events_path, root / "cancelled-events-source-skill", cancelled_reason) + run_runtime_scaffold_rejected(cancelled_events_path, root / "cancelled-events-runtime-skill", cancelled_reason) + + blocking_metadata_path = write_recording( + root / "blocking-diagnostic-session", + include_blocking_diagnostic=True, + ) + blocking_reason = "recording has blocking diagnostics; fix recording permissions and re-record before creating a skill" + run_source_summary_rejects_blocking_diagnostics(blocking_metadata_path) + run_validator_skill_draft_gate_rejected(blocking_metadata_path, blocking_reason) + run_source_scaffold_rejected(blocking_metadata_path, root / "blocking-source-skill", blocking_reason) + run_runtime_scaffold_rejected(blocking_metadata_path, root / "blocking-runtime-skill", blocking_reason) + + incomplete_metadata_path = write_recording( + root / "incomplete-session", + state="recording", + end_reason="", + include_session_ended=False, + ) + incomplete_reason = "recording is not complete; stop the recording before creating a skill" + run_source_summary_rejects_incomplete_recording(incomplete_metadata_path) + run_validator_skill_draft_gate_rejected(incomplete_metadata_path, incomplete_reason) + run_source_scaffold_rejected(incomplete_metadata_path, root / "incomplete-source-skill", incomplete_reason) + run_runtime_scaffold_rejected(incomplete_metadata_path, root / "incomplete-runtime-skill", incomplete_reason) + + missing_start_metadata_path = write_missing_session_started_recording(root / "missing-session-started") + invalid_start_count_reason = "recording must contain exactly one session.started event before creating a skill" + run_source_summary_rejects_session_started_count(missing_start_metadata_path, 0) + run_validator_skill_draft_gate_rejected(missing_start_metadata_path, invalid_start_count_reason) + run_source_scaffold_rejected(missing_start_metadata_path, root / "missing-start-source-skill", invalid_start_count_reason) + run_runtime_scaffold_rejected(missing_start_metadata_path, root / "missing-start-runtime-skill", invalid_start_count_reason) + + duplicate_start_metadata_path = write_duplicate_session_started_recording(root / "duplicate-session-started") + run_source_summary_rejects_session_started_count(duplicate_start_metadata_path, 2) + run_validator_skill_draft_gate_rejected(duplicate_start_metadata_path, invalid_start_count_reason) + run_source_scaffold_rejected(duplicate_start_metadata_path, root / "duplicate-start-source-skill", invalid_start_count_reason) + run_runtime_scaffold_rejected(duplicate_start_metadata_path, root / "duplicate-start-runtime-skill", invalid_start_count_reason) + + start_not_first_metadata_path = write_session_started_not_first_recording(root / "session-started-not-first") + start_not_first_reason = "recording has events before session.started; start must be the first event before creating a skill" + run_source_summary_rejects_session_started_not_first(start_not_first_metadata_path) + run_validator_skill_draft_gate_rejected(start_not_first_metadata_path, start_not_first_reason) + run_source_scaffold_rejected(start_not_first_metadata_path, root / "start-not-first-source-skill", start_not_first_reason) + run_runtime_scaffold_rejected(start_not_first_metadata_path, root / "start-not-first-runtime-skill", start_not_first_reason) + + not_final_metadata_path = write_session_ended_not_final_recording(root / "session-ended-not-final") + not_final_reason = "recording has events after session.ended; stop or cancel must be the final event before creating a skill" + run_source_summary_rejects_session_ended_not_final(not_final_metadata_path) + run_validator_skill_draft_gate_rejected(not_final_metadata_path, not_final_reason) + run_source_scaffold_rejected(not_final_metadata_path, root / "not-final-source-skill", not_final_reason) + run_runtime_scaffold_rejected(not_final_metadata_path, root / "not-final-runtime-skill", not_final_reason) + + duplicate_end_metadata_path = write_duplicate_session_ended_recording(root / "duplicate-session-ended") + duplicate_end_reason = "recording has multiple session.ended events; stop or cancel must close the event stream exactly once" + run_source_summary_rejects_duplicate_session_ended(duplicate_end_metadata_path) + run_validator_skill_draft_gate_rejected(duplicate_end_metadata_path, duplicate_end_reason) + run_source_scaffold_rejected(duplicate_end_metadata_path, root / "duplicate-end-source-skill", duplicate_end_reason) + run_runtime_scaffold_rejected(duplicate_end_metadata_path, root / "duplicate-end-runtime-skill", duplicate_end_reason) + + invalid_count_metadata_path = write_invalid_event_count_recording(root / "invalid-count-session") + invalid_count_reason = "eventCount=999 does not match events.jsonl lines=9" + run_source_scaffold_rejected( + invalid_count_metadata_path, + root / "invalid-count-source-skill", + invalid_count_reason, + ) + run_runtime_scaffold_rejected( + invalid_count_metadata_path, + root / "invalid-count-runtime-skill", + invalid_count_reason, + ) + + print( + json.dumps( + { + "checkedExternalScreenshotPathBoundary": True, + "ok": True, + "scaffold": "event-stream-skill", + "runtimeCli": True, + } + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs b/scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs new file mode 100755 index 00000000..7051d472 --- /dev/null +++ b/scripts/test-npm-record-and-replay-skill-repo-scaffold.mjs @@ -0,0 +1,1265 @@ +#!/usr/bin/env node + +import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { spawnSync } from "node:child_process"; +import { fileURLToPath } from "node:url"; + +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(__dirname, ".."); + +function run(command, args, options = {}) { + const result = spawnSync(command, args, { + cwd: repoRoot, + encoding: "utf8", + ...options, + }); + if (result.status !== 0) { + throw new Error( + [ + `${command} ${args.join(" ")} failed with exit code ${result.status ?? "unknown"}`, + result.stdout, + result.stderr, + ].filter(Boolean).join("\n"), + ); + } + return result; +} + +function runAllowFailure(command, args, options = {}) { + return spawnSync(command, args, { + cwd: repoRoot, + encoding: "utf8", + ...options, + }); +} + +function readJSON(filePath) { + return JSON.parse(readFileSync(filePath, "utf8")); +} + +function writeJSON(filePath, value) { + writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8"); +} + +function parseJSONObjectLines(stdout) { + return stdout + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line.startsWith("{")) + .map((line) => JSON.parse(line)); +} + +const tmp = mkdtempSync(path.join(tmpdir(), "ocu-npm-rnr-skill-repo-")); +const baselineSummaryArtifact = path.join(repoRoot, "dist", "record-and-replay-baseline-summary.json"); +const baselineSummaryOriginal = readFileSync(baselineSummaryArtifact, "utf8"); + +function writeCurrentBaselineSummaryArtifact() { + const baselineSummary = JSON.parse(baselineSummaryOriginal); + baselineSummary.evidence ??= {}; + baselineSummary.evidence.fixtureIngestPipelines ??= {}; + baselineSummary.evidence.fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff = true; + writeJSON(baselineSummaryArtifact, baselineSummary); + return baselineSummary; +} + +try { + const baselineSummaryBackup = `${baselineSummaryArtifact}.bak-test-${process.pid}`; + renameSync(baselineSummaryArtifact, baselineSummaryBackup); + try { + const missingBaselineSummary = runAllowFailure("node", [ + path.join(repoRoot, "scripts", "npm", "build-packages.mjs"), + "--skip-build", + "--out-dir", + path.join(tmp, "missing-baseline-summary-npm"), + "--package", + "open-computer-use", + ]); + if (missingBaselineSummary.status === 0) { + throw new Error("npm package build unexpectedly succeeded without the Record & Replay baseline summary artifact"); + } + if (!missingBaselineSummary.stderr.includes("Missing Record & Replay baseline summary artifact")) { + throw new Error(missingBaselineSummary.stderr || missingBaselineSummary.stdout); + } + if (!missingBaselineSummary.stderr.includes("make record-and-replay-baseline-audit")) { + throw new Error(missingBaselineSummary.stderr || missingBaselineSummary.stdout); + } + } finally { + renameSync(baselineSummaryBackup, baselineSummaryArtifact); + } + const currentBaselineSummary = writeCurrentBaselineSummaryArtifact(); + const sourceHasSuccessfulRecordingGolden = + currentBaselineSummary.status?.officialSuccessfulRecordingGoldenComplete === true; + + const npmOutDir = path.join(tmp, "npm"); + const generatedRepo = path.join(tmp, "generated-rnr-skill-repo"); + const build = run("node", [ + path.join(repoRoot, "scripts", "npm", "build-packages.mjs"), + "--skip-build", + "--out-dir", + npmOutDir, + "--package", + "open-computer-use", + ]); + if (build.stderr) { + process.stderr.write(build.stderr); + } + + const packageRoot = path.join(npmOutDir, "open-computer-use"); + const launcher = path.join(packageRoot, "bin", "open-computer-use"); + if (!existsSync(path.join(packageRoot, "scripts", "record_and_replay_scenarios.py"))) { + throw new Error("npm package is missing scripts/record_and_replay_scenarios.py"); + } + if (!existsSync(path.join(packageRoot, "dist", "record-and-replay-baseline-summary.json"))) { + throw new Error("npm package is missing dist/record-and-replay-baseline-summary.json"); + } + + const help = run("node", [launcher, "help", "scaffold-record-and-replay-skill-repo"]); + if (!help.stdout.includes("open-computer-use scaffold-record-and-replay-skill-repo --output-dir ")) { + throw new Error(help.stdout); + } + if (!help.stdout.includes("Generates a standalone Record & Replay thin skill repo")) { + throw new Error(help.stdout); + } + if (!help.stdout.includes("Python 3 must be available")) { + throw new Error(help.stdout); + } + if (help.stdout.includes("updates a local MCP or plugin config")) { + throw new Error(help.stdout); + } + + const missingPython = runAllowFailure("node", [ + launcher, + "scaffold-record-and-replay-skill-repo", + "--output-dir", + path.join(tmp, "missing-python-output"), + ], { + env: { + ...process.env, + PYTHON: path.join(tmp, "missing-python3"), + }, + }); + if (missingPython.status === 0) { + throw new Error("scaffold unexpectedly succeeded with missing PYTHON"); + } + if (!missingPython.stderr.includes("Python 3 is required to run")) { + throw new Error(missingPython.stderr); + } + if (!missingPython.stderr.includes("PYTHON=/path/to/python3")) { + throw new Error(missingPython.stderr); + } + + const fakePython2 = path.join(tmp, "python2-fake"); + writeFileSync(fakePython2, "#!/bin/sh\necho 'Python 2.7.18'\nexit 0\n", "utf8"); + chmodSync(fakePython2, 0o755); + const python2 = runAllowFailure("node", [ + launcher, + "scaffold-record-and-replay-skill-repo", + "--output-dir", + path.join(tmp, "python2-output"), + ], { + env: { + ...process.env, + PYTHON: fakePython2, + }, + }); + if (python2.status === 0) { + throw new Error("scaffold unexpectedly accepted Python 2"); + } + if (!python2.stderr.includes("Python 3 is required to run")) { + throw new Error(python2.stderr); + } + + const validateHelp = run("node", [launcher, "event-stream", "validate", "--help"]); + if (!validateHelp.stdout.includes("--require-skill-draft")) { + throw new Error( + [ + "staged open-computer-use binary does not support event-stream validate --require-skill-draft", + "Rebuild dist native artifacts before running make npm-record-and-replay-skill-repo-smoke.", + validateHelp.stdout, + ].join("\n"), + ); + } + + const preflightSession = path.join(tmp, "declared-path-preflight", "session"); + mkdirSync(preflightSession, { recursive: true }); + const preflightMetadataPath = path.join(preflightSession, "metadata.json"); + const preflightSessionPath = path.join(preflightSession, "session.json"); + const preflightEventsPath = path.join(preflightSession, "events.jsonl"); + const preflightSuppressedEventsPath = path.join(preflightSession, "suppressed.jsonl"); + const preflightEvents = [ + { type: "session.started", timestamp: "2026-06-27T00:00:00.000Z" }, + { + type: "mouse.click", + timestamp: "2026-06-27T00:00:01.000Z", + location: { x: 20, y: 30 }, + targetAccessibilityElement: { role: "AXButton", title: "Continue" }, + }, + { + type: "session.ended", + timestamp: "2026-06-27T00:00:02.000Z", + endReason: "recording_controls_stopped", + }, + ]; + writeFileSync(preflightEventsPath, `${preflightEvents.map((event) => JSON.stringify(event)).join("\n")}\n`, "utf8"); + writeFileSync(preflightSuppressedEventsPath, "", "utf8"); + const preflightMetadata = { + sessionId: "npm-declared-path-preflight", + state: "stopped", + active: false, + endReason: "recording_controls_stopped", + eventCount: preflightEvents.length, + suppressedEventCount: 0, + metadataPath: preflightMetadataPath, + sessionPath: preflightSessionPath, + eventsPath: preflightEventsPath, + suppressedEventsPath: preflightSuppressedEventsPath, + }; + writeJSON(preflightMetadataPath, preflightMetadata); + writeJSON(preflightSessionPath, preflightMetadata); + const declaredPathPreflight = run("node", [ + launcher, + "event-stream", + "validate", + "--json", + "--strict-ocu", + "--require-skill-draft", + preflightMetadataPath, + ]); + const declaredPathPayload = JSON.parse(declaredPathPreflight.stdout); + const declaredPaths = declaredPathPayload.declaredPaths ?? {}; + const missingDeclaredPath = ["metadataPath", "sessionPath", "eventsPath", "suppressedEventsPath"].find((key) => { + return declaredPaths[key]?.exists !== true; + }); + if (missingDeclaredPath) { + throw new Error( + [ + "staged open-computer-use binary does not emit strict declaredPaths evidence.", + "Rebuild dist native artifacts before running make npm-record-and-replay-skill-repo-smoke.", + JSON.stringify(declaredPathPayload, null, 2), + ].join("\n"), + ); + } + + const externalScreenshotSession = path.join(tmp, "screenshot-containment-preflight", "session"); + mkdirSync(externalScreenshotSession, { recursive: true }); + const externalScreenshot = path.join(tmp, "outside-session-screenshot.png"); + writeFileSync(externalScreenshot, "not a real png, only path containment evidence\n", "utf8"); + const externalScreenshotMetadataPath = path.join(externalScreenshotSession, "metadata.json"); + const externalScreenshotSessionPath = path.join(externalScreenshotSession, "session.json"); + const externalScreenshotEventsPath = path.join(externalScreenshotSession, "events.jsonl"); + const externalScreenshotSuppressedEventsPath = path.join(externalScreenshotSession, "suppressed.jsonl"); + const externalScreenshotEvents = [ + { type: "session.started", timestamp: "2026-06-27T00:00:00.000Z" }, + { + type: "AX.focusedWindowChanged", + timestamp: "2026-06-27T00:00:01.000Z", + accessibilityInspectorPayload: { + screenshotNeededForContext: true, + screenshotAvailable: true, + screenshotPath: externalScreenshot, + }, + }, + { + type: "mouse.click", + timestamp: "2026-06-27T00:00:02.000Z", + location: { x: 20, y: 30 }, + targetAccessibilityElement: { role: "AXButton", title: "Continue" }, + }, + { + type: "session.ended", + timestamp: "2026-06-27T00:00:03.000Z", + endReason: "recording_controls_stopped", + }, + ]; + writeFileSync( + externalScreenshotEventsPath, + `${externalScreenshotEvents.map((event) => JSON.stringify(event)).join("\n")}\n`, + "utf8", + ); + writeFileSync(externalScreenshotSuppressedEventsPath, "", "utf8"); + const externalScreenshotMetadata = { + sessionId: "npm-screenshot-containment-preflight", + state: "stopped", + active: false, + endReason: "recording_controls_stopped", + eventCount: externalScreenshotEvents.length, + suppressedEventCount: 0, + metadataPath: externalScreenshotMetadataPath, + sessionPath: externalScreenshotSessionPath, + eventsPath: externalScreenshotEventsPath, + suppressedEventsPath: externalScreenshotSuppressedEventsPath, + }; + writeJSON(externalScreenshotMetadataPath, externalScreenshotMetadata); + writeJSON(externalScreenshotSessionPath, externalScreenshotMetadata); + const screenshotContainmentPreflight = runAllowFailure("node", [ + launcher, + "event-stream", + "validate", + "--json", + "--strict-ocu", + externalScreenshotMetadataPath, + ]); + if (screenshotContainmentPreflight.status === 0) { + throw new Error( + [ + "staged open-computer-use binary does not reject screenshotPath outside the session directory.", + "Rebuild dist native artifacts before running make npm-record-and-replay-skill-repo-smoke.", + screenshotContainmentPreflight.stdout, + ].join("\n"), + ); + } + const screenshotContainmentPayload = JSON.parse( + screenshotContainmentPreflight.stdout || screenshotContainmentPreflight.stderr, + ); + const expectedScreenshotContainmentError = + `screenshotPath from event line 2 must stay inside session directory: ${externalScreenshot}`; + if (!screenshotContainmentPayload.errors?.includes(expectedScreenshotContainmentError)) { + throw new Error( + [ + "staged open-computer-use binary does not emit screenshotPath containment evidence.", + "Rebuild dist native artifacts before running make npm-record-and-replay-skill-repo-smoke.", + JSON.stringify(screenshotContainmentPayload, null, 2), + ].join("\n"), + ); + } + + const scaffold = run("node", [ + launcher, + "scaffold-record-and-replay-skill-repo", + "--output-dir", + generatedRepo, + ]); + const scaffoldPayload = JSON.parse(scaffold.stdout); + if (scaffoldPayload.ok !== true) { + throw new Error(scaffold.stdout); + } + + const readme = readFileSync(path.join(generatedRepo, "README.md"), "utf8"); + const verifyManifestPath = path.join(generatedRepo, "scripts", "verify-manifest.py"); + const verifySourceSummaryPath = path.join(generatedRepo, "scripts", "verify-source-baseline-summary.py"); + const verifyReadmePath = path.join(generatedRepo, "scripts", "verify-readme-handoff.py"); + const verifyManifest = readFileSync(verifyManifestPath, "utf8"); + const verifyRuntime = readFileSync(path.join(generatedRepo, "scripts", "verify-runtime.py"), "utf8"); + const manifestPath = path.join(generatedRepo, "record-and-replay-skill-repo.json"); + const sourceSummaryPath = path.join(generatedRepo, "evidence", "source-baseline-summary.json"); + const manifest = readJSON(manifestPath); + const sourceSummary = readJSON(sourceSummaryPath); + if (!readme.includes("Python 3 must be available as `python3`")) { + throw new Error("generated README.md is missing the Python 3 prerequisite"); + } + if (!readme.includes("PYTHON=/path/to/python3")) { + throw new Error("generated README.md is missing the npm launcher Python override guidance"); + } + if (!readme.includes("make record-and-replay-baseline-audit")) { + throw new Error("generated README.md is missing the source repo baseline audit handoff"); + } + if (!readme.includes("make record-and-replay-official-golden-gate-audit")) { + throw new Error("generated README.md is missing the strict official golden audit handoff"); + } + if (!readme.includes("dist/record-and-replay-baseline-summary.json")) { + throw new Error("generated README.md is missing the baseline summary artifact handoff"); + } + if (!readme.includes("evidence/source-baseline-summary.json")) { + throw new Error("generated README.md is missing copied baseline summary evidence handoff"); + } + if (!readme.includes("dist/record-and-replay-official-golden-gate-summary.json")) { + throw new Error("generated README.md is missing the strict official golden summary artifact handoff"); + } + if (!readme.includes("--allow-strict-official-golden-missing")) { + throw new Error("generated README.md is missing the strict expected-failure audit handoff"); + } + if (!verifyManifest.includes("checkedManifestContract")) { + throw new Error("generated verify-manifest.py is missing manifest contract evidence"); + } + if (!verifyManifest.includes("strictOfficialGoldenExpectedFailureAudit")) { + throw new Error("generated verify-manifest.py is missing strict expected-failure audit validation"); + } + if (!readFileSync(verifySourceSummaryPath, "utf8").includes("checkedSourceBaselineSummaryEvidence")) { + throw new Error("generated verify-source-baseline-summary.py is missing source summary evidence validation"); + } + if (!verifyRuntime.includes("OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS")) { + throw new Error("generated verify-runtime.py is missing runtime timeout override support"); + } + if (!verifyRuntime.includes("runtimeVersion=")) { + throw new Error("generated verify-runtime.py is missing runtime version timeout diagnostics"); + } + if (!verifyRuntime.includes("Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime")) { + throw new Error("generated verify-runtime.py is missing stale runtime guidance"); + } + const expectedOfficialEvidence = { + baselineVersion: "record-and-replay/1.0.857", + nonRecordingSurfaceFixture: "record-and-replay-event-stream-surface-1.0.857.json", + noActiveStatusStopFixture: "record-and-replay-official-no-active-status-stop-1.0.857.json", + hostlessRawStartTimeoutFixture: "record-and-replay-official-raw-start-timeout-1.0.857.json", + sourceRepoBaselineChecks: { + baselineContract: "baseline-contract-smoke", + officialRawStartTimeout: "official-raw-start-timeout-fixture-smoke", + officialFixtureSetGate: { + check: "official-fixture-set-smoke", + sameScenarioComparePolicy: { + requiresAxDiffEvidence: true, + requiresSameAxDiffMarkers: true, + requiresSameSuppressedEventSequence: true, + requiresSameSuppressedSchema: true, + }, + }, + officialFixtureIngest: { + check: "official-fixture-ingest-smoke", + requiredEvidence: [ + "checkedOfficialSessionDirectoryPathHandoff", + ], + }, + officialGoldenCapturePreflight: { + check: "official-golden-capture-preflight-smoke", + requiredEvidence: [ + "checkedOfficialCapturePacketInputSemanticGuard", + "checkedOfficialCapturePacketSetContractManifest", + "checkedOfficialCapturePacketPostCaptureWorkflow", + "checkedOfficialCapturePacketWorkflowVerifier", + "checkedOfficialCapturePacketSetPostCaptureWorkflow", + "checkedOfficialCapturePacketSetWorkflowVerifier", + "checkedOfficialCapturePacketStrictAuditHandoff", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ], + }, + ocuCandidatePairingPreflight: "ocu-candidate-pairing-preflight-smoke", + }, + sourceRepoBaselineAudit: { + usableBaseline: "make record-and-replay-baseline-audit", + strictOfficialGoldenGate: "make record-and-replay-official-golden-gate-audit", + auditTargetDryRunSmoke: "make record-and-replay-baseline-audit-targets-smoke", + baselineSummaryArtifact: "dist/record-and-replay-baseline-summary.json", + copiedBaselineSummaryEvidence: "evidence/source-baseline-summary.json", + strictOfficialGoldenSummaryArtifact: "dist/record-and-replay-official-golden-gate-summary.json", + baselineSummaryEnvVar: "RNR_BASELINE_SUMMARY_JSON", + strictOfficialGoldenSummaryEnvVar: "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON", + strictOfficialGoldenExpectedFailureAudit: "scripts/check-record-and-replay-baseline-summary.py dist/record-and-replay-official-golden-gate-summary.json --allow-strict-official-golden-missing", + verifiesSummaryArtifactSeparation: true, + verifiesSummaryEnvVarIsolation: true, + }, + standaloneRepoBoundary: { + defaultChecksDoNotStartOfficialRecording: true, + preflightScriptsRemainInOpenComputerUseRepo: true, + doesNotCopyOpenComputerUseRuntimeSource: true, + }, + hasSuccessfulRecordingGolden: sourceHasSuccessfulRecordingGolden, + requiredSuccessfulRecordingScenarios: [ + "simple-action-stop", + ], + recommendedSuccessfulRecordingScenarios: [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", + ], + scenarioRecipes: { + "simple-action-stop": { + scenario: "simple-action-stop", + priority: "required", + captureGoal: "Record one low-risk left click and finish with the Record & Replay Done control.", + userAction: "Click a stable, harmless UI target once, then click Done in the official recording controls.", + expectedActionEvents: ["mouse.click"], + expectedEndReason: "recording_controls_stopped", + expectedEvidence: [ + "event_stream_start/status/stop hosted MCP response shape", + "metadata/session/events/suppressed handoff paths", + "one session.started as the first event", + "one session.ended as the final event", + "mouse.click action event", + "AX payload for the clicked target/window", + ], + ocuCandidateSourceKind: "run-action-smoke", + notes: [ + "Use this as the minimum official successful recording golden fixture.", + "Keep the clicked target harmless and easy to describe after redaction.", + ], + }, + "keyboard-input-stop": { + scenario: "keyboard-input-stop", + priority: "recommended", + captureGoal: "Record a small text input and finish with the Record & Replay Done control.", + userAction: "Type a short non-sensitive string into a harmless text field, then click Done.", + expectedActionEvents: ["keyboard.text_input"], + expectedEndReason: "recording_controls_stopped", + expectedEvidence: [ + "keyboard.text_input action event", + "focusedAccessibilityElement or equivalent keyboard target evidence", + "AX payload for the edited field/window", + "completed hosted MCP response shape", + ], + ocuCandidateSourceKind: "recording-required", + notes: [ + "Do not use synthetic --run-action-smoke for this scenario; macOS may filter synthetic keyboard events.", + "Use non-sensitive placeholder text only.", + ], + }, + "drag-stop": { + scenario: "drag-stop", + priority: "recommended", + captureGoal: "Record one low-risk drag gesture and finish with the Record & Replay Done control.", + userAction: "Drag a harmless target a short distance, release it, then click Done.", + expectedActionEvents: ["mouse.drag"], + expectedEndReason: "recording_controls_stopped", + expectedEvidence: [ + "mouse.drag action event", + "start/end location evidence", + "targetAccessibilityElement or equivalent drag target evidence", + "AX payload for the affected window", + "completed hosted MCP response shape", + ], + ocuCandidateSourceKind: "run-action-smoke", + notes: [ + "Use a fixture or harmless UI target where drag has no external side effects.", + ], + }, + cancel: { + scenario: "cancel", + priority: "recommended", + captureGoal: "Record the official cancellation path.", + userAction: "Start recording, optionally perform no action, then click Discard/Cancel in the recording controls.", + expectedActionEvents: [], + expectedEndReason: "recording_controls_cancelled", + expectedEvidence: [ + "session.ended endReason=recording_controls_cancelled", + "cancelled hosted MCP/status response shape when available", + "no skill creation from the cancelled recording", + ], + ocuCandidateSourceKind: "recording-required", + notes: [ + "Cancelled recordings are evidence for lifecycle semantics only and must not be used to scaffold a skill.", + ], + }, + timeout: { + scenario: "timeout", + priority: "recommended", + captureGoal: "Record the official maximum-duration timeout path.", + userAction: "Start recording and let the official time limit expire without using Done or Discard.", + expectedActionEvents: [], + expectedEndReason: null, + expectedEvidence: [ + "one session.started as the first event", + "one session.ended as the final event", + "official timeout endReason once observed", + "completed or timeout hosted MCP/status response shape", + ], + ocuCandidateSourceKind: "recording-required", + notes: [ + "OCU currently uses recording_time_limit_reached as a baseline; official timeout endReason still needs calibration.", + "This sample may be slow because the official limit is expected to be long.", + ], + }, + }, + successfulRecordingGoldenRequiredFor: [ + "session file schema equivalence", + "event field schema equivalence", + "AX compact diff algorithm equivalence", + "screenshot trigger equivalence", + "timeout endReason equivalence", + ], + }; + if (JSON.stringify(manifest.officialEvidence) !== JSON.stringify(expectedOfficialEvidence)) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + const expectedReadmeScenarioList = [ + "The minimum required official successful recording scenario is", + "`simple-action-stop`. The recommended calibration set is", + "`simple-action-stop`, `keyboard-input-stop`, `drag-stop`, `cancel`, and `timeout`.", + ].join("\n"); + if (!readme.includes(expectedReadmeScenarioList)) { + throw new Error( + [ + "Generated standalone README scenario list drifted.", + expectedReadmeScenarioList, + readme, + ].join("\n"), + ); + } + if (manifest.extensionLayer?.waitNotify?.callbackFailureMakesCliFail !== true) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.extensionLayer?.waitNotify?.callbackTimeoutMakesCliFail !== true) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if ( + !manifest.extensionLayer?.waitNotify?.environmentVariables?.includes( + "OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH", + ) + ) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.checks?.skillWorkflow !== "scripts/verify-skill-workflow.py") { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.checks?.manifestContract !== "scripts/verify-manifest.py") { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.checks?.packageArtifact !== "scripts/verify-package-artifact.py") { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.checks?.sourceBaselineSummaryEvidence !== "scripts/verify-source-baseline-summary.py") { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.checks?.readmeHandoffContract !== "scripts/verify-readme-handoff.py") { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (manifest.mcpServer?.rejectedRequestsDoNotCreateSessionFiles !== true) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + const recordingToSkill = manifest.recordingToSkill; + const strictValidation = recordingToSkill?.strictValidation; + if ( + strictValidation?.command !== + "event-stream validate --json --strict-ocu --require-skill-draft " || + strictValidation?.requiresMetadataSessionAlias !== true || + strictValidation?.requiresScreenshotPathsInsideSession !== true || + strictValidation?.requiresSkillDraftReady !== true || + JSON.stringify(strictValidation?.requiresDeclaredHandoffPaths) !== + JSON.stringify(["metadataPath", "sessionPath", "eventsPath", "suppressedEventsPath"]) + ) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + const eventsOnlyValidation = recordingToSkill?.eventsOnlyValidation; + if ( + eventsOnlyValidation?.command !== + "event-stream validate --json --require-skill-draft " || + eventsOnlyValidation?.provesMetadataSessionAlias !== false || + eventsOnlyValidation?.provesDeclaredHandoffPaths !== false + ) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + const scaffoldSkill = recordingToSkill?.scaffoldSkill; + if ( + scaffoldSkill?.command !== + "event-stream scaffold-skill --json " || + scaffoldSkill?.runsSkillDraftValidationGate !== true + ) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + if (recordingToSkill?.rejectsCancelledRecordings !== true) { + throw new Error(JSON.stringify(manifest, null, 2)); + } + + const manifestCheck = run("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + const manifestPayload = JSON.parse(manifestCheck.stdout); + if ( + manifestPayload.ok !== true || + manifestPayload.checkedManifestContract !== true || + manifestPayload.checkedStrictExpectedFailureAudit !== true || + manifestPayload.checkedOfficialFixtureSetComparePolicy !== true || + manifestPayload.checkedSourceBaselineSummaryEvidenceCheck !== true + ) { + throw new Error(manifestCheck.stdout); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineAudit.strictOfficialGoldenExpectedFailureAudit = "missing"; + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("strict expected-failure audit command drifted")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + + const sourceSummaryCheck = run("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + const sourceSummaryPayload = JSON.parse(sourceSummaryCheck.stdout); + if ( + sourceSummaryPayload.ok !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryEvidence !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryOfficialGoldenState !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryOfficialGoldenGap !== !sourceHasSuccessfulRecordingGolden || + sourceSummaryPayload.checkedSourceBaselineSummaryOfficialGoldenComplete !== sourceHasSuccessfulRecordingGolden || + sourceSummaryPayload.checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketWorkflowVerifier !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff !== true || + sourceSummaryPayload.checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff !== true + ) { + throw new Error(sourceSummaryCheck.stdout); + } + if ( + sourceSummary.status?.usableBaseline !== true || + sourceSummary.status?.standaloneRepoBaselineReady !== true || + sourceSummary.status?.officialSuccessfulRecordingGoldenComplete !== sourceHasSuccessfulRecordingGolden || + sourceSummary.evidence?.fixtureIngestPipelines?.checkedOfficialSessionDirectoryPathHandoff !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketSetContractManifest !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketPostCaptureWorkflow !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketWorkflowVerifier !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketSetPostCaptureWorkflow !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketSetWorkflowVerifier !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketStrictAuditHandoff !== true || + sourceSummary.evidence?.preflightPipelines?.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff !== true || + sourceSummary.evidence?.standaloneSkillRepo?.checkedOfficialFixtureSetComparePolicyManifest !== true || + sourceSummary.evidence?.npmStagedSkillRepo?.checkedOfficialFixtureSetComparePolicyManifest !== true + ) { + throw new Error(JSON.stringify(sourceSummary, null, 2)); + } + if (!sourceHasSuccessfulRecordingGolden) { + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.status.officialGoldenGatePassed = true; + mutatedSourceSummary.status.officialSuccessfulRecordingGoldenComplete = true; + mutatedSourceSummary.status.officialSuccessfulRecordingEquivalenceReady = true; + mutatedSourceSummary.status.requiresOfficialGoldenCapture = false; + mutatedSourceSummary.status.missingRequiredOfficialSuccessfulRecordingScenarios = []; + mutatedSourceSummary.status.notReadyRequiredOfficialSuccessfulRecordingScenarios = []; + mutatedSourceSummary.status.officialFixtureCoverageErrors = []; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const completedSourceSummary = run("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + const completedPayload = JSON.parse(completedSourceSummary.stdout); + if ( + completedPayload.checkedSourceBaselineSummaryOfficialGoldenState !== true || + completedPayload.checkedSourceBaselineSummaryOfficialGoldenGap !== false || + completedPayload.checkedSourceBaselineSummaryOfficialGoldenComplete !== true + ) { + throw new Error(completedSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.status.officialGoldenGatePassed = true; + mutatedSourceSummary.status.officialSuccessfulRecordingGoldenComplete = true; + mutatedSourceSummary.status.officialSuccessfulRecordingEquivalenceReady = true; + mutatedSourceSummary.status.requiresOfficialGoldenCapture = false; + mutatedSourceSummary.status.missingRequiredOfficialSuccessfulRecordingScenarios = ["simple-action-stop"]; + mutatedSourceSummary.status.notReadyRequiredOfficialSuccessfulRecordingScenarios = []; + mutatedSourceSummary.status.officialFixtureCoverageErrors = []; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if ( + !invalidSourceSummary.stderr.includes( + "official golden state must be either current required gap or completed equivalence", + ) + ) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("official sessionDirectoryPath handoff evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketSetContractManifest = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet set contract manifest evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet post-capture workflow evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet set post-capture workflow evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketWorkflowVerifier = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet workflow verifier evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketSetWorkflowVerifier = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet set workflow verifier evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet strict audit handoff evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedSourceSummary = JSON.parse(JSON.stringify(sourceSummary)); + mutatedSourceSummary.evidence.preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff = false; + writeJSON(sourceSummaryPath, mutatedSourceSummary); + const invalidSourceSummary = runAllowFailure("python3", [verifySourceSummaryPath], { + cwd: generatedRepo, + }); + if (invalidSourceSummary.status === 0) { + throw new Error(invalidSourceSummary.stdout); + } + if (!invalidSourceSummary.stderr.includes("capture packet strict expected-failure audit handoff evidence missing")) { + throw new Error(invalidSourceSummary.stderr || invalidSourceSummary.stdout); + } + } finally { + writeJSON(sourceSummaryPath, sourceSummary); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialFixtureSetGate + .sameScenarioComparePolicy.requiresSameSuppressedSchema = false; + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official fixture set suppressed schema policy missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketSetContractManifest", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet set contract manifest evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketPostCaptureWorkflow", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet post-capture workflow evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketWorkflowVerifier", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet workflow verifier evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketSetPostCaptureWorkflow", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet set post-capture workflow evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketSetWorkflowVerifier", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet set workflow verifier evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketStrictAuditHandoff", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet strict audit handoff evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + try { + const mutatedManifest = JSON.parse(JSON.stringify(manifest)); + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence = + mutatedManifest.officialEvidence.sourceRepoBaselineChecks.officialGoldenCapturePreflight.requiredEvidence.filter( + (value) => value !== "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ); + writeJSON(manifestPath, mutatedManifest); + const invalidManifest = runAllowFailure("python3", [verifyManifestPath], { + cwd: generatedRepo, + }); + if (invalidManifest.status === 0) { + throw new Error(invalidManifest.stdout); + } + if (!invalidManifest.stderr.includes("official capture packet strict expected-failure audit handoff evidence missing")) { + throw new Error(invalidManifest.stderr || invalidManifest.stdout); + } + } finally { + writeJSON(manifestPath, manifest); + } + + const readmeCheck = run("python3", [verifyReadmePath], { + cwd: generatedRepo, + }); + const readmePayload = JSON.parse(readmeCheck.stdout); + if ( + readmePayload.ok !== true || + readmePayload.checkedReadmeHandoffContract !== true || + readmePayload.checkedReadmeOfficialEvidenceHandoff !== true || + readmePayload.checkedReadmeOfficialGoldenGap !== true || + readmePayload.checkedReadmeWaitNotifyBoundary !== true + ) { + throw new Error(readmeCheck.stdout); + } + try { + writeFileSync( + path.join(generatedRepo, "README.md"), + readme.replace( + "make record-and-replay-official-golden-gate-audit", + "missing-strict-gate", + ), + ); + const invalidReadme = runAllowFailure("python3", [verifyReadmePath], { + cwd: generatedRepo, + }); + if (invalidReadme.status === 0) { + throw new Error(invalidReadme.stdout); + } + if (!invalidReadme.stderr.includes("missingRequiredSnippets")) { + throw new Error(invalidReadme.stderr || invalidReadme.stdout); + } + } finally { + writeFileSync(path.join(generatedRepo, "README.md"), readme); + } + + const workflow = run(path.join(generatedRepo, "scripts", "verify-skill-workflow.py"), [], { + cwd: generatedRepo, + }); + const workflowPayload = JSON.parse(workflow.stdout); + if (workflowPayload.ok !== true || workflowPayload.checkedSkillWorkflow !== true) { + throw new Error(workflow.stdout); + } + + const check = run(path.join(generatedRepo, "scripts", "check.sh"), [], { + cwd: generatedRepo, + env: { + ...process.env, + OPEN_COMPUTER_USE_CLI: launcher, + OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY: "1", + }, + }); + const checkPayloads = parseJSONObjectLines(check.stdout); + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedStrictValidation === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedEventsOnlyValidation === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedScaffoldSkill === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedNoActiveStatusStop === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedInitializeSurfaceContract === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedToolMetadataContract === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedToolInputSchemaNoArguments === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRequiresObjectParams === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRequiresStringToolName === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRequiresObjectArguments === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRejectsUnexpectedArguments === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRejectsNonObjectArguments === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedRejectedRequestsDoNotCreateSessionFiles === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedManifestContract === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedSourceBaselineSummaryEvidence === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedSourceBaselineSummaryOfficialGoldenState === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedReadmeHandoffContract === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedSkillWorkflow === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedStatusNotUsedAsWaitLoopGuard === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedMcpNoDirectEventContentsGuard === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedWaitNotifyContract === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedNotifyCallbackFailureExit === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedNotifyCallbackFailureReason === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedNotifyCallbackTimeoutFailureExit === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedNotifyCallbackTimeoutReason === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedSkillCreatorHandoff === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedDeclaredHandoffPaths === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedScreenshotPathContainment === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedScaffoldSkillFailureExit === true)) { + throw new Error(check.stdout); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedCancelledRecordingRejected === true)) { + throw new Error(check.stdout); + } + const packagedSkill = path.join( + generatedRepo, + "dist", + "skills", + "open-computer-use-record-and-replay.skill", + ); + if (!existsSync(packagedSkill)) { + throw new Error(`generated repo self-check did not package skill: ${packagedSkill}`); + } + if (!checkPayloads.some((payload) => payload.ok === true && payload.checkedPackageArtifact === true)) { + throw new Error(check.stdout); + } + + console.log(JSON.stringify({ + ok: true, + checkedNpmLauncher: true, + checkedNpmPythonLauncherDiagnostics: true, + checkedStandaloneRepoScaffold: true, + checkedManifestContract: true, + checkedReadmeHandoffContract: true, + checkedReadmeOfficialEvidenceHandoff: true, + checkedReadmeOfficialGoldenGap: true, + checkedReadmeWaitNotifyBoundary: true, + checkedGeneratedReadmePrerequisites: true, + checkedGeneratedReadmeScenarioList: true, + checkedOfficialEvidenceManifest: true, + checkedOfficialEvidenceScenarioManifest: true, + checkedOfficialEvidencePreflightManifest: true, + checkedOfficialEvidenceAuditManifest: true, + checkedOfficialFixtureSetComparePolicyManifest: true, + checkedSourceBaselineSummaryEvidence: true, + checkedSourceBaselineSummaryOfficialGoldenState: true, + checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff: true, + checkedSourceBaselineSummaryCapturePacketSetContractManifest: true, + checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow: true, + checkedSourceBaselineSummaryCapturePacketWorkflowVerifier: true, + checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow: true, + checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier: true, + checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff: true, + checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff: true, + checkedPackageArtifact: true, + checkedGeneratedRepoSelfCheck: true, + checkedRuntimeTimeoutDiagnostics: true, + checkedInitializeSurfaceContract: true, + checkedToolMetadataContract: true, + checkedToolInputSchemaNoArguments: true, + checkedNoActiveStatusStop: true, + checkedRequiresObjectParams: true, + checkedRequiresStringToolName: true, + checkedRequiresObjectArguments: true, + checkedRejectsUnexpectedArguments: true, + checkedRejectsNonObjectArguments: true, + checkedRejectedRequestsDoNotCreateSessionFiles: true, + checkedSkillWorkflow: true, + checkedStatusNotUsedAsWaitLoopGuard: true, + checkedMcpNoDirectEventContentsGuard: true, + checkedWaitNotifyContract: true, + checkedNotifySuppressedEventsPathEnv: true, + checkedNotifyCallbackFailureExit: true, + checkedNotifyCallbackFailureReason: true, + checkedNotifyCallbackTimeoutFailureExit: true, + checkedNotifyCallbackTimeoutReason: true, + checkedSkillPackaging: true, + checkedRecordingToSkillManifestContract: true, + checkedCancelledRecordingContract: true, + checkedStrictValidation: true, + checkedDeclaredHandoffPaths: true, + checkedScreenshotPathContainment: true, + checkedEventsOnlyValidation: true, + checkedScaffoldSkill: true, + checkedScaffoldSkillFailureExit: true, + checkedCancelledRecordingRejected: true, + checkedRecordingToSkillHandoff: true, + checkedSkillCreatorHandoff: true, + }, null, 2)); +} finally { + writeFileSync(baselineSummaryArtifact, baselineSummaryOriginal, "utf8"); + rmSync(tmp, { recursive: true, force: true }); +} diff --git a/scripts/test-ocu-record-and-replay-candidate-ingest.py b/scripts/test-ocu-record-and-replay-candidate-ingest.py new file mode 100755 index 00000000..5ac10782 --- /dev/null +++ b/scripts/test-ocu-record-and-replay-candidate-ingest.py @@ -0,0 +1,398 @@ +#!/usr/bin/env python3 + +import json +import importlib.util +import pathlib +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import DEFAULT_REQUIRED_SCENARIOS + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text("\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n") + + +def action_event(event_type: str, session_id: str) -> dict: + if event_type == "mouse.click": + return { + "type": "mouse.click", + "sessionId": session_id, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Private OCU Button", + "value": "private-ocu-value", + "actions": ["AXPress"], + }, + } + if event_type == "keyboard.text_input": + return { + "type": "keyboard.text_input", + "sessionId": session_id, + "textLength": 11, + "focusedAccessibilityElement": { + "role": "AXTextField", + "title": "Private OCU Text Field", + "value": "private-ocu-keyboard-value", + }, + } + if event_type == "mouse.drag": + return { + "type": "mouse.drag", + "sessionId": session_id, + "startLocation": {"x": 10, "y": 20}, + "endLocation": {"x": 80, "y": 120}, + "distance": 122.1, + "durationSeconds": 0.42, + "targetAccessibilityElement": { + "role": "AXSlider", + "title": "Private OCU Drag Target", + "value": "private-ocu-drag-value", + }, + } + raise ValueError(f"unsupported action event type: {event_type}") + + +def make_recording( + root: pathlib.Path, + session_id: str = "ocu-secret-session", + event_type: str = "mouse.click", +) -> dict[str, pathlib.Path]: + recording = root / "recordings" / session_id + recording.mkdir(parents=True) + metadata_path = recording / "metadata.json" + session_path = recording / "session.json" + events_path = recording / "events.jsonl" + suppressed_path = recording / "suppressed.jsonl" + events = [ + {"type": "session.started", "sessionId": session_id}, + { + "type": "AX.focusedWindowChanged", + "sessionId": session_id, + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "renderedText": "Private OCU app content", + "treeLines": ["AXWindow Private OCU Title", "+ AXButton Private OCU Button"], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + }, + }, + action_event(event_type, session_id), + { + "type": "session.ended", + "sessionId": session_id, + "endReason": "recording_controls_stopped", + }, + ] + write_jsonl(events_path, events) + write_jsonl(suppressed_path, []) + metadata = { + "sessionId": session_id, + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": 0, + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "eventsPath": str(events_path), + "suppressedEventsPath": str(suppressed_path), + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + return { + "recording": recording, + "metadata": metadata_path, + "session": session_path, + "events": events_path, + "suppressed": suppressed_path, + "recordingsRoot": recording.parent, + "sessionId": session_id, + } + + +def response_shape(session_id: str, paths: dict[str, pathlib.Path]): + return { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": session_id, + "metadataPath": str(paths["metadata"]), + "sessionPath": str(paths["session"]), + "eventsPath": str(paths["events"]), + "suppressedEventsPath": str(paths["suppressed"]), + } + ), + } + ] + } + } + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + default_required_scenario = DEFAULT_REQUIRED_SCENARIOS[0] + ingest_script = repo / "scripts/ingest-ocu-record-and-replay-candidate.py" + import_script = repo / "scripts/import-event-stream-fixture.py" + spec = importlib.util.spec_from_file_location("ocu_candidate_ingest", ingest_script) + assert spec and spec.loader + ingest_module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(ingest_module) + + assert ingest_module.action_smoke_scenario_for_fixture_scenario("simple-action-stop") == "simple-action-stop" + assert ingest_module.action_smoke_scenario_for_fixture_scenario("drag-stop") == "drag-stop" + assert ingest_module.action_smoke_scenario_for_fixture_scenario("cancel") == "mixed-action-stop" + try: + ingest_module.action_smoke_scenario_for_fixture_scenario("keyboard-input-stop") + except ValueError as error: + assert "does not support --run-action-smoke yet" in str(error) + else: + raise AssertionError("keyboard-input-stop should not map to --run-action-smoke") + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + paths = make_recording(root) + transcript = root / "candidate-transcript.json" + shape = response_shape(paths["sessionId"], paths) + write_json( + transcript, + { + "startResponseShape": shape, + "repeatStartResponseShape": shape, + "statusResponseShape": shape, + "stopResponseShape": shape, + "repeatStopResponseShape": shape, + "finalStatusResponseShape": shape, + }, + ) + + official_root = root / "official-fixtures" + official_import = run( + [ + sys.executable, + str(import_script), + str(paths["recording"]), + "--name", + "official-simple-action", + "--output-dir", + str(official_root), + "--source", + "official", + "--scenario", + "simple-action-stop", + "--official-plugin-version", + "record-and-replay 1.0.857", + "--mcp-transcript", + str(transcript), + ], + repo, + ) + assert official_import.returncode == 0, official_import.stderr + + candidate_root = root / "candidate-fixtures" + direct = run( + [ + sys.executable, + str(ingest_script), + "--recording", + str(paths["recording"]), + "--name", + "ocu-simple-action", + "--output-dir", + str(candidate_root), + "--mcp-transcript", + str(transcript), + "--require-mcp-transcript-evidence", + "--official-root", + str(official_root), + "--check-fixture-set", + ], + repo, + ) + assert direct.returncode == 0, direct.stderr + direct_json = json.loads(direct.stdout) + assert direct_json["ok"] is True + assert direct_json["validation"]["ok"] is True + assert direct_json["readiness"]["ok"] is True + assert direct_json["fixtureSetGate"]["ok"] is True + assert direct_json["usedMcpTranscript"] == str(transcript) + commands = direct_json["commands"] + assert commands["pairingPreflightShell"].startswith( + "python3 scripts/prepare-record-and-replay-ocu-candidate-pairing.py" + ) + assert "--require-candidate-ready" in commands["pairingPreflight"] + assert str(official_root) in commands["pairingPreflight"] + assert str(candidate_root) in commands["pairingPreflight"] + assert commands["fixtureSetGateShell"].startswith( + "python3 scripts/check-event-stream-official-fixture-set.py" + ) + assert "--require-scenario" in commands["fixtureSetGate"] + assert "simple-action-stop" in commands["fixtureSetGate"] + + fixture = candidate_root / "ocu-simple-action" + manifest = json.loads((fixture / "fixture-manifest.json").read_text()) + assert manifest["source"] == "ocu" + assert manifest["scenario"] == default_required_scenario + assert manifest["files"]["mcpTranscript"] == "mcp-transcript.json" + combined = "\n".join(path.read_text() for path in fixture.glob("*") if path.is_file()) + for secret in [ + paths["sessionId"], + "Private OCU app content", + "Private OCU Title", + "Private OCU Button", + "Private OCU Text Field", + "Private OCU Drag Target", + "private-ocu-value", + "private-ocu-keyboard-value", + "private-ocu-drag-value", + str(root), + ]: + assert secret not in combined, secret + + scenario_cases = [ + ("keyboard-input-stop", "keyboard.text_input", "ocu-keyboard-input"), + ("drag-stop", "mouse.drag", "ocu-drag"), + ] + for scenario, event_type, name in scenario_cases: + scenario_paths = make_recording( + root, + session_id=f"{name}-secret-session", + event_type=event_type, + ) + scenario_transcript = root / f"{name}-transcript.json" + scenario_shape = response_shape(scenario_paths["sessionId"], scenario_paths) + write_json( + scenario_transcript, + { + "startResponseShape": scenario_shape, + "repeatStartResponseShape": scenario_shape, + "statusResponseShape": scenario_shape, + "stopResponseShape": scenario_shape, + "repeatStopResponseShape": scenario_shape, + "finalStatusResponseShape": scenario_shape, + }, + ) + scenario_result = run( + [ + sys.executable, + str(ingest_script), + "--recording", + str(scenario_paths["recording"]), + "--name", + name, + "--scenario", + scenario, + "--output-dir", + str(candidate_root), + "--mcp-transcript", + str(scenario_transcript), + "--require-mcp-transcript-evidence", + ], + repo, + ) + assert scenario_result.returncode == 0, scenario_result.stderr + scenario_json = json.loads(scenario_result.stdout) + assert scenario_json["ok"] is True + assert scenario_json["validation"]["ok"] is True + assert scenario_json["readiness"]["ok"] is True + assert scenario_json["scenario"] == scenario + event_counts = scenario_json["readiness"]["recordings"][0]["eventTypes"] + assert event_counts[event_type] == 1 + + smoke_json = root / "action-smoke.jsonl" + write_jsonl( + smoke_json, + [ + {"mode": "ignored", "ok": True}, + { + "mode": "actions", + "ok": True, + "sessionId": paths["sessionId"], + "recordingsRoot": str(paths["recordingsRoot"]), + "mcpTranscriptPath": str(transcript), + "eventTypes": ["session.started", "AX.focusedWindowChanged", "mouse.click", "session.ended"], + }, + ], + ) + from_smoke = run( + [ + sys.executable, + str(ingest_script), + "--smoke-json", + str(smoke_json), + "--name", + "ocu-simple-action-from-smoke", + "--scenario", + "simple-action-stop", + "--output-dir", + str(candidate_root), + "--require-mcp-transcript-evidence", + ], + repo, + ) + assert from_smoke.returncode == 0, from_smoke.stderr + from_smoke_json = json.loads(from_smoke.stdout) + assert from_smoke_json["ok"] is True + assert from_smoke_json["smokeRecord"]["mode"] == "actions" + assert from_smoke_json["recordingInput"] == str(paths["recording"]) + assert from_smoke_json["usedMcpTranscript"] == str(transcript) + + missing_mode = run( + [ + sys.executable, + str(ingest_script), + "--smoke-json", + str(smoke_json), + "--smoke-mode", + "missing", + "--name", + "missing-mode", + "--output-dir", + str(candidate_root), + ], + repo, + ) + assert missing_mode.returncode == 1 + missing_mode_json = json.loads(missing_mode.stderr) + assert "smoke JSONL has no record with mode='missing'" in missing_mode_json["errors"][0] + + print( + json.dumps( + { + "ok": True, + "checkedOcuCandidateIngest": True, + "checkedCandidateIngestHandoffCommands": True, + "checkedSmokeJsonImport": True, + "checkedOfficialCandidatePairing": True, + "checkedCandidateRedaction": True, + "checkedKeyboardInputScenarioReadiness": True, + "checkedDragScenarioReadiness": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-official-record-and-replay-fixture-ingest.py b/scripts/test-official-record-and-replay-fixture-ingest.py new file mode 100755 index 00000000..666901e8 --- /dev/null +++ b/scripts/test-official-record-and-replay-fixture-ingest.py @@ -0,0 +1,751 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import DEFAULT_REQUIRED_SCENARIOS + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text("\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n") + + +def action_event(event_type: str, session_id: str) -> dict: + if event_type == "mouse.click": + return { + "type": "mouse.click", + "sessionId": session_id, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Private Button", + "value": "private-value", + }, + } + if event_type == "keyboard.text_input": + return { + "type": "keyboard.text_input", + "sessionId": session_id, + "textLength": 4, + "focusedAccessibilityElement": { + "role": "AXTextField", + "title": "Private Field", + "value": "private-keyboard-value", + }, + } + if event_type == "mouse.drag": + return { + "type": "mouse.drag", + "sessionId": session_id, + "button": "left", + "startLocation": {"x": 10, "y": 20}, + "endLocation": {"x": 80, "y": 120}, + "distance": 122.07, + "durationMs": 400, + "targetAccessibilityElement": { + "role": "AXSlider", + "title": "Private Drag Target", + "value": "private-drag-value", + }, + } + raise ValueError(f"unsupported action event type: {event_type}") + + +def make_recording(root: pathlib.Path, event_type: str = "mouse.click") -> dict[str, pathlib.Path]: + recording = root / "official-recording" + recording.mkdir(parents=True) + metadata_path = recording / "metadata.json" + session_path = recording / "session.json" + events_path = recording / "events.jsonl" + suppressed_path = recording / "suppressed.jsonl" + events = [ + {"type": "session.started", "sessionId": "official-secret-session"}, + { + "type": "AX.focusedWindowChanged", + "sessionId": "official-secret-session", + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "renderedText": "Private app content", + "treeLines": ["AXWindow Private Title", "+ AXButton Private Button"], + "screenshotNeededForContext": False, + "screenshotAvailable": False, + }, + }, + action_event(event_type, "official-secret-session"), + { + "type": "session.ended", + "sessionId": "official-secret-session", + "endReason": "recording_controls_stopped", + }, + ] + write_jsonl(events_path, events) + write_jsonl(suppressed_path, []) + metadata = { + "sessionId": "official-secret-session", + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": 0, + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "eventsPath": str(events_path), + "suppressedEventsPath": str(suppressed_path), + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + return { + "recording": recording, + "metadata": metadata_path, + "session": session_path, + "events": events_path, + "suppressed": suppressed_path, + } + + +def response_shape(session_id: str, paths: dict[str, pathlib.Path]): + return { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionId": session_id, + "metadataPath": str(paths["metadata"]), + "sessionPath": str(paths["session"]), + "eventsPath": str(paths["events"]), + "suppressedEventsPath": str(paths["suppressed"]), + } + ), + } + ] + } + } + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + default_required_scenario = DEFAULT_REQUIRED_SCENARIOS[0] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + paths = make_recording(root) + status_json = root / "official-stop-response.json" + write_json( + status_json, + { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": str(paths["metadata"]), + "eventsPath": str(paths["events"]), + } + ), + } + ] + } + }, + ) + transcript = root / "official-transcript.json" + shape = response_shape("official-secret-session", paths) + write_json( + transcript, + { + "startResponseShape": shape, + "repeatStartResponseShape": shape, + "statusResponseShape": shape, + "stopResponseShape": shape, + "repeatStopResponseShape": shape, + "finalStatusResponseShape": shape, + }, + ) + inspect_output_dir = root / "inspect-only-fixtures" + inspect_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(status_json), + "--name", + "official-simple-action-inspect", + "--scenario", + "simple-action-stop", + "--output-dir", + str(inspect_output_dir), + "--mcp-transcript", + str(transcript), + "--require-mcp-transcript-evidence", + "--inspect-only", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert inspect_completed.returncode == 0, inspect_completed.stderr + inspect_payload = json.loads(inspect_completed.stdout) + assert inspect_payload["ok"] is True + assert inspect_payload["stage"] == "inspect" + assert inspect_payload["wouldImport"] is False + assert inspect_payload["recordingInput"] == str(paths["metadata"]) + assert inspect_payload["recordingInputInspection"]["metadataPath"]["exists"] is True + assert inspect_payload["recordingInputInspection"]["sessionPath"]["exists"] is True + assert inspect_payload["recordingInputInspection"]["eventsPath"]["exists"] is True + assert inspect_payload["mcpTranscriptInspection"]["looksLikeMcpTranscript"] is True + assert not inspect_output_dir.exists() + + inspect_missing_transcript = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(status_json), + "--name", + "official-simple-action-inspect-missing-transcript", + "--scenario", + "simple-action-stop", + "--output-dir", + str(root / "inspect-missing-transcript-fixtures"), + "--require-mcp-transcript-evidence", + "--inspect-only", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert inspect_missing_transcript.returncode == 1 + inspect_missing_transcript_payload = json.loads(inspect_missing_transcript.stderr) + assert inspect_missing_transcript_payload["stage"] == "inspect" + assert inspect_missing_transcript_payload["recordingInputInspection"]["exists"] is True + assert inspect_missing_transcript_payload["mcpTranscriptInspection"][ + "looksLikeMcpTranscript" + ] is False + + output_dir = root / "fixtures" + completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(status_json), + "--name", + "official-simple-action", + "--output-dir", + str(output_dir), + "--mcp-transcript", + str(transcript), + "--require-mcp-transcript-evidence", + "--check-fixture-set", + "--check-coverage", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert completed.returncode == 0, completed.stderr + payload = json.loads(completed.stdout) + assert payload["ok"] is True + assert payload["scenario"] == default_required_scenario + assert payload["readiness"]["ok"] is True + assert payload["fixtureSetGate"]["ok"] is True + assert payload["coverage"]["coverageOk"] is True + assert payload["coverage"]["scenarioCoverageOk"] is True + assert payload["coverage"]["hasRequiredOfficialSuccessfulFixture"] is True + assert payload["coverage"]["requiredOfficialReadinessChecked"] is True + assert payload["coverage"]["requiredOfficialReadinessOk"] is True + assert payload["coverage"]["officialFixtureSetReadiness"]["ok"] is True + assert payload["coverage"]["missingOfficialScenarios"] == [] + assert payload["extractedPaths"]["metadataPath"] == str(paths["metadata"]) + assert payload["extractedPaths"]["eventsPath"] == str(paths["events"]) + + scenario_cases = [ + ("keyboard-input-stop", "keyboard.text_input", "official-keyboard-input"), + ("drag-stop", "mouse.drag", "official-drag"), + ] + for scenario, event_type, name in scenario_cases: + scenario_paths = make_recording(root / f"{name}-recording", event_type=event_type) + scenario_status = root / f"{name}-response.json" + write_json( + scenario_status, + { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": str(scenario_paths["metadata"]), + "eventsPath": str(scenario_paths["events"]), + } + ), + } + ] + } + }, + ) + scenario_transcript = root / f"{name}-transcript.json" + scenario_shape = response_shape("official-secret-session", scenario_paths) + write_json( + scenario_transcript, + { + "startResponseShape": scenario_shape, + "repeatStartResponseShape": scenario_shape, + "statusResponseShape": scenario_shape, + "stopResponseShape": scenario_shape, + "repeatStopResponseShape": scenario_shape, + "finalStatusResponseShape": scenario_shape, + }, + ) + scenario_output_dir = root / f"{name}-fixtures" + scenario_result = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(scenario_status), + "--name", + name, + "--scenario", + scenario, + "--output-dir", + str(scenario_output_dir), + "--mcp-transcript", + str(scenario_transcript), + "--require-mcp-transcript-evidence", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert scenario_result.returncode == 0, scenario_result.stderr + scenario_payload = json.loads(scenario_result.stdout) + assert scenario_payload["ok"] is True + assert scenario_payload["scenario"] == scenario + assert scenario_payload["readiness"]["ok"] is True + event_counts = scenario_payload["readiness"]["recordings"][0]["eventTypes"] + assert event_counts[event_type] == 1 + response_shapes = scenario_payload["readiness"]["recordings"][0]["mcpResponseShapes"] + assert response_shapes["startResponseShape"]["hasResult"] is True + assert response_shapes["statusResponseShape"]["hasResult"] is True + assert response_shapes["stopResponseShape"]["hasResult"] is True + assert response_shapes["finalStatusResponseShape"]["hasResult"] is True + + stdin_status_json = { + **response_shape("official-secret-session", paths), + "startResponseShape": shape, + "repeatStartResponseShape": shape, + "statusResponseShape": shape, + "stopResponseShape": shape, + "repeatStopResponseShape": shape, + "finalStatusResponseShape": shape, + } + inspect_stdin_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + "-", + "--name", + "official-simple-action-inspect-stdin", + "--scenario", + "simple-action-stop", + "--output-dir", + str(root / "inspect-stdin-fixtures"), + "--use-status-json-as-transcript", + "--require-mcp-transcript-evidence", + "--inspect-only", + ], + cwd=repo, + input=json.dumps(stdin_status_json), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert inspect_stdin_completed.returncode == 0, inspect_stdin_completed.stderr + inspect_stdin_payload = json.loads(inspect_stdin_completed.stdout) + assert inspect_stdin_payload["ok"] is True + assert inspect_stdin_payload["usedMcpTranscript"] == "" + assert inspect_stdin_payload["mcpTranscriptInspection"]["looksLikeMcpTranscript"] is True + assert inspect_stdin_payload["mcpTranscriptInspection"][ + "statusJsonLooksLikeMcpTranscript" + ] is True + + stdin_output_dir = root / "stdin-fixtures" + stdin_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + "-", + "--name", + "official-simple-action-stdin", + "--scenario", + "simple-action-stop", + "--output-dir", + str(stdin_output_dir), + "--use-status-json-as-transcript", + "--require-mcp-transcript-evidence", + "--check-fixture-set", + ], + cwd=repo, + input=json.dumps(stdin_status_json), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert stdin_completed.returncode == 0, stdin_completed.stderr + stdin_payload = json.loads(stdin_completed.stdout) + assert stdin_payload["ok"] is True + assert stdin_payload["usedMcpTranscript"] == "" + assert stdin_payload["readiness"]["ok"] is True + assert stdin_payload["fixtureSetGate"]["ok"] is True + assert stdin_payload["extractedPaths"]["metadataPath"] == str(paths["metadata"]) + assert stdin_payload["extractedPaths"]["sessionPath"] == str(paths["session"]) + + relative_status_json = { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": "official-recording/metadata.json", + "eventsPath": "official-recording/events.jsonl", + } + ), + } + ] + } + } + relative_output_dir = root / "relative-fixtures" + relative_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + "-", + "--status-json-base-dir", + str(root), + "--name", + "official-simple-action-relative", + "--scenario", + "simple-action-stop", + "--output-dir", + str(relative_output_dir), + ], + cwd=repo, + input=json.dumps(relative_status_json), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert relative_completed.returncode == 0, relative_completed.stderr + relative_payload = json.loads(relative_completed.stdout) + assert relative_payload["ok"] is True + assert relative_payload["statusJsonBaseDir"] == str(root) + assert relative_payload["recordingInput"] == str(paths["metadata"]) + assert relative_payload["extractedPaths"]["metadataPath"] == "official-recording/metadata.json" + + minimal_recording = root / "official-minimal-recording" + minimal_recording.mkdir() + minimal_events = minimal_recording / "events.jsonl" + minimal_session = minimal_recording / "session.json" + minimal_session_id = "official-minimal-secret-session" + write_jsonl( + minimal_events, + [ + {"kind": "session.started", "sessionId": minimal_session_id}, + { + "kind": "AX.focusedWindowChanged", + "sessionId": minimal_session_id, + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "fullTree": ["AXWindow Private Minimal Title"], + }, + }, + { + "kind": "mouse.click", + "sessionId": minimal_session_id, + "button": "left", + "location": {"x": 12, "y": 34}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Private Minimal Button", + }, + }, + { + "kind": "session.ended", + "sessionId": minimal_session_id, + "endReason": "recording_controls_stopped", + }, + ], + ) + write_json( + minimal_session, + { + "id": minimal_session_id, + "startedAt": "2026-06-29T00:00:00Z", + "endedAt": "2026-06-29T00:00:03Z", + "endReason": "recording_controls_stopped", + "eventsPath": str(minimal_events), + }, + ) + minimal_status = root / "official-minimal-session-dir-response.json" + write_json( + minimal_status, + { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "sessionDirectoryPath": str(minimal_recording), + } + ), + } + ] + } + }, + ) + minimal_output_dir = root / "minimal-session-dir-fixtures" + minimal_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(minimal_status), + "--name", + "official-minimal-session-dir", + "--scenario", + "simple-action-stop", + "--output-dir", + str(minimal_output_dir), + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert minimal_completed.returncode == 0, minimal_completed.stderr + minimal_payload = json.loads(minimal_completed.stdout) + assert minimal_payload["ok"] is True + assert minimal_payload["recordingInput"] == str(minimal_recording) + assert minimal_payload["extractedPaths"]["sessionDirectoryPath"] == str(minimal_recording) + assert minimal_payload["readiness"]["ok"] is True + minimal_fixture = pathlib.Path(minimal_payload["fixtureDir"]) + minimal_metadata = json.loads((minimal_fixture / "metadata.json").read_text()) + assert minimal_metadata["sessionId"] == "fixture-session" + assert minimal_metadata["metadataPath"] == "metadata.json" + assert minimal_metadata["sessionPath"] == "session.json" + assert minimal_metadata["suppressedEventsPath"] == "suppressed.jsonl" + + transcript_stdin_output_dir = root / "transcript-stdin-fixtures" + transcript_stdin_completed = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(status_json), + "--name", + "official-simple-action-transcript-stdin", + "--scenario", + "simple-action-stop", + "--output-dir", + str(transcript_stdin_output_dir), + "--mcp-transcript", + "-", + "--require-mcp-transcript-evidence", + "--check-fixture-set", + ], + cwd=repo, + input=transcript.read_text(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert transcript_stdin_completed.returncode == 0, transcript_stdin_completed.stderr + transcript_stdin_payload = json.loads(transcript_stdin_completed.stdout) + assert transcript_stdin_payload["ok"] is True + assert transcript_stdin_payload["usedMcpTranscript"] == "" + assert transcript_stdin_payload["readiness"]["ok"] is True + assert transcript_stdin_payload["fixtureSetGate"]["ok"] is True + + cancel_recording_root = root / "cancel-recording" + cancel_recording_root.mkdir() + cancel_paths = make_recording(cancel_recording_root) + cancel_metadata = json.loads(cancel_paths["metadata"].read_text()) + cancel_metadata["endReason"] = "recording_controls_cancelled" + cancel_events = [ + {"type": "session.started", "sessionId": "official-cancel-session"}, + { + "type": "session.ended", + "sessionId": "official-cancel-session", + "endReason": "recording_controls_cancelled", + }, + ] + cancel_metadata["eventCount"] = len(cancel_events) + write_json(cancel_paths["metadata"], cancel_metadata) + write_json(cancel_paths["session"], cancel_metadata) + write_jsonl(cancel_paths["events"], cancel_events) + cancel_status = root / "official-cancel-response.json" + write_json( + cancel_status, + { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": str(cancel_paths["metadata"]), + "eventsPath": str(cancel_paths["events"]), + } + ), + } + ] + } + }, + ) + cancel_coverage_output_dir = root / "cancel-coverage-fixtures" + cancel_coverage = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(cancel_status), + "--name", + "official-cancel-only", + "--scenario", + "cancel", + "--output-dir", + str(cancel_coverage_output_dir), + "--require-coverage", + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert cancel_coverage.returncode == 1 + cancel_coverage_payload = json.loads(cancel_coverage.stderr) + assert cancel_coverage_payload["ok"] is False + assert cancel_coverage_payload["readiness"]["ok"] is True + assert cancel_coverage_payload["coverage"]["coverageOk"] is False + assert cancel_coverage_payload["coverage"]["scenarioCoverageOk"] is False + assert cancel_coverage_payload["coverage"]["requiredOfficialReadinessChecked"] is True + assert cancel_coverage_payload["coverage"]["requiredOfficialReadinessOk"] is False + assert cancel_coverage_payload["coverage"]["missingOfficialScenarios"] == [ + default_required_scenario + ] + + both_stdin = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + "-", + "--mcp-transcript", + "-", + "--name", + "both-stdin", + "--output-dir", + str(root / "both-stdin-fixtures"), + ], + cwd=repo, + input=json.dumps(stdin_status_json), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert both_stdin.returncode == 1 + both_stdin_json = json.loads(both_stdin.stderr) + assert "cannot read both --status-json and --mcp-transcript from stdin" in both_stdin_json["errors"][0] + + fixture = output_dir / "official-simple-action" + combined = "\n".join(path.read_text() for path in fixture.glob("*") if path.is_file()) + for secret in [ + "official-secret-session", + "Private app content", + "Private Title", + "Private Button", + "private-value", + str(root), + ]: + assert secret not in combined, secret + + missing = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(root / "missing-paths.json"), + "--name", + "missing", + "--output-dir", + str(output_dir), + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert missing.returncode == 1 + + no_paths = root / "no-paths.json" + write_json(no_paths, {"result": {"content": [{"type": "text", "text": "{}"}]}}) + no_paths_result = subprocess.run( + [ + sys.executable, + str(repo / "scripts/ingest-official-record-and-replay-fixture.py"), + "--status-json", + str(no_paths), + "--name", + "no-paths", + "--output-dir", + str(output_dir), + ], + cwd=repo, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert no_paths_result.returncode == 1 + no_paths_json = json.loads(no_paths_result.stderr) + assert "could not find metadataPath" in no_paths_json["errors"][0] + assert "sessionDirectoryPath" in no_paths_json["errors"][0] + + print( + json.dumps( + { + "ok": True, + "checkedOfficialFixtureIngest": True, + "checkedOfficialFixtureInspectOnly": True, + "checkedOfficialSessionDirectoryPathHandoff": True, + "checkedPostIngestCoverageReport": True, + "checkedPostIngestCoverageReadiness": True, + "checkedPostIngestRequireCoverageFailure": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-baseline-audit-make-targets.py b/scripts/test-record-and-replay-baseline-audit-make-targets.py new file mode 100755 index 00000000..e45dc0ab --- /dev/null +++ b/scripts/test-record-and-replay-baseline-audit-make-targets.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import subprocess + + +def run_make(repo: pathlib.Path, *args: str) -> str: + env = os.environ.copy() + env.pop("RNR_BASELINE_SUMMARY_JSON", None) + env.pop("RNR_OFFICIAL_GOLDEN_SUMMARY_JSON", None) + result = subprocess.run( + ["make", "-n", *args], + cwd=repo, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + if result.returncode != 0: + raise AssertionError( + "make dry-run failed\n" + + "stdout:\n" + + result.stdout + + "\nstderr:\n" + + result.stderr + ) + return result.stdout + + +def assert_contains(output: str, expected: str) -> None: + if expected not in output: + raise AssertionError(f"missing {expected!r} in make dry-run output:\n{output}") + + +def main() -> None: + repo = pathlib.Path(__file__).resolve().parents[1] + + default_output = run_make(repo, "record-and-replay-baseline-audit") + assert_contains(default_output, "dist/record-and-replay-baseline-summary.json") + assert_contains( + default_output, + './scripts/run-record-and-replay-baseline-smoke.sh --summary-json "dist/record-and-replay-baseline-summary.json"', + ) + + custom_baseline_output = run_make( + repo, + "record-and-replay-baseline-audit", + "RNR_BASELINE_SUMMARY_JSON=/tmp/ocu-rnr-baseline-summary.json", + ) + assert_contains(custom_baseline_output, "/tmp/ocu-rnr-baseline-summary.json") + assert_contains( + custom_baseline_output, + './scripts/run-record-and-replay-baseline-smoke.sh --summary-json "/tmp/ocu-rnr-baseline-summary.json"', + ) + + baseline_ignores_strict_var_output = run_make( + repo, + "record-and-replay-baseline-audit", + "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/should-not-affect-baseline-summary.json", + ) + assert_contains( + baseline_ignores_strict_var_output, + './scripts/run-record-and-replay-baseline-smoke.sh --summary-json "dist/record-and-replay-baseline-summary.json"', + ) + if "/tmp/should-not-affect-baseline-summary.json" in baseline_ignores_strict_var_output: + raise AssertionError( + "baseline audit must ignore the strict official golden summary variable:\n" + + baseline_ignores_strict_var_output + ) + + strict_default_output = run_make( + repo, + "record-and-replay-official-golden-gate-audit", + ) + assert_contains( + strict_default_output, + "dist/record-and-replay-official-golden-gate-summary.json", + ) + assert_contains( + strict_default_output, + './scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden --summary-json "dist/record-and-replay-official-golden-gate-summary.json"', + ) + if "dist/record-and-replay-baseline-summary.json" in strict_default_output: + raise AssertionError( + "strict official golden audit must not use the baseline summary path by default:\n" + + strict_default_output + ) + + strict_ignores_baseline_var_output = run_make( + repo, + "record-and-replay-official-golden-gate-audit", + "RNR_BASELINE_SUMMARY_JSON=/tmp/should-not-affect-strict-summary.json", + ) + assert_contains( + strict_ignores_baseline_var_output, + './scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden --summary-json "dist/record-and-replay-official-golden-gate-summary.json"', + ) + if "/tmp/should-not-affect-strict-summary.json" in strict_ignores_baseline_var_output: + raise AssertionError( + "strict official golden audit must ignore the baseline summary variable:\n" + + strict_ignores_baseline_var_output + ) + + strict_custom_output = run_make( + repo, + "record-and-replay-official-golden-gate-audit", + "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON=/tmp/ocu-rnr-official-golden-summary.json", + ) + assert_contains(strict_custom_output, "/tmp/ocu-rnr-official-golden-summary.json") + assert_contains( + strict_custom_output, + './scripts/run-record-and-replay-baseline-smoke.sh --require-official-golden --summary-json "/tmp/ocu-rnr-official-golden-summary.json"', + ) + + print( + json.dumps( + { + "ok": True, + "checkedBaselineAuditMakeTarget": True, + "checkedBaselineAuditDefaultSummaryPath": True, + "checkedBaselineAuditCustomSummaryPath": True, + "checkedBaselineAuditIgnoresStrictSummaryVar": True, + "checkedStrictOfficialGoldenAuditMakeTarget": True, + "checkedStrictOfficialGoldenAuditDefaultSummaryPath": True, + "checkedStrictOfficialGoldenAuditCustomSummaryPath": True, + "checkedStrictOfficialGoldenAuditIgnoresBaselineSummaryVar": True, + "checkedStrictOfficialGoldenAuditSeparateSummaryPath": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-baseline-contract.py b/scripts/test-record-and-replay-baseline-contract.py new file mode 100755 index 00000000..7f60280e --- /dev/null +++ b/scripts/test-record-and-replay-baseline-contract.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 + +import json + +from record_and_replay_baseline_contract import ( + NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS, + NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + REQUIRED_BASELINE_CHECKS, + STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS, + STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, +) + + +LIFECYCLE_SUMMARY_RENAMES = { + "checkedOneActive": "checkedLifecycleOneActive", + "checkedIdempotentStop": "checkedLifecycleIdempotentStop", + "checkedFinalStatus": "checkedLifecycleFinalStatus", +} + + +def assert_no_duplicates(label: str, values: tuple[str, ...]) -> None: + seen = set() + duplicates = [] + for value in values: + if value in seen: + duplicates.append(value) + seen.add(value) + assert not duplicates, f"{label} has duplicate keys: {duplicates}" + + +def renamed_standalone_summary_keys() -> tuple[str, ...]: + return tuple( + LIFECYCLE_SUMMARY_RENAMES.get(key, key) + for key in STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS + ) + + +def main() -> None: + constants = { + "REQUIRED_BASELINE_CHECKS": REQUIRED_BASELINE_CHECKS, + "STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS": STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS, + "NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS": NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS, + "STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS": STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + "NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS": NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + } + for label, values in constants.items(): + assert values, f"{label} is empty" + assert_no_duplicates(label, values) + + assert NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS == NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS + assert STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS == renamed_standalone_summary_keys() + + for smoke_key, summary_key in LIFECYCLE_SUMMARY_RENAMES.items(): + assert smoke_key in STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS + assert smoke_key not in STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS + assert summary_key in STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS + assert summary_key not in STANDALONE_SKILL_REPO_SMOKE_REQUIRED_KEYS + + assert set(NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS) == set(NPM_STAGED_SKILL_REPO_SMOKE_REQUIRED_KEYS) + assert set(STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS) == set(renamed_standalone_summary_keys()) + + print( + json.dumps( + { + "ok": True, + "checkedRequiredBaselineChecks": True, + "checkedNoDuplicateRequiredBaselineChecks": True, + "checkedStandaloneSmokeRequiredKeys": True, + "checkedStandaloneSummaryEvidenceKeys": True, + "checkedStandaloneLifecycleSummaryRenames": True, + "checkedNpmStagedSmokeRequiredKeys": True, + "checkedNpmStagedSummaryEvidenceKeys": True, + "checkedNpmStagedSummaryMatchesSmoke": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-baseline-runner-summary-json.py b/scripts/test-record-and-replay-baseline-runner-summary-json.py new file mode 100755 index 00000000..b6000b62 --- /dev/null +++ b/scripts/test-record-and-replay-baseline-runner-summary-json.py @@ -0,0 +1,307 @@ +#!/usr/bin/env python3 + +import importlib.util +import json +import pathlib +import shutil +import subprocess +import sys +import tempfile + + +def load_summary_fixtures(repo: pathlib.Path): + module_path = repo / "scripts/test-record-and-replay-baseline-summary.py" + spec = importlib.util.spec_from_file_location("baseline_summary_fixtures", module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"failed to load {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run(command: list[str], cwd: pathlib.Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def parse_last_json(stdout: str) -> dict: + decoder = json.JSONDecoder() + records = [] + index = 0 + while index < len(stdout): + start = stdout.find("{", index) + if start < 0: + break + try: + payload, end = decoder.raw_decode(stdout[start:]) + except json.JSONDecodeError: + index = start + 1 + continue + if isinstance(payload, dict): + records.append(payload) + index = start + end + assert records, stdout + return records[-1] + + +def read_json(path: pathlib.Path) -> dict: + return json.loads(path.read_text()) + + +def write_json(path: pathlib.Path, value: dict) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path: pathlib.Path, records: list[dict]) -> None: + path.write_text("\n".join(json.dumps(record, sort_keys=True) for record in records) + "\n") + + +def split_action_records(action_path: pathlib.Path, output_dir: pathlib.Path) -> dict[str, pathlib.Path]: + records = [ + json.loads(line) + for line in action_path.read_text().splitlines() + if line.strip() + ] + by_scenario = { + record.get("actionScenario", "mixed-action-stop"): record + for record in records + } + scenario_paths = {} + for scenario in ("mixed-action-stop", "simple-action-stop", "drag-stop"): + if scenario not in by_scenario: + raise AssertionError(f"missing action fixture for {scenario}") + scenario_path = output_dir / f"action-{scenario}.jsonl" + write_jsonl(scenario_path, [by_scenario[scenario]]) + scenario_paths[scenario] = scenario_path + return scenario_paths + + +def make_shell_stub(path: pathlib.Path, body: str) -> None: + path.write_text("#!/usr/bin/env bash\nset -euo pipefail\n" + body) + path.chmod(0o755) + + +def make_cat_stub(path: pathlib.Path, source: pathlib.Path) -> None: + make_shell_stub(path, f"cat {str(source)!r}\n") + + +def build_fake_repo(repo: pathlib.Path, root: pathlib.Path) -> pathlib.Path: + fixture_module = load_summary_fixtures(repo) + evidence_dir = root / "evidence" + paths = fixture_module.write_inputs(evidence_dir, has_official_golden=False) + action_paths = split_action_records(paths["action"], evidence_dir) + + surface_payload = read_json(paths["surface"]) + surface_payload["ok"] = True + write_json(paths["surface"], surface_payload) + + screenshot_records = [ + json.loads(line) + for line in paths["screenshot"].read_text().splitlines() + if line.strip() + ] + screenshot_records[-1]["screenshotContextChecked"] = True + paths["screenshot"].write_text( + "\n".join(json.dumps(record, sort_keys=True) for record in screenshot_records) + + "\n" + ) + + fixture_set_payload = read_json(paths["fixture_set"]) + fixture_set_payload.update( + { + "checkedImportScenarioManifest": True, + "checkedMissingCandidateFailure": True, + "checkedMissingScenarioFailure": True, + } + ) + write_json(paths["fixture_set"], fixture_set_payload) + + fake_repo = root / "fake-repo" + fake_scripts = fake_repo / "scripts" + fake_scripts.mkdir(parents=True) + shutil.copy2( + repo / "scripts/run-record-and-replay-baseline-smoke.sh", + fake_scripts / "run-record-and-replay-baseline-smoke.sh", + ) + shutil.copy2( + repo / "scripts/build-record-and-replay-baseline-summary.py", + fake_scripts / "build-record-and-replay-baseline-summary.py", + ) + shutil.copy2( + repo / "scripts/check-record-and-replay-baseline-summary.py", + fake_scripts / "check-record-and-replay-baseline-summary.py", + ) + shutil.copy2( + repo / "scripts/record_and_replay_baseline_contract.py", + fake_scripts / "record_and_replay_baseline_contract.py", + ) + shutil.copy2( + repo / "scripts/test-record-and-replay-baseline-contract.py", + fake_scripts / "test-record-and-replay-baseline-contract.py", + ) + (fake_scripts / "run-record-and-replay-baseline-smoke.sh").chmod(0o755) + (fake_scripts / "build-record-and-replay-baseline-summary.py").chmod(0o755) + (fake_scripts / "check-record-and-replay-baseline-summary.py").chmod(0o755) + (fake_scripts / "test-record-and-replay-baseline-contract.py").chmod(0o755) + + make_cat_stub(fake_scripts / "run-event-stream-smoke-matrix.sh", paths["matrix"]) + make_shell_stub( + fake_scripts / "run-event-stream-smoke-tests.sh", + "\n".join( + [ + 'if [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_SCREENSHOTS:-}" == "1" ]]; then', + f" cat {str(paths['screenshot'])!r}", + 'elif [[ "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTIONS:-}" == "1" ]]; then', + ' case "${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO:-mixed-action-stop}" in', + ' "mixed-action-stop")', + f" cat {str(action_paths['mixed-action-stop'])!r}", + " ;;", + ' "simple-action-stop")', + f" cat {str(action_paths['simple-action-stop'])!r}", + " ;;", + ' "drag-stop")', + f" cat {str(action_paths['drag-stop'])!r}", + " ;;", + " *)", + ' echo "unexpected action scenario: ${OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_ACTION_SCENARIO}" >&2', + " exit 2", + " ;;", + " esac", + "else", + ' echo "unexpected event-stream smoke mode" >&2', + " exit 2", + "fi", + "", + ] + ), + ) + make_cat_stub(fake_scripts / "compare-event-stream-surface.py", paths["surface"]) + make_cat_stub(fake_scripts / "compare-event-stream-no-active.py", paths["no_active"]) + make_cat_stub(fake_scripts / "test-event-stream-probe-fixtures.py", paths["raw_timeout"]) + make_cat_stub(fake_scripts / "test-event-stream-official-fixture-set.py", paths["fixture_set"]) + make_cat_stub( + fake_scripts / "check-event-stream-official-fixture-coverage.py", + paths["coverage"], + ) + make_cat_stub( + fake_scripts / "test-official-record-and-replay-fixture-ingest.py", + paths["official_ingest"], + ) + make_cat_stub( + fake_scripts / "test-ocu-record-and-replay-candidate-ingest.py", + paths["candidate_ingest"], + ) + make_cat_stub( + fake_scripts / "test-record-and-replay-official-golden-capture-preflight.py", + paths["official_capture_preflight"], + ) + make_cat_stub( + fake_scripts / "test-record-and-replay-ocu-candidate-pairing-preflight.py", + paths["ocu_pairing_preflight"], + ) + make_cat_stub( + fake_scripts / "test-record-and-replay-baseline-audit-make-targets.py", + paths["baseline_audit_targets"], + ) + make_cat_stub( + fake_scripts / "test-record-and-replay-skill-repo-scaffold.py", + paths["standalone"], + ) + (fake_scripts / "test-npm-record-and-replay-skill-repo-scaffold.mjs").write_text( + "import fs from 'node:fs';\n" + f"process.stdout.write(fs.readFileSync({str(paths['npm'])!r}, 'utf8'));\n" + ) + + return fake_repo + + +def assert_summary_file_matches_stdout(result: subprocess.CompletedProcess[str], path: pathlib.Path) -> dict: + assert path.exists(), result.stderr + stdout_summary = parse_last_json(result.stdout) + file_summary = read_json(path) + assert file_summary == stdout_summary + return file_summary + + +def main() -> None: + repo = pathlib.Path(__file__).resolve().parents[1] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + fake_repo = build_fake_repo(repo, root) + runner = fake_repo / "scripts/run-record-and-replay-baseline-smoke.sh" + + default_summary_path = root / "artifacts/default-summary.json" + default_result = run([str(runner), "--summary-json", str(default_summary_path)], fake_repo) + assert default_result.returncode == 0, default_result.stderr + default_summary = assert_summary_file_matches_stdout( + default_result, + default_summary_path, + ) + assert default_summary["ok"] is True + assert default_summary["status"]["usableBaseline"] is True + assert default_summary["status"]["officialGoldenRequirementSatisfied"] is True + assert default_summary["status"]["officialGoldenGatePassed"] is False + assert "checkedAllowsMissingOfficialGoldenButNotEquivalence" in ( + default_result.stderr + ) + + strict_summary_path = root / "artifacts/strict-summary.json" + strict_result = run( + [ + str(runner), + "--require-official-golden", + "--summary-json", + str(strict_summary_path), + ], + fake_repo, + ) + assert strict_result.returncode == 1 + strict_summary = assert_summary_file_matches_stdout( + strict_result, + strict_summary_path, + ) + assert strict_summary["ok"] is False + assert strict_summary["status"]["usableBaseline"] is True + assert strict_summary["status"]["strictOfficialGoldenRequired"] is True + assert strict_summary["status"]["officialGoldenRequirementSatisfied"] is False + assert strict_summary["status"]["officialGoldenGatePassed"] is False + assert "missing=simple-action-stop" in strict_result.stderr + assert "checkedOfficialGoldenGatePassed" in strict_result.stderr + + standalone_evidence_path = root / "evidence/standalone.json" + standalone_evidence = read_json(standalone_evidence_path) + standalone_evidence["checkedNotifyCallbackTimeoutReason"] = False + write_json(standalone_evidence_path, standalone_evidence) + early_gate_summary_path = root / "artifacts/early-gate-summary.json" + early_gate_result = run( + [str(runner), "--summary-json", str(early_gate_summary_path)], + fake_repo, + ) + assert early_gate_result.returncode == 1 + assert not early_gate_summary_path.exists() + assert "expected true JSON keys missing" in early_gate_result.stderr + assert "checkedNotifyCallbackTimeoutReason" in early_gate_result.stderr + + print( + json.dumps( + { + "ok": True, + "checkedRunnerInvokesSummaryAudit": True, + "checkedSummaryJsonMatchesStdout": True, + "checkedSummaryJsonDefaultExitCode": True, + "checkedSummaryJsonStrictFailureExitCode": True, + "checkedStandaloneEarlyEvidenceGate": True, + "checkedActionScenarioEnvPropagation": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-baseline-summary-audit.py b/scripts/test-record-and-replay-baseline-summary-audit.py new file mode 100755 index 00000000..73d94c58 --- /dev/null +++ b/scripts/test-record-and-replay-baseline-summary-audit.py @@ -0,0 +1,888 @@ +#!/usr/bin/env python3 + +import importlib.util +import json +import pathlib +import subprocess +import sys +import tempfile + + +def load_summary_fixtures(repo: pathlib.Path): + module_path = repo / "scripts/test-record-and-replay-baseline-summary.py" + spec = importlib.util.spec_from_file_location("baseline_summary_fixtures", module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"failed to load {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def load_audit_module(repo: pathlib.Path): + module_path = repo / "scripts/check-record-and-replay-baseline-summary.py" + spec = importlib.util.spec_from_file_location("baseline_summary_audit", module_path) + if spec is None or spec.loader is None: + raise RuntimeError(f"failed to load {module_path}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def run(command: list[str], cwd: pathlib.Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def parse_json(stdout: str) -> dict: + return json.loads(stdout) + + +def write_json(path: pathlib.Path, value: dict) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def build_summary( + repo: pathlib.Path, + fixture_module, + root: pathlib.Path, + has_official_golden: bool, + require_official_golden: bool = False, +) -> tuple[pathlib.Path, dict]: + builder = repo / "scripts/build-record-and-replay-baseline-summary.py" + paths = fixture_module.write_inputs(root, has_official_golden=has_official_golden) + command = fixture_module.summary_command(builder, paths) + if require_official_golden: + command.append("--require-official-golden") + result = run(command, repo) + assert result.returncode == 0, result.stderr + summary = fixture_module.parse_last_json(result.stdout) + summary_path = root / "baseline-summary.json" + write_json(summary_path, summary) + return summary_path, summary + + +def main() -> None: + repo = pathlib.Path(__file__).resolve().parents[1] + fixture_module = load_summary_fixtures(repo) + audit_module = load_audit_module(repo) + audit = repo / "scripts/check-record-and-replay-baseline-summary.py" + + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + + default_summary_path, default_summary = build_summary( + repo, + fixture_module, + root / "default", + has_official_golden=False, + ) + assert default_summary["status"]["usableBaseline"] is True + assert default_summary["status"]["officialSuccessfulRecordingGoldenComplete"] is False + assert default_summary["checks"] == list(audit_module.REQUIRED_BASELINE_CHECKS) + assert set(default_summary["checks"]) == set(audit_module.REQUIRED_BASELINE_CHECKS) + default_audit = run([sys.executable, str(audit), str(default_summary_path)], repo) + assert default_audit.returncode == 0, default_audit.stderr + default_audit_json = parse_json(default_audit.stdout) + assert default_audit_json["ok"] is True + assert default_audit_json["checks"]["checkedAllowsMissingOfficialGoldenButNotEquivalence"] is True + + strict_missing_audit = run( + [ + sys.executable, + str(audit), + str(default_summary_path), + "--require-official-golden", + ], + repo, + ) + assert strict_missing_audit.returncode == 1 + strict_missing_json = parse_json(strict_missing_audit.stdout) + assert strict_missing_json["ok"] is False + assert "checkedStrictModeMatchesExpectation" in strict_missing_json["failures"] + assert "checkedOfficialGoldenGatePassed" in strict_missing_json["failures"] + assert "checkedOfficialSuccessfulRecordingEquivalenceReady" in strict_missing_json[ + "failures" + ] + + expected_strict_missing_summary = json.loads(json.dumps(default_summary)) + expected_strict_missing_summary["ok"] = False + expected_strict_missing_summary["status"][ + "strictOfficialGoldenRequired" + ] = True + expected_strict_missing_summary["status"][ + "officialGoldenRequirementSatisfied" + ] = False + expected_strict_missing_path = root / "expected-strict-missing-summary.json" + write_json(expected_strict_missing_path, expected_strict_missing_summary) + expected_strict_missing_audit = run( + [ + sys.executable, + str(audit), + str(expected_strict_missing_path), + "--allow-strict-official-golden-missing", + ], + repo, + ) + assert expected_strict_missing_audit.returncode == 0, ( + expected_strict_missing_audit.stderr + ) + expected_strict_missing_json = parse_json(expected_strict_missing_audit.stdout) + assert expected_strict_missing_json["ok"] is True + assert expected_strict_missing_json["checks"][ + "checkedStrictOfficialGoldenMissingFailure" + ] is True + assert expected_strict_missing_json["checks"][ + "checkedStrictMissingGoldenHasRequiredGap" + ] is True + + default_with_allow_missing_audit = run( + [ + sys.executable, + str(audit), + str(default_summary_path), + "--allow-strict-official-golden-missing", + ], + repo, + ) + assert default_with_allow_missing_audit.returncode == 1 + default_with_allow_missing_json = parse_json(default_with_allow_missing_audit.stdout) + assert "checkedStrictModeMatchesExpectation" in default_with_allow_missing_json[ + "failures" + ] + + unexplained_strict_missing_summary = json.loads( + json.dumps(expected_strict_missing_summary) + ) + unexplained_strict_missing_summary["status"][ + "missingRequiredOfficialSuccessfulRecordingScenarios" + ] = [] + unexplained_strict_missing_summary["status"][ + "notReadyRequiredOfficialSuccessfulRecordingScenarios" + ] = [] + unexplained_strict_missing_path = ( + root / "unexplained-strict-missing-summary.json" + ) + write_json(unexplained_strict_missing_path, unexplained_strict_missing_summary) + unexplained_strict_missing_audit = run( + [ + sys.executable, + str(audit), + str(unexplained_strict_missing_path), + "--allow-strict-official-golden-missing", + ], + repo, + ) + assert unexplained_strict_missing_audit.returncode == 1 + unexplained_strict_missing_json = parse_json( + unexplained_strict_missing_audit.stdout + ) + assert "checkedStrictMissingGoldenHasRequiredGap" in ( + unexplained_strict_missing_json["failures"] + ) + + strict_summary_path, strict_summary = build_summary( + repo, + fixture_module, + root / "strict-covered", + has_official_golden=True, + require_official_golden=True, + ) + assert strict_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] is True + strict_audit = run( + [ + sys.executable, + str(audit), + str(strict_summary_path), + "--require-official-golden", + ], + repo, + ) + assert strict_audit.returncode == 0, strict_audit.stderr + assert parse_json(strict_audit.stdout)["ok"] is True + + inconsistent_summary = dict(default_summary) + inconsistent_summary["status"] = dict(default_summary["status"]) + inconsistent_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] = True + inconsistent_path = root / "inconsistent-summary.json" + write_json(inconsistent_path, inconsistent_summary) + inconsistent_audit = run([sys.executable, str(audit), str(inconsistent_path)], repo) + assert inconsistent_audit.returncode == 1 + inconsistent_json = parse_json(inconsistent_audit.stdout) + assert "checkedEquivalenceRequiresGolden" in inconsistent_json["failures"] + assert "checkedEquivalenceReadyMatchesPolicy" in inconsistent_json["failures"] + assert "checkedAllowsMissingOfficialGoldenButNotEquivalence" in inconsistent_json[ + "failures" + ] + + derived_status_summary = json.loads(json.dumps(default_summary)) + derived_status = derived_status_summary["status"] + derived_status["requiresOfficialGoldenCapture"] = False + derived_status["missingRequiredOfficialSuccessfulRecordingScenarios"] = [] + derived_status["notReadyRequiredOfficialSuccessfulRecordingScenarios"] = [] + derived_status["officialFixtureCoverageErrors"] = [] + derived_status["standaloneRepoBaselineReady"] = True + derived_status["usableBaseline"] = False + derived_status_path = root / "derived-status-summary.json" + write_json(derived_status_path, derived_status_summary) + derived_status_audit = run( + [sys.executable, str(audit), str(derived_status_path)], + repo, + ) + assert derived_status_audit.returncode == 1 + derived_status_json = parse_json(derived_status_audit.stdout) + assert "checkedTopLevelOkMatchesPolicy" in derived_status_json["failures"] + assert "checkedUsableBaseline" in derived_status_json["failures"] + assert "checkedStandaloneReadyRequiresUsableBaseline" in derived_status_json[ + "failures" + ] + assert "checkedRequiresOfficialGoldenCaptureConsistency" in derived_status_json[ + "failures" + ] + assert "checkedMissingGoldenHasScenarioGapOrCoverageError" in derived_status_json[ + "failures" + ] + + complete_with_gaps_summary = json.loads(json.dumps(strict_summary)) + complete_with_gaps_summary["status"][ + "missingRequiredOfficialSuccessfulRecordingScenarios" + ] = ["simple-action-stop"] + complete_with_gaps_path = root / "complete-with-gaps-summary.json" + write_json(complete_with_gaps_path, complete_with_gaps_summary) + complete_with_gaps_audit = run( + [sys.executable, str(audit), str(complete_with_gaps_path)], + repo, + ) + assert complete_with_gaps_audit.returncode == 1 + complete_with_gaps_json = parse_json(complete_with_gaps_audit.stdout) + assert "checkedOfficialGoldenCompleteHasNoRequiredGaps" in complete_with_gaps_json[ + "failures" + ] + + incomplete_summary_path, incomplete_summary = build_summary( + repo, + fixture_module, + root / "incomplete-base", + has_official_golden=False, + ) + incomplete_summary["status"]["usableBaseline"] = False + incomplete_summary["status"]["missingUsableBaselineEvidence"] = [ + "standaloneSkillRepo.checkedRuntimeContract" + ] + write_json(incomplete_summary_path, incomplete_summary) + incomplete_audit = run([sys.executable, str(audit), str(incomplete_summary_path)], repo) + assert incomplete_audit.returncode == 1 + incomplete_json = parse_json(incomplete_audit.stdout) + assert "checkedUsableBaseline" in incomplete_json["failures"] + assert "checkedNoMissingUsableBaselineEvidence" in incomplete_json["failures"] + + top_level_mismatch_summary = dict(default_summary) + top_level_mismatch_summary["ok"] = False + top_level_mismatch_path = root / "top-level-mismatch-summary.json" + write_json(top_level_mismatch_path, top_level_mismatch_summary) + top_level_mismatch_audit = run( + [sys.executable, str(audit), str(top_level_mismatch_path)], + repo, + ) + assert top_level_mismatch_audit.returncode == 1 + top_level_mismatch_json = parse_json(top_level_mismatch_audit.stdout) + assert "checkedTopLevelOkMatchesPolicy" in top_level_mismatch_json["failures"] + + baseline_name_mismatch_summary = dict(default_summary) + baseline_name_mismatch_summary["baseline"] = "record-and-replay-copy" + baseline_name_mismatch_path = root / "baseline-name-mismatch-summary.json" + write_json(baseline_name_mismatch_path, baseline_name_mismatch_summary) + baseline_name_mismatch_audit = run( + [sys.executable, str(audit), str(baseline_name_mismatch_path)], + repo, + ) + assert baseline_name_mismatch_audit.returncode == 1 + baseline_name_mismatch_json = parse_json(baseline_name_mismatch_audit.stdout) + assert "checkedBaselineName" in baseline_name_mismatch_json["failures"] + + declared_checks_summary = json.loads(json.dumps(default_summary)) + declared_checks_summary["checks"] = [ + check + for check in declared_checks_summary["checks"] + if check + not in { + "official-no-active-response-compare", + "npm-staged-skill-repo-smoke", + } + ] + declared_checks_path = root / "declared-checks-summary.json" + write_json(declared_checks_path, declared_checks_summary) + declared_checks_audit = run( + [sys.executable, str(audit), str(declared_checks_path)], + repo, + ) + assert declared_checks_audit.returncode == 1 + declared_checks_json = parse_json(declared_checks_audit.stdout) + assert "checkedRequiredBaselineChecksDeclared" in declared_checks_json["failures"] + assert declared_checks_json["declaredChecks"]["missingRequired"] == [ + "npm-staged-skill-repo-smoke", + "official-no-active-response-compare", + ] + assert declared_checks_json["declaredChecks"]["unknown"] == [] + assert declared_checks_json["declaredChecks"]["duplicates"] == [] + + extra_declared_checks_summary = json.loads(json.dumps(default_summary)) + extra_declared_checks_summary["checks"].extend( + [ + "event-stream-smoke-matrix", + "obsolete-record-and-replay-smoke", + ] + ) + extra_declared_checks_path = root / "extra-declared-checks-summary.json" + write_json(extra_declared_checks_path, extra_declared_checks_summary) + extra_declared_checks_audit = run( + [sys.executable, str(audit), str(extra_declared_checks_path)], + repo, + ) + assert extra_declared_checks_audit.returncode == 1 + extra_declared_checks_json = parse_json(extra_declared_checks_audit.stdout) + assert "checkedNoUnknownBaselineChecksDeclared" in extra_declared_checks_json[ + "failures" + ] + assert "checkedNoDuplicateBaselineChecksDeclared" in extra_declared_checks_json[ + "failures" + ] + assert extra_declared_checks_json["declaredChecks"]["unknown"] == [ + "obsolete-record-and-replay-smoke" + ] + assert extra_declared_checks_json["declaredChecks"]["duplicates"] == [ + "event-stream-smoke-matrix" + ] + + audit_target_evidence_summary = dict(default_summary) + audit_target_evidence_summary["evidence"] = dict(default_summary["evidence"]) + audit_target_evidence_summary["evidence"]["preflightPipelines"] = dict( + default_summary["evidence"]["preflightPipelines"] + ) + audit_target_evidence_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditMakeTargets" + ] = False + audit_target_evidence_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditIgnoresStrictSummaryVar" + ] = False + audit_target_evidence_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenTarget" + ] = False + audit_target_evidence_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar" + ] = False + audit_target_evidence_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath" + ] = False + audit_target_evidence_path = root / "audit-target-evidence-summary.json" + write_json(audit_target_evidence_path, audit_target_evidence_summary) + audit_target_evidence_audit = run( + [sys.executable, str(audit), str(audit_target_evidence_path)], + repo, + ) + assert audit_target_evidence_audit.returncode == 1 + audit_target_evidence_json = parse_json(audit_target_evidence_audit.stdout) + assert "checkedBaselineAuditMakeTargetsEvidence" in audit_target_evidence_json[ + "failures" + ] + assert ( + "checkedBaselineAuditIgnoresStrictSummaryVarEvidence" + in audit_target_evidence_json["failures"] + ) + assert ( + "checkedBaselineAuditStrictOfficialGoldenTargetEvidence" + in audit_target_evidence_json["failures"] + ) + assert ( + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVarEvidence" + in audit_target_evidence_json["failures"] + ) + assert ( + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPathEvidence" + in audit_target_evidence_json["failures"] + ) + + preflight_evidence_summary = json.loads(json.dumps(default_summary)) + preflight_evidence = preflight_evidence_summary["evidence"]["preflightPipelines"] + preflight_evidence["checkedOfficialCapturePacketPostCaptureWorkflow"] = False + preflight_evidence["checkedOfficialCapturePacketWorkflowVerifier"] = False + preflight_evidence["checkedOfficialCapturePacketHandoffScripts"] = False + preflight_evidence["checkedOfficialCapturePacketStrictAuditHandoff"] = False + preflight_evidence[ + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + ] = False + preflight_evidence["checkedOfficialCapturePacketOcuCandidateOutputDir"] = False + preflight_evidence["checkedOfficialCapturePacketNoTranscript"] = False + preflight_evidence["checkedOfficialCapturePacketTranscriptManifest"] = False + preflight_evidence["checkedOfficialCapturePacketInputSemanticGuard"] = False + preflight_evidence["checkedOfficialCapturePacketSetOcuCandidateHandoff"] = False + preflight_evidence["checkedOfficialCapturePacketSetPostCaptureWorkflow"] = False + preflight_evidence["checkedOfficialCapturePacketSetWorkflowVerifier"] = False + preflight_evidence["checkedOfficialCapturePacketSetContractManifest"] = False + preflight_evidence["checkedOfficialCapturePacketSetVerifyAll"] = False + preflight_evidence["checkedOcuPairingPairedCandidatePreflight"] = False + preflight_evidence_path = root / "preflight-evidence-summary.json" + write_json(preflight_evidence_path, preflight_evidence_summary) + preflight_evidence_audit = run( + [sys.executable, str(audit), str(preflight_evidence_path)], + repo, + ) + assert preflight_evidence_audit.returncode == 1 + preflight_evidence_json = parse_json(preflight_evidence_audit.stdout) + assert ( + "checkedOfficialCapturePacketPostCaptureWorkflowEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketWorkflowVerifierEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketHandoffScriptsEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketStrictAuditHandoffEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoffEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketOcuCandidateOutputDirEvidence" + in preflight_evidence_json["failures"] + ) + assert "checkedOfficialCapturePacketNoTranscriptEvidence" in preflight_evidence_json[ + "failures" + ] + assert ( + "checkedOfficialCapturePacketTranscriptManifestEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketInputSemanticGuardEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketSetOcuCandidateHandoffEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketSetPostCaptureWorkflowEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketSetWorkflowVerifierEvidence" + in preflight_evidence_json["failures"] + ) + assert ( + "checkedOfficialCapturePacketSetContractManifestEvidence" + in preflight_evidence_json["failures"] + ) + assert "checkedOfficialCapturePacketSetVerifyAllEvidence" in preflight_evidence_json[ + "failures" + ] + assert "checkedOcuPairingPairedCandidatePreflightEvidence" in preflight_evidence_json[ + "failures" + ] + + contract_evidence_summary = json.loads(json.dumps(default_summary)) + contract_evidence_summary["evidence"]["baselineContract"][ + "checkedStandaloneLifecycleSummaryRenames" + ] = False + contract_evidence_path = root / "contract-evidence-summary.json" + write_json(contract_evidence_path, contract_evidence_summary) + contract_evidence_audit = run( + [sys.executable, str(audit), str(contract_evidence_path)], + repo, + ) + assert contract_evidence_audit.returncode == 1 + contract_evidence_json = parse_json(contract_evidence_audit.stdout) + assert ( + "checkedBaselineContractStandaloneSummaryRenamesEvidence" + in contract_evidence_json["failures"] + ) + assert ( + "checkedBaselineContractCheckedStandaloneLifecycleSummaryRenamesEvidence" + in contract_evidence_json["failures"] + ) + + usable_evidence_summary = json.loads(json.dumps(default_summary)) + usable_evidence = usable_evidence_summary["evidence"] + usable_evidence["eventStreamMatrix"]["ok"] = False + usable_evidence["screenshotContextSmoke"][ + "checkedScreenshotNeededForContext" + ] = False + usable_evidence["realInputActionSmoke"][ + "checkedMcpResponseShapesCaptured" + ] = False + usable_evidence["realInputActionSmoke"][ + "checkedSimpleActionStopCandidate" + ] = False + usable_evidence["realInputActionSmoke"]["checkedDragStopCandidate"] = False + usable_evidence["officialFixtureSetGate"]["checkedCandidatePairing"] = False + usable_evidence["officialFixtureSetGate"][ + "checkedAxDiffComparisonPolicy" + ] = False + usable_evidence["officialFixtureSetGate"][ + "checkedSuppressedStreamComparisonPolicy" + ] = False + usable_evidence["officialFixtureSetGate"][ + "checkedAxDiffComparisonFailure" + ] = False + usable_evidence["fixtureIngestPipelines"][ + "checkedOfficialFixtureInspectOnly" + ] = False + usable_evidence["fixtureIngestPipelines"][ + "checkedOfficialSessionDirectoryPathHandoff" + ] = False + usable_evidence["fixtureIngestPipelines"][ + "checkedOcuCandidateIngestHandoffCommands" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedGeneratedReadmePrerequisites" + ] = False + usable_evidence["standaloneSkillRepo"]["checkedManifestContract"] = False + usable_evidence["standaloneSkillRepo"][ + "checkedOfficialEvidenceAuditManifest" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedOfficialFixtureSetComparePolicyManifest" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryEvidence" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryOfficialGoldenState" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetContractManifest" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier" + ] = False + usable_evidence["standaloneSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff" + ] = False + usable_evidence["standaloneSkillRepo"]["checkedPackageArtifact"] = False + usable_evidence["standaloneSkillRepo"][ + "checkedNotifySuppressedEventsPathEnv" + ] = False + usable_evidence["standaloneSkillRepo"]["checkedGeneratedReadmeScenarioList"] = False + usable_evidence["standaloneSkillRepo"]["checkedRuntimeTimeoutDiagnostics"] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedNpmPythonLauncherDiagnostics" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedGeneratedReadmePrerequisites" + ] = False + usable_evidence["npmStagedSkillRepo"]["checkedManifestContract"] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedOfficialEvidenceAuditManifest" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedOfficialFixtureSetComparePolicyManifest" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryEvidence" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryOfficialGoldenState" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetContractManifest" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier" + ] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff" + ] = False + usable_evidence["npmStagedSkillRepo"]["checkedPackageArtifact"] = False + usable_evidence["npmStagedSkillRepo"][ + "checkedNotifySuppressedEventsPathEnv" + ] = False + usable_evidence["npmStagedSkillRepo"]["checkedRuntimeTimeoutDiagnostics"] = False + usable_evidence["npmStagedSkillRepo"]["checkedScaffoldSkill"] = False + usable_evidence_path = root / "usable-evidence-summary.json" + write_json(usable_evidence_path, usable_evidence_summary) + usable_evidence_audit = run( + [sys.executable, str(audit), str(usable_evidence_path)], + repo, + ) + assert usable_evidence_audit.returncode == 1 + usable_evidence_json = parse_json(usable_evidence_audit.stdout) + assert "checkedEventStreamMatrixOkEvidence" in usable_evidence_json["failures"] + assert ( + "checkedScreenshotContextSmokeCheckedScreenshotNeededForContextEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedRealInputActionSmokeCheckedMcpResponseShapesCapturedEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedRealInputActionSmokeCheckedSimpleActionStopCandidateEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedRealInputActionSmokeCheckedDragStopCandidateEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedOfficialFixtureSetGateCheckedCandidatePairingEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedFixtureIngestPipelinesCheckedOfficialFixtureInspectOnlyEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedGeneratedReadmeScenarioListEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedGeneratedReadmePrerequisitesEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedManifestContractEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketSetContractManifestEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflowEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketWorkflowVerifierEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflowEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketSetWorkflowVerifierEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryCapturePacketStrictAuditHandoffEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedPackageArtifactEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedSourceBaselineSummaryOfficialGoldenStateEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedRuntimeTimeoutDiagnosticsEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedStandaloneSkillRepoCheckedNotifySuppressedEventsPathEnvEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedGeneratedReadmePrerequisitesEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedManifestContractEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketSetContractManifestEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketPostCaptureWorkflowEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketWorkflowVerifierEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflowEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketSetWorkflowVerifierEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryCapturePacketStrictAuditHandoffEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedPackageArtifactEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedSourceBaselineSummaryOfficialGoldenStateEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedRuntimeTimeoutDiagnosticsEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedNotifySuppressedEventsPathEnvEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedNpmPythonLauncherDiagnosticsEvidence" + in usable_evidence_json["failures"] + ) + assert ( + "checkedNpmStagedSkillRepoCheckedScaffoldSkillEvidence" + in usable_evidence_json["failures"] + ) + + next_action_command_summary = json.loads(json.dumps(default_summary)) + for action in next_action_command_summary["nextActions"]: + if action.get("kind") == "capture-official-successful-recording-golden": + action["commands"] = [ + command + for command in action["commands"] + if "record-and-replay-official-golden-capture-packet" not in command + and "./verify-inputs.sh" not in command + ] + elif action.get("kind") == "capture-recommended-official-golden-set": + action["commands"] = [ + command + for command in action["commands"] + if "./verify-all.sh" not in command + and "./ingest-ocu-candidates.sh" not in command + ] + elif action.get("kind") == "scaffold-standalone-record-and-replay-repo": + action["commands"] = [ + command + for command in action["commands"] + if "record-and-replay-baseline-audit" not in command + and "./scripts/check.sh" not in command + ] + next_action_command_path = root / "next-action-command-summary.json" + write_json(next_action_command_path, next_action_command_summary) + next_action_command_audit = run( + [sys.executable, str(audit), str(next_action_command_path)], + repo, + ) + assert next_action_command_audit.returncode == 1 + next_action_command_json = parse_json(next_action_command_audit.stdout) + assert ( + "checkedMissingGoldenNextActionPacketMakeCommand" + in next_action_command_json["failures"] + ) + assert ( + "checkedMissingGoldenNextActionVerifyStep" + in next_action_command_json["failures"] + ) + assert ( + "checkedRecommendedGoldenNextActionVerifyAllStep" + in next_action_command_json["failures"] + ) + assert ( + "checkedRecommendedGoldenNextActionIngestOcuCandidatesStep" + in next_action_command_json["failures"] + ) + assert ( + "checkedStandaloneNextActionBaselineAuditCommand" + in next_action_command_json["failures"] + ) + assert ( + "checkedStandaloneNextActionCheckCommand" + in next_action_command_json["failures"] + ) + + official_boundary_evidence_summary = json.loads(json.dumps(default_summary)) + official_boundary_evidence_summary["evidence"]["officialSurfaceCompare"]["official"][ + "ok" + ] = False + official_boundary_evidence_summary["evidence"]["officialNoActiveResponse"][ + "checkedStopShape" + ] = False + official_boundary_evidence_summary["evidence"]["officialRawStartTimeout"][ + "checkedOfficialRawStartDoesNotReturnRecordingPaths" + ] = False + official_boundary_evidence_path = root / "official-boundary-evidence-summary.json" + write_json(official_boundary_evidence_path, official_boundary_evidence_summary) + official_boundary_evidence_audit = run( + [sys.executable, str(audit), str(official_boundary_evidence_path)], + repo, + ) + assert official_boundary_evidence_audit.returncode == 1 + official_boundary_evidence_json = parse_json( + official_boundary_evidence_audit.stdout + ) + assert "checkedOfficialSurfaceBundledEvidence" in official_boundary_evidence_json[ + "failures" + ] + assert "checkedOfficialNoActiveStopShapeEvidence" in official_boundary_evidence_json[ + "failures" + ] + assert ( + "checkedOfficialRawStartDoesNotReturnRecordingPathsEvidence" + in official_boundary_evidence_json["failures"] + ) + + print( + json.dumps( + { + "ok": True, + "checkedDefaultMissingOfficialGoldenAudit": True, + "checkedBuilderChecksMatchAuditRequirements": True, + "checkedStrictMissingOfficialGoldenAuditFails": True, + "checkedExpectedStrictMissingOfficialGoldenAudit": True, + "checkedStrictCoveredOfficialGoldenAudit": True, + "checkedInconsistentEquivalenceAuditFails": True, + "checkedDerivedStatusAuditFails": True, + "checkedCompleteGoldenWithGapsAuditFails": True, + "checkedIncompleteBaselineAuditFails": True, + "checkedTopLevelOkPolicyAuditFails": True, + "checkedBaselineNameAuditFails": True, + "checkedDeclaredChecksAuditFails": True, + "checkedExtraDeclaredChecksAuditFails": True, + "checkedBaselineAuditTargetEvidenceAuditFails": True, + "checkedPreflightPipelineEvidenceAuditFails": True, + "checkedUsableBaselineDirectEvidenceAuditFails": True, + "checkedNextActionCommandAuditFails": True, + "checkedOfficialBoundaryEvidenceAuditFails": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-baseline-summary.py b/scripts/test-record-and-replay-baseline-summary.py new file mode 100755 index 00000000..55edfa2c --- /dev/null +++ b/scripts/test-record-and-replay-baseline-summary.py @@ -0,0 +1,2273 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import shlex +import subprocess +import sys +import tempfile + +from record_and_replay_baseline_contract import ( + NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, + REQUIRED_BASELINE_CHECKS, + STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS, +) + + +TOOL_NAMES = ["event_stream_start", "event_stream_status", "event_stream_stop"] +NO_ACTIVE = {"isRecording": False, "maxDurationSeconds": 1800} + + +def write_json(path: pathlib.Path, value: dict) -> None: + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path: pathlib.Path, records: list[dict]) -> None: + path.write_text("\n".join(json.dumps(record, sort_keys=True) for record in records) + "\n") + + +def run(command: list[str], cwd: pathlib.Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def parse_last_json(stdout: str) -> dict: + records = [] + for line in stdout.splitlines(): + line = line.strip() + if not line.startswith("{"): + continue + records.append(json.loads(line)) + assert records, stdout + return records[-1] + + +def write_inputs(root: pathlib.Path, has_official_golden: bool) -> dict[str, pathlib.Path]: + root.mkdir(parents=True, exist_ok=True) + paths = { + "standalone": root / "standalone.json", + "npm": root / "npm.json", + "surface": root / "surface.json", + "no_active": root / "no-active.json", + "raw_timeout": root / "raw-timeout.json", + "fixture_set": root / "fixture-set.json", + "coverage": root / "coverage.json", + "official_ingest": root / "official-ingest.json", + "candidate_ingest": root / "candidate-ingest.json", + "official_capture_preflight": root / "official-capture-preflight.json", + "ocu_pairing_preflight": root / "ocu-pairing-preflight.json", + "baseline_audit_targets": root / "baseline-audit-targets.json", + "baseline_contract": root / "baseline-contract.json", + "matrix": root / "matrix.jsonl", + "screenshot": root / "screenshot.jsonl", + "action": root / "action.jsonl", + } + write_json( + paths["baseline_contract"], + { + "ok": True, + "checkedRequiredBaselineChecks": True, + "checkedNoDuplicateRequiredBaselineChecks": True, + "checkedStandaloneSmokeRequiredKeys": True, + "checkedStandaloneSummaryEvidenceKeys": True, + "checkedStandaloneLifecycleSummaryRenames": True, + "checkedNpmStagedSmokeRequiredKeys": True, + "checkedNpmStagedSummaryEvidenceKeys": True, + "checkedNpmStagedSummaryMatchesSmoke": True, + }, + ) + write_json( + paths["standalone"], + { + "checkedGeneratedRepoSelfCheck": True, + "checkedManifestContract": True, + "checkedReadmeHandoffContract": True, + "checkedReadmeOfficialEvidenceHandoff": True, + "checkedReadmeOfficialGoldenGap": True, + "checkedReadmeWaitNotifyBoundary": True, + "checkedGeneratedReadmePrerequisites": True, + "checkedGeneratedReadmeScenarioList": True, + "checkedOfficialEvidenceManifest": True, + "checkedOfficialEvidenceScenarioManifest": True, + "checkedOfficialEvidencePreflightManifest": True, + "checkedOfficialEvidenceAuditManifest": True, + "checkedOfficialFixtureSetComparePolicyManifest": True, + "checkedSourceBaselineSummaryEvidence": True, + "checkedSourceBaselineSummaryOfficialGoldenState": True, + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": True, + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": True, + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": True, + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedPackageArtifact": True, + "checkedRuntimeContract": True, + "checkedRuntimeTimeoutDiagnostics": True, + "checkedInitializeSurfaceContract": True, + "checkedToolMetadataContract": True, + "checkedToolInputSchemaNoArguments": True, + "checkedNoActiveStatusStop": True, + "checkedRejectsUnexpectedArguments": True, + "checkedRejectsNonObjectArguments": True, + "checkedRequiresObjectParams": True, + "checkedRequiresStringToolName": True, + "checkedRequiresObjectArguments": True, + "checkedRejectedRequestsDoNotCreateSessionFiles": True, + "checkedStatusNotUsedAsWaitLoopGuard": True, + "checkedMcpNoDirectEventContentsGuard": True, + "checkedWaitNotifyContract": True, + "checkedNotifySuppressedEventsPathEnv": True, + "checkedNotifyCallbackFailureExit": True, + "checkedNotifyCallbackFailureReason": True, + "checkedNotifyCallbackTimeoutFailureExit": True, + "checkedNotifyCallbackTimeoutReason": True, + "checkedRecordingToSkillHandoff": True, + "checkedStrictValidation": True, + "checkedDeclaredHandoffPaths": True, + "checkedEventsOnlyValidation": True, + "checkedScaffoldSkill": True, + "checkedScaffoldSkillFailureExit": True, + "checkedSkillCreatorHandoff": True, + "checkedCancelledRecordingContract": True, + "checkedCancelledRecordingRejected": True, + "checkedScreenshotPathContainment": True, + "checkedLifecycleSmoke": True, + "checkedOneActive": True, + "checkedIdempotentStop": True, + "checkedFinalStatus": True, + }, + ) + write_json( + paths["npm"], + { + "checkedNpmLauncher": True, + "checkedNpmPythonLauncherDiagnostics": True, + "checkedStandaloneRepoScaffold": True, + "checkedManifestContract": True, + "checkedReadmeHandoffContract": True, + "checkedReadmeOfficialEvidenceHandoff": True, + "checkedReadmeOfficialGoldenGap": True, + "checkedReadmeWaitNotifyBoundary": True, + "checkedGeneratedRepoSelfCheck": True, + "checkedGeneratedReadmePrerequisites": True, + "checkedGeneratedReadmeScenarioList": True, + "checkedOfficialEvidenceManifest": True, + "checkedOfficialEvidenceScenarioManifest": True, + "checkedOfficialEvidencePreflightManifest": True, + "checkedOfficialEvidenceAuditManifest": True, + "checkedOfficialFixtureSetComparePolicyManifest": True, + "checkedSourceBaselineSummaryEvidence": True, + "checkedSourceBaselineSummaryOfficialGoldenState": True, + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": True, + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": True, + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": True, + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedPackageArtifact": True, + "checkedNoActiveStatusStop": True, + "checkedRuntimeTimeoutDiagnostics": True, + "checkedInitializeSurfaceContract": True, + "checkedToolMetadataContract": True, + "checkedToolInputSchemaNoArguments": True, + "checkedRequiresObjectParams": True, + "checkedRequiresStringToolName": True, + "checkedRequiresObjectArguments": True, + "checkedRejectsUnexpectedArguments": True, + "checkedRejectsNonObjectArguments": True, + "checkedRejectedRequestsDoNotCreateSessionFiles": True, + "checkedSkillWorkflow": True, + "checkedStatusNotUsedAsWaitLoopGuard": True, + "checkedMcpNoDirectEventContentsGuard": True, + "checkedWaitNotifyContract": True, + "checkedNotifySuppressedEventsPathEnv": True, + "checkedNotifyCallbackFailureExit": True, + "checkedNotifyCallbackFailureReason": True, + "checkedNotifyCallbackTimeoutFailureExit": True, + "checkedNotifyCallbackTimeoutReason": True, + "checkedSkillPackaging": True, + "checkedRecordingToSkillManifestContract": True, + "checkedCancelledRecordingContract": True, + "checkedStrictValidation": True, + "checkedDeclaredHandoffPaths": True, + "checkedScreenshotPathContainment": True, + "checkedEventsOnlyValidation": True, + "checkedScaffoldSkill": True, + "checkedScaffoldSkillFailureExit": True, + "checkedCancelledRecordingRejected": True, + "checkedRecordingToSkillHandoff": True, + "checkedSkillCreatorHandoff": True, + }, + ) + write_json( + paths["surface"], + { + "results": [ + { + "label": "local-open-computer-use", + "ok": True, + "protocolVersion": "2025-11-25", + "serverName": "Record & Replay", + "toolNames": TOOL_NAMES, + }, + { + "label": "official-record-and-replay", + "ok": True, + "protocolVersion": "2025-11-25", + "serverName": "Record & Replay", + "toolNames": TOOL_NAMES, + }, + ] + }, + ) + write_json( + paths["no_active"], + { + "ok": True, + "fixture": "/tmp/record-and-replay-official-no-active-status-stop-1.0.857.json", + "checkedTools": ["event_stream_status", "event_stream_stop"], + "createdSessionFiles": False, + "actual": { + "event_stream_status": NO_ACTIVE, + "event_stream_stop": NO_ACTIVE, + }, + }, + ) + write_json( + paths["raw_timeout"], + { + "ok": True, + "checkedOfficialRawStartTimeoutBoundary": True, + "checkedOfficialRawStartStatusStopTimeout": True, + "checkedOfficialRawStartDoesNotReturnRecordingPaths": True, + "checkedOfficialRawSurface": True, + "checkedRawProbeFixtureRedaction": True, + "officialRawToolCallCount": 3, + "officialRawTimeoutCount": 3, + "fixtures": [ + "docs/references/codex-computer-use-reverse-engineering/fixtures/record-and-replay-official-raw-start-timeout-1.0.857.json" + ], + }, + ) + write_json( + paths["fixture_set"], + { + "ok": True, + "checkedFixtureSetGate": True, + "checkedRequiredSimpleActionStopScenario": True, + "checkedCandidatePairing": True, + "checkedCandidateReadinessFailure": True, + "checkedStopEndReasonPolicy": True, + "checkedAxDiffComparisonPolicy": True, + "checkedSuppressedStreamComparisonPolicy": True, + "checkedAxDiffComparisonFailure": True, + "checkedCancelScenarioPolicy": True, + "checkedKeyboardInputScenarioPolicy": True, + "checkedDragScenarioPolicy": True, + "checkedTimeoutScenarioPolicy": True, + }, + ) + missing_required = [] if has_official_golden else ["simple-action-stop"] + missing_recommended = [] if has_official_golden else ["simple-action-stop"] + not_ready = [] if has_official_golden else [] + write_json( + paths["coverage"], + { + "coverageOk": has_official_golden, + "scenarioCoverageOk": has_official_golden, + "hasRequiredOfficialSuccessfulFixture": has_official_golden, + "requiredOfficialReadinessChecked": True, + "requiredOfficialReadinessOk": has_official_golden, + "hasRecommendedOfficialScenarioCoverage": has_official_golden, + "requiredOfficialScenarios": ["simple-action-stop"], + "recommendedOfficialScenarios": ["simple-action-stop"], + "availableOfficialScenarios": ["simple-action-stop"] if has_official_golden else [], + "missingOfficialScenarios": missing_required, + "missingRecommendedOfficialScenarios": missing_recommended, + "notReadyOfficialScenarios": not_ready, + }, + ) + write_json( + paths["official_ingest"], + { + "checkedOfficialFixtureIngest": True, + "checkedOfficialFixtureInspectOnly": True, + "checkedOfficialSessionDirectoryPathHandoff": True, + "checkedPostIngestCoverageReport": True, + "checkedPostIngestCoverageReadiness": True, + "checkedPostIngestRequireCoverageFailure": True, + }, + ) + write_json( + paths["candidate_ingest"], + { + "checkedOcuCandidateIngest": True, + "checkedCandidateIngestHandoffCommands": True, + "checkedSmokeJsonImport": True, + "checkedOfficialCandidatePairing": True, + "checkedCandidateRedaction": True, + "checkedKeyboardInputScenarioReadiness": True, + "checkedDragScenarioReadiness": True, + }, + ) + write_json( + paths["official_capture_preflight"], + { + "ok": True, + "checkedMissingScenarioPreflight": True, + "checkedCapturePacket": True, + "checkedCapturePacketPostCaptureWorkflow": True, + "checkedCapturePacketWorkflowVerifier": True, + "checkedCapturePacketHandoffScripts": True, + "checkedCapturePacketStrictAuditHandoff": True, + "checkedCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedCapturePacketOcuCandidateOutputDir": True, + "checkedCapturePacketNoTranscript": True, + "checkedCapturePacketSet": True, + "checkedCapturePacketSetPostCaptureWorkflow": True, + "checkedCapturePacketSetWorkflowVerifier": True, + "checkedCapturePacketSetNoTranscript": True, + "checkedCapturePacketSetOcuCandidateHandoff": True, + "checkedCapturePacketSetContractManifest": True, + "checkedCapturePacketTranscriptManifest": True, + "checkedMakeCapturePacketTargets": True, + "checkedCapturePacketInputSemanticGuard": True, + "checkedCapturePacketPlaceholderGuard": True, + "checkedCapturePacketVerifyInputs": True, + "checkedCapturePacketImportPlaceholderGuard": True, + "checkedCapturePacketTranscriptPlaceholderGuard": True, + "checkedCapturePacketSetRootPlaceholderGuard": True, + "checkedCapturePacketSetRootPreflightPlaceholderGuard": True, + "checkedCapturePacketSetVerifyAll": True, + "checkedRequireReadyFailure": True, + "checkedMissingPluginFailure": True, + "checkedAllowedMissingPluginKeyboardScenario": True, + "checkedCoverageErrorFailure": True, + }, + ) + write_json( + paths["ocu_pairing_preflight"], + { + "ok": True, + "checkedMissingOfficialPreflight": True, + "checkedNoCandidatePreflight": True, + "checkedPairedCandidatePreflight": True, + "checkedKeyboardRecordingRequiredScenario": True, + "checkedCoverageErrorReport": True, + }, + ) + write_json( + paths["baseline_audit_targets"], + { + "ok": True, + "checkedBaselineAuditMakeTarget": True, + "checkedBaselineAuditDefaultSummaryPath": True, + "checkedBaselineAuditCustomSummaryPath": True, + "checkedBaselineAuditIgnoresStrictSummaryVar": True, + "checkedStrictOfficialGoldenAuditMakeTarget": True, + "checkedStrictOfficialGoldenAuditDefaultSummaryPath": True, + "checkedStrictOfficialGoldenAuditCustomSummaryPath": True, + "checkedStrictOfficialGoldenAuditIgnoresBaselineSummaryVar": True, + "checkedStrictOfficialGoldenAuditSeparateSummaryPath": True, + }, + ) + write_jsonl( + paths["matrix"], + [ + {"ok": True, "handoffChecked": True}, + {"ok": True, "mode": "no-active"}, + {"ok": True, "mode": "timeout"}, + {"ok": True, "mode": "wait-timeout"}, + {"ok": True, "mode": "approval"}, + {"ok": True, "mode": "mcp-elicitation"}, + {"ok": True, "mode": "app-agent-wait"}, + {"ok": True, "matrix": "event-stream"}, + ], + ) + write_jsonl( + paths["screenshot"], + [ + { + "ok": True, + "screenshotPolicy": "always", + "screenshotNeededForContextCount": 1, + "screenshotAvailableCount": 0, + "screenshotPathCount": 0, + } + ], + ) + write_jsonl( + paths["action"], + [ + { + "ok": True, + "mode": "actions", + "actionScenario": "mixed-action-stop", + "eventCount": 7, + "eventTypes": [ + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.click", + "session.ended", + ], + "skillPath": "/tmp/SKILL.md", + "mcpTranscriptPath": "/tmp/mcp-transcript.json", + "checkedMcpResponseShapesCaptured": True, + "checkedSkillReadinessCanCreateDraft": True, + "checkedSkillCreatorFinalizationHandoff": True, + "checkedGeneratedSkillPathRedaction": True, + }, + { + "ok": True, + "mode": "actions", + "actionScenario": "simple-action-stop", + "eventCount": 5, + "eventTypes": [ + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.click", + "session.ended", + ], + "skillPath": "/tmp/simple/SKILL.md", + "mcpTranscriptPath": "/tmp/simple-mcp-transcript.json", + "checkedMcpResponseShapesCaptured": True, + "checkedSkillReadinessCanCreateDraft": True, + "checkedSkillCreatorFinalizationHandoff": True, + "checkedGeneratedSkillPathRedaction": True, + }, + { + "ok": True, + "mode": "actions", + "actionScenario": "drag-stop", + "eventCount": 5, + "eventTypes": [ + "session.started", + "window.changed", + "AX.focusedWindowChanged", + "mouse.drag", + "session.ended", + ], + "skillPath": "/tmp/drag/SKILL.md", + "mcpTranscriptPath": "/tmp/drag-mcp-transcript.json", + "checkedMcpResponseShapesCaptured": True, + "checkedSkillReadinessCanCreateDraft": True, + "checkedSkillCreatorFinalizationHandoff": True, + "checkedGeneratedSkillPathRedaction": True, + } + ], + ) + return paths + + +def summary_command(script: pathlib.Path, paths: dict[str, pathlib.Path]) -> list[str]: + return [ + sys.executable, + str(script), + "--standalone-json", + str(paths["standalone"]), + "--npm-json", + str(paths["npm"]), + "--official-surface-json", + str(paths["surface"]), + "--official-no-active-json", + str(paths["no_active"]), + "--official-raw-timeout-json", + str(paths["raw_timeout"]), + "--official-fixture-set-json", + str(paths["fixture_set"]), + "--official-fixture-coverage-json", + str(paths["coverage"]), + "--official-fixture-ingest-json", + str(paths["official_ingest"]), + "--ocu-candidate-ingest-json", + str(paths["candidate_ingest"]), + "--official-capture-preflight-json", + str(paths["official_capture_preflight"]), + "--ocu-pairing-preflight-json", + str(paths["ocu_pairing_preflight"]), + "--baseline-audit-targets-json", + str(paths["baseline_audit_targets"]), + "--baseline-contract-json", + str(paths["baseline_contract"]), + "--matrix-jsonl", + str(paths["matrix"]), + "--screenshot-jsonl", + str(paths["screenshot"]), + "--action-jsonl", + str(paths["action"]), + ] + + +def main() -> None: + repo = pathlib.Path(__file__).resolve().parents[1] + script = repo / "scripts/build-record-and-replay-baseline-summary.py" + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + missing_paths = write_inputs(root / "missing", has_official_golden=False) + + default_result = run(summary_command(script, missing_paths), repo) + assert default_result.returncode == 0, default_result.stderr + default_summary = parse_last_json(default_result.stdout) + assert default_summary["ok"] is True + assert default_summary["checks"] == list(REQUIRED_BASELINE_CHECKS) + assert default_summary["status"]["usableBaseline"] is True + assert default_summary["evidence"]["baselineContract"] == { + "ok": True, + "checkedRequiredBaselineChecks": True, + "checkedNoDuplicateRequiredBaselineChecks": True, + "checkedStandaloneSmokeRequiredKeys": True, + "checkedStandaloneSummaryEvidenceKeys": True, + "checkedStandaloneLifecycleSummaryRenames": True, + "checkedNpmStagedSmokeRequiredKeys": True, + "checkedNpmStagedSummaryEvidenceKeys": True, + "checkedNpmStagedSummaryMatchesSmoke": True, + } + assert default_summary["status"]["missingUsableBaselineEvidence"] == [] + assert default_summary["status"]["strictOfficialGoldenRequired"] is False + assert default_summary["status"]["officialGoldenRequirementSatisfied"] is True + assert default_summary["status"]["officialGoldenGatePassed"] is False + assert default_summary["status"]["officialRawStartTimeoutBoundaryVerified"] is True + assert default_summary["status"]["standaloneRepoBaselineReady"] is True + assert default_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] is False + assert set(default_summary["evidence"]["standaloneSkillRepo"]) == set( + STANDALONE_SKILL_REPO_SUMMARY_EVIDENCE_KEYS + ) + assert set(default_summary["evidence"]["npmStagedSkillRepo"]) == set( + NPM_STAGED_SKILL_REPO_SUMMARY_EVIDENCE_KEYS + ) + assert default_summary["evidence"]["officialRawStartTimeout"][ + "checkedOfficialRawStartDoesNotReturnRecordingPaths" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSet" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketPostCaptureWorkflow" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketWorkflowVerifier" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetPostCaptureWorkflow" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetWorkflowVerifier" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictAuditHandoff" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketOcuCandidateOutputDir" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketNoTranscript" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetNoTranscript" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetOcuCandidateHandoff" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetContractManifest" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketTranscriptManifest" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketMakeTargets" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketInputSemanticGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketPlaceholderGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketVerifyInputs" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketImportPlaceholderGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketTranscriptPlaceholderGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetRootPlaceholderGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetVerifyAll" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedOcuPairingPairedCandidatePreflight" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditMakeTargets" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditDefaultSummaryPath" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditCustomSummaryPath" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditIgnoresStrictSummaryVar" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenTarget" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenDefaultSummaryPath" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenCustomSummaryPath" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar" + ] is True + assert default_summary["evidence"]["preflightPipelines"][ + "checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath" + ] is True + assert default_summary["evidence"]["standaloneSkillRepo"][ + "checkedOfficialEvidencePreflightManifest" + ] is True + assert default_summary["evidence"]["standaloneSkillRepo"][ + "checkedOfficialEvidenceAuditManifest" + ] is True + assert default_summary["evidence"]["npmStagedSkillRepo"][ + "checkedOfficialEvidencePreflightManifest" + ] is True + assert default_summary["evidence"]["npmStagedSkillRepo"][ + "checkedOfficialEvidenceAuditManifest" + ] is True + assert default_summary["status"]["officialSuccessfulRecordingGoldenComplete"] is False + assert default_summary["status"]["missingRequiredOfficialSuccessfulRecordingScenarios"] == [ + "simple-action-stop" + ] + default_next_action_kinds = { + action.get("kind") for action in default_summary["nextActions"] + } + assert "capture-official-successful-recording-golden" in default_next_action_kinds + assert "capture-recommended-official-golden-set" in default_next_action_kinds + assert "scaffold-standalone-record-and-replay-repo" in default_next_action_kinds + assert "fix-usable-baseline-evidence" not in default_next_action_kinds + default_next_actions_by_kind = { + action.get("kind"): action for action in default_summary["nextActions"] + } + required_capture_commands = default_next_actions_by_kind[ + "capture-official-successful-recording-golden" + ]["commands"] + recommended_capture_commands = default_next_actions_by_kind[ + "capture-recommended-official-golden-set" + ]["commands"] + standalone_commands = default_next_actions_by_kind[ + "scaffold-standalone-record-and-replay-repo" + ]["commands"] + assert ( + "make record-and-replay-official-golden-capture-packet " + "RNR_SCENARIO=simple-action-stop RNR_PACKET_DIR=" + ) in required_capture_commands + assert "cd && ./verify-inputs.sh" in required_capture_commands + assert "cd && ./inspect-only.sh" in required_capture_commands + assert "cd && ./import-fixture.sh" in required_capture_commands + assert ( + "make record-and-replay-official-golden-capture-packet-set " + "RNR_PACKET_DIR=" + ) in recommended_capture_commands + assert "cd && ./verify-all.sh" in recommended_capture_commands + assert "cd && ./inspect-all.sh" in recommended_capture_commands + assert "cd && ./import-all.sh" in recommended_capture_commands + assert "cd && ./ingest-ocu-candidates.sh" in recommended_capture_commands + assert "make record-and-replay-baseline-audit" in standalone_commands + assert ( + "open-computer-use scaffold-record-and-replay-skill-repo " + "--output-dir " + ) in standalone_commands + assert "cd && ./scripts/check.sh" in standalone_commands + assert "cd && ./scripts/recording-lifecycle-smoke.py" in standalone_commands + fixture_root = root / "summary-next-action-fixtures" + fixture_root.mkdir() + required_packet_dir = root / "summary-next-action-required-packet" + required_packet_command = shlex.split( + required_capture_commands[0].replace("", str(required_packet_dir)) + ) + [ + f"RNR_FIXTURE_ROOT={fixture_root}", + f"RNR_OFFICIAL_PLUGIN_ROOT={root / 'missing-official-plugin'}", + "RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1", + ] + required_packet_result = run(required_packet_command, repo) + assert required_packet_result.returncode == 0, required_packet_result.stderr + required_packet_preflight = json.loads((required_packet_dir / "preflight.json").read_text()) + required_packet_recipe = json.loads( + (required_packet_dir / "scenario-recipe.json").read_text() + ) + assert required_packet_preflight["stage"] == "preflight" + assert required_packet_preflight["scenario"] == "simple-action-stop" + assert required_packet_recipe["scenario"] == "simple-action-stop" + assert (required_packet_dir / "verify-inputs.sh").exists() + assert (required_packet_dir / "inspect-only.sh").exists() + assert (required_packet_dir / "import-fixture.sh").exists() + recommended_packet_dir = root / "summary-next-action-recommended-packet-set" + recommended_packet_command = shlex.split( + recommended_capture_commands[0].replace("", str(recommended_packet_dir)) + ) + [ + f"RNR_FIXTURE_ROOT={fixture_root}", + f"RNR_OFFICIAL_PLUGIN_ROOT={root / 'missing-official-plugin'}", + "RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1", + ] + recommended_packet_result = run(recommended_packet_command, repo) + assert recommended_packet_result.returncode == 0, recommended_packet_result.stderr + recommended_packet_manifest = json.loads( + (recommended_packet_dir / "capture-packets.json").read_text() + ) + assert recommended_packet_manifest["stage"] == "capture-packet-set" + assert recommended_packet_manifest["scenarios"] == [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", + ] + assert (recommended_packet_dir / "verify-all.sh").exists() + assert (recommended_packet_dir / "inspect-all.sh").exists() + assert (recommended_packet_dir / "import-all.sh").exists() + assert (recommended_packet_dir / "ingest-ocu-candidates.sh").exists() + + strict_missing = run( + summary_command(script, missing_paths) + ["--require-official-golden"], + repo, + ) + assert strict_missing.returncode == 1 + strict_missing_summary = parse_last_json(strict_missing.stdout) + assert strict_missing_summary["ok"] is False + assert strict_missing_summary["status"]["usableBaseline"] is True + assert strict_missing_summary["status"]["strictOfficialGoldenRequired"] is True + assert strict_missing_summary["status"]["officialGoldenRequirementSatisfied"] is False + assert strict_missing_summary["status"]["officialGoldenGatePassed"] is False + assert "simple-action-stop" in strict_missing.stderr + + incomplete_paths = write_inputs(root / "incomplete", has_official_golden=False) + npm_payload = json.loads(incomplete_paths["npm"].read_text()) + npm_payload["checkedSkillCreatorHandoff"] = False + write_json(incomplete_paths["npm"], npm_payload) + incomplete_result = run(summary_command(script, incomplete_paths), repo) + assert incomplete_result.returncode == 1 + incomplete_summary = parse_last_json(incomplete_result.stdout) + assert incomplete_summary["ok"] is False + assert incomplete_summary["status"]["usableBaseline"] is False + assert incomplete_summary["status"]["standaloneRepoBaselineReady"] is False + assert ( + incomplete_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] + is False + ) + assert incomplete_summary["status"]["officialGoldenRequirementSatisfied"] is True + assert incomplete_summary["status"]["officialGoldenGatePassed"] is False + assert "npmStagedSkillRepo.checkedSkillCreatorHandoff" in incomplete_summary["status"][ + "missingUsableBaselineEvidence" + ] + incomplete_next_action_kinds = { + action.get("kind") for action in incomplete_summary["nextActions"] + } + assert "fix-usable-baseline-evidence" in incomplete_next_action_kinds + assert "scaffold-standalone-record-and-replay-repo" not in incomplete_next_action_kinds + assert "usable baseline evidence is incomplete" in incomplete_result.stderr + + action_evidence_paths = write_inputs( + root / "action-evidence", + has_official_golden=False, + ) + action_records = [ + json.loads(line) + for line in action_evidence_paths["action"].read_text().splitlines() + if line.strip() + ] + action_records[0]["checkedMcpResponseShapesCaptured"] = False + action_records[0]["checkedSkillReadinessCanCreateDraft"] = False + action_records[0]["checkedSkillCreatorFinalizationHandoff"] = False + action_records[0]["checkedGeneratedSkillPathRedaction"] = False + action_records[1]["eventTypes"].remove("mouse.click") + action_records[2]["eventTypes"].remove("mouse.drag") + write_jsonl(action_evidence_paths["action"], action_records) + action_evidence_result = run(summary_command(script, action_evidence_paths), repo) + assert action_evidence_result.returncode == 1 + action_evidence_summary = parse_last_json(action_evidence_result.stdout) + assert action_evidence_summary["status"]["usableBaseline"] is False + assert ( + "realInputActionSmoke.checkedMcpResponseShapesCaptured" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "realInputActionSmoke.checkedSkillReadinessCanCreateDraft" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "realInputActionSmoke.checkedSkillCreatorFinalizationHandoff" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "realInputActionSmoke.checkedGeneratedSkillPathRedaction" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "realInputActionSmoke.checkedSimpleActionStopCandidate" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "realInputActionSmoke.checkedDragStopCandidate" + in action_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + + contract_evidence_paths = write_inputs( + root / "contract-evidence", + has_official_golden=False, + ) + contract_payload = json.loads(contract_evidence_paths["baseline_contract"].read_text()) + contract_payload["checkedStandaloneLifecycleSummaryRenames"] = False + write_json(contract_evidence_paths["baseline_contract"], contract_payload) + contract_evidence_result = run(summary_command(script, contract_evidence_paths), repo) + assert contract_evidence_result.returncode == 1 + contract_evidence_summary = parse_last_json(contract_evidence_result.stdout) + assert contract_evidence_summary["status"]["usableBaseline"] is False + assert ( + "baselineContract.checkedStandaloneLifecycleSummaryRenames" + in contract_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + + wait_notify_failure_paths = write_inputs( + root / "wait-notify-failure-evidence", + has_official_golden=False, + ) + wait_notify_standalone_payload = json.loads( + wait_notify_failure_paths["standalone"].read_text() + ) + wait_notify_standalone_payload["checkedNotifyCallbackFailureExit"] = False + wait_notify_standalone_payload["checkedNotifySuppressedEventsPathEnv"] = False + wait_notify_standalone_payload["checkedNotifyCallbackTimeoutFailureExit"] = False + write_json(wait_notify_failure_paths["standalone"], wait_notify_standalone_payload) + wait_notify_npm_payload = json.loads(wait_notify_failure_paths["npm"].read_text()) + wait_notify_npm_payload["checkedNotifySuppressedEventsPathEnv"] = False + wait_notify_npm_payload["checkedNotifyCallbackFailureReason"] = False + wait_notify_npm_payload["checkedNotifyCallbackTimeoutReason"] = False + write_json(wait_notify_failure_paths["npm"], wait_notify_npm_payload) + wait_notify_failure_result = run( + summary_command(script, wait_notify_failure_paths), + repo, + ) + assert wait_notify_failure_result.returncode == 1 + wait_notify_failure_summary = parse_last_json(wait_notify_failure_result.stdout) + assert wait_notify_failure_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedNotifyCallbackFailureExit" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedNotifySuppressedEventsPathEnv" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedNotifySuppressedEventsPathEnv" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedNotifyCallbackFailureReason" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedNotifyCallbackTimeoutFailureExit" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedNotifyCallbackTimeoutReason" + in wait_notify_failure_summary["status"]["missingUsableBaselineEvidence"] + ) + + malformed_evidence_paths = write_inputs(root / "malformed-evidence", has_official_golden=False) + standalone_payload = json.loads(malformed_evidence_paths["standalone"].read_text()) + standalone_payload["checkedRejectedRequestsDoNotCreateSessionFiles"] = False + write_json(malformed_evidence_paths["standalone"], standalone_payload) + malformed_evidence_result = run(summary_command(script, malformed_evidence_paths), repo) + assert malformed_evidence_result.returncode == 1 + malformed_evidence_summary = parse_last_json(malformed_evidence_result.stdout) + assert malformed_evidence_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedRejectedRequestsDoNotCreateSessionFiles" + in malformed_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + + npm_request_shape_paths = write_inputs(root / "npm-request-shape", has_official_golden=False) + npm_request_shape_payload = json.loads(npm_request_shape_paths["npm"].read_text()) + npm_request_shape_payload["checkedRequiresObjectParams"] = False + write_json(npm_request_shape_paths["npm"], npm_request_shape_payload) + npm_request_shape_result = run(summary_command(script, npm_request_shape_paths), repo) + assert npm_request_shape_result.returncode == 1 + npm_request_shape_summary = parse_last_json(npm_request_shape_result.stdout) + assert npm_request_shape_summary["status"]["usableBaseline"] is False + assert ( + "npmStagedSkillRepo.checkedRequiresObjectParams" + in npm_request_shape_summary["status"]["missingUsableBaselineEvidence"] + ) + + runtime_timeout_paths = write_inputs( + root / "runtime-timeout-diagnostics", + has_official_golden=False, + ) + runtime_timeout_standalone_payload = json.loads( + runtime_timeout_paths["standalone"].read_text() + ) + runtime_timeout_standalone_payload["checkedRuntimeTimeoutDiagnostics"] = False + write_json( + runtime_timeout_paths["standalone"], + runtime_timeout_standalone_payload, + ) + runtime_timeout_npm_payload = json.loads(runtime_timeout_paths["npm"].read_text()) + runtime_timeout_npm_payload["checkedRuntimeTimeoutDiagnostics"] = False + write_json(runtime_timeout_paths["npm"], runtime_timeout_npm_payload) + runtime_timeout_result = run(summary_command(script, runtime_timeout_paths), repo) + assert runtime_timeout_result.returncode == 1 + runtime_timeout_summary = parse_last_json(runtime_timeout_result.stdout) + assert runtime_timeout_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedRuntimeTimeoutDiagnostics" + in runtime_timeout_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedRuntimeTimeoutDiagnostics" + in runtime_timeout_summary["status"]["missingUsableBaselineEvidence"] + ) + + npm_python_launcher_paths = write_inputs( + root / "npm-python-launcher-diagnostics", + has_official_golden=False, + ) + npm_python_launcher_payload = json.loads( + npm_python_launcher_paths["npm"].read_text() + ) + npm_python_launcher_payload["checkedNpmPythonLauncherDiagnostics"] = False + write_json(npm_python_launcher_paths["npm"], npm_python_launcher_payload) + npm_python_launcher_result = run( + summary_command(script, npm_python_launcher_paths), + repo, + ) + assert npm_python_launcher_result.returncode == 1 + npm_python_launcher_summary = parse_last_json(npm_python_launcher_result.stdout) + assert npm_python_launcher_summary["status"]["usableBaseline"] is False + assert ( + "npmStagedSkillRepo.checkedNpmPythonLauncherDiagnostics" + in npm_python_launcher_summary["status"]["missingUsableBaselineEvidence"] + ) + + lifecycle_evidence_paths = write_inputs(root / "lifecycle-evidence", has_official_golden=False) + lifecycle_payload = json.loads(lifecycle_evidence_paths["standalone"].read_text()) + lifecycle_payload["checkedOneActive"] = False + lifecycle_payload["checkedIdempotentStop"] = False + lifecycle_payload["checkedFinalStatus"] = False + write_json(lifecycle_evidence_paths["standalone"], lifecycle_payload) + lifecycle_result = run(summary_command(script, lifecycle_evidence_paths), repo) + assert lifecycle_result.returncode == 1 + lifecycle_summary = parse_last_json(lifecycle_result.stdout) + assert lifecycle_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedLifecycleOneActive" + in lifecycle_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedLifecycleIdempotentStop" + in lifecycle_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedLifecycleFinalStatus" + in lifecycle_summary["status"]["missingUsableBaselineEvidence"] + ) + + recording_to_skill_evidence_paths = write_inputs( + root / "recording-to-skill-evidence", + has_official_golden=False, + ) + recording_to_skill_payload = json.loads( + recording_to_skill_evidence_paths["standalone"].read_text() + ) + recording_to_skill_payload["checkedStrictValidation"] = False + recording_to_skill_payload["checkedEventsOnlyValidation"] = False + recording_to_skill_payload["checkedScaffoldSkill"] = False + recording_to_skill_payload["checkedSkillCreatorHandoff"] = False + write_json( + recording_to_skill_evidence_paths["standalone"], + recording_to_skill_payload, + ) + recording_to_skill_result = run( + summary_command(script, recording_to_skill_evidence_paths), + repo, + ) + assert recording_to_skill_result.returncode == 1 + recording_to_skill_summary = parse_last_json(recording_to_skill_result.stdout) + assert recording_to_skill_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedStrictValidation" + in recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedEventsOnlyValidation" + in recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedScaffoldSkill" + in recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedSkillCreatorHandoff" + in recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + + npm_recording_to_skill_paths = write_inputs( + root / "npm-recording-to-skill-evidence", + has_official_golden=False, + ) + npm_recording_to_skill_payload = json.loads( + npm_recording_to_skill_paths["npm"].read_text() + ) + npm_recording_to_skill_payload["checkedStrictValidation"] = False + npm_recording_to_skill_payload["checkedEventsOnlyValidation"] = False + npm_recording_to_skill_payload["checkedScaffoldSkill"] = False + write_json(npm_recording_to_skill_paths["npm"], npm_recording_to_skill_payload) + npm_recording_to_skill_result = run( + summary_command(script, npm_recording_to_skill_paths), + repo, + ) + assert npm_recording_to_skill_result.returncode == 1 + npm_recording_to_skill_summary = parse_last_json( + npm_recording_to_skill_result.stdout + ) + assert npm_recording_to_skill_summary["status"]["usableBaseline"] is False + assert ( + "npmStagedSkillRepo.checkedStrictValidation" + in npm_recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedEventsOnlyValidation" + in npm_recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedScaffoldSkill" + in npm_recording_to_skill_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_manifest_paths = write_inputs( + root / "preflight-manifest", + has_official_golden=False, + ) + preflight_manifest_payload = json.loads( + preflight_manifest_paths["standalone"].read_text() + ) + preflight_manifest_payload["checkedOfficialEvidencePreflightManifest"] = False + write_json(preflight_manifest_paths["standalone"], preflight_manifest_payload) + preflight_manifest_result = run( + summary_command(script, preflight_manifest_paths), + repo, + ) + assert preflight_manifest_result.returncode == 1 + preflight_manifest_summary = parse_last_json(preflight_manifest_result.stdout) + assert preflight_manifest_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedOfficialEvidencePreflightManifest" + in preflight_manifest_summary["status"]["missingUsableBaselineEvidence"] + ) + + audit_manifest_paths = write_inputs( + root / "audit-manifest", + has_official_golden=False, + ) + audit_manifest_payload = json.loads( + audit_manifest_paths["standalone"].read_text() + ) + audit_manifest_payload["checkedOfficialEvidenceAuditManifest"] = False + write_json(audit_manifest_paths["standalone"], audit_manifest_payload) + audit_manifest_result = run( + summary_command(script, audit_manifest_paths), + repo, + ) + assert audit_manifest_result.returncode == 1 + audit_manifest_summary = parse_last_json(audit_manifest_result.stdout) + assert audit_manifest_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedOfficialEvidenceAuditManifest" + in audit_manifest_summary["status"]["missingUsableBaselineEvidence"] + ) + + fixture_set_policy_paths = write_inputs( + root / "fixture-set-policy-manifest", + has_official_golden=False, + ) + fixture_set_policy_payload = json.loads( + fixture_set_policy_paths["npm"].read_text() + ) + fixture_set_policy_payload[ + "checkedOfficialFixtureSetComparePolicyManifest" + ] = False + write_json(fixture_set_policy_paths["npm"], fixture_set_policy_payload) + fixture_set_policy_result = run( + summary_command(script, fixture_set_policy_paths), + repo, + ) + assert fixture_set_policy_result.returncode == 1 + fixture_set_policy_summary = parse_last_json(fixture_set_policy_result.stdout) + assert fixture_set_policy_summary["status"]["usableBaseline"] is False + assert ( + "npmStagedSkillRepo.checkedOfficialFixtureSetComparePolicyManifest" + in fixture_set_policy_summary["status"]["missingUsableBaselineEvidence"] + ) + + source_summary_evidence_paths = write_inputs( + root / "source-summary-evidence", + has_official_golden=False, + ) + source_summary_evidence_payload = json.loads( + source_summary_evidence_paths["npm"].read_text() + ) + source_summary_evidence_payload[ + "checkedSourceBaselineSummaryEvidence" + ] = False + write_json( + source_summary_evidence_paths["npm"], + source_summary_evidence_payload, + ) + source_summary_evidence_result = run( + summary_command(script, source_summary_evidence_paths), + repo, + ) + assert source_summary_evidence_result.returncode == 1 + source_summary_evidence_summary = parse_last_json( + source_summary_evidence_result.stdout + ) + assert source_summary_evidence_summary["status"]["usableBaseline"] is False + assert ( + "npmStagedSkillRepo.checkedSourceBaselineSummaryEvidence" + in source_summary_evidence_summary["status"]["missingUsableBaselineEvidence"] + ) + + source_summary_state_paths = write_inputs( + root / "source-summary-official-state-evidence", + has_official_golden=False, + ) + source_summary_state_payload = json.loads( + source_summary_state_paths["standalone"].read_text() + ) + source_summary_state_payload[ + "checkedSourceBaselineSummaryOfficialGoldenState" + ] = False + write_json( + source_summary_state_paths["standalone"], + source_summary_state_payload, + ) + source_summary_state_result = run( + summary_command(script, source_summary_state_paths), + repo, + ) + assert source_summary_state_result.returncode == 1 + source_summary_state_summary = parse_last_json( + source_summary_state_result.stdout + ) + assert source_summary_state_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryOfficialGoldenState" + in source_summary_state_summary["status"]["missingUsableBaselineEvidence"] + ) + + source_summary_session_dir_paths = write_inputs( + root / "source-summary-session-directory-handoff", + has_official_golden=False, + ) + source_summary_session_dir_payload = json.loads( + source_summary_session_dir_paths["standalone"].read_text() + ) + source_summary_session_dir_payload[ + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff" + ] = False + write_json( + source_summary_session_dir_paths["standalone"], + source_summary_session_dir_payload, + ) + source_summary_session_dir_result = run( + summary_command(script, source_summary_session_dir_paths), + repo, + ) + assert source_summary_session_dir_result.returncode == 1 + source_summary_session_dir_summary = parse_last_json( + source_summary_session_dir_result.stdout + ) + assert source_summary_session_dir_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff" + in source_summary_session_dir_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + source_summary_contract_paths = write_inputs( + root / "source-summary-contract-evidence", + has_official_golden=False, + ) + source_summary_contract_payload = json.loads( + source_summary_contract_paths["standalone"].read_text() + ) + source_summary_contract_payload[ + "checkedSourceBaselineSummaryCapturePacketSetContractManifest" + ] = False + write_json( + source_summary_contract_paths["standalone"], + source_summary_contract_payload, + ) + source_summary_contract_result = run( + summary_command(script, source_summary_contract_paths), + repo, + ) + assert source_summary_contract_result.returncode == 1 + source_summary_contract_summary = parse_last_json( + source_summary_contract_result.stdout + ) + assert source_summary_contract_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketSetContractManifest" + in source_summary_contract_summary["status"]["missingUsableBaselineEvidence"] + ) + + source_summary_workflow_paths = write_inputs( + root / "source-summary-post-capture-workflow-evidence", + has_official_golden=False, + ) + source_summary_workflow_payload = json.loads( + source_summary_workflow_paths["standalone"].read_text() + ) + source_summary_workflow_payload[ + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow" + ] = False + source_summary_workflow_payload[ + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier" + ] = False + write_json( + source_summary_workflow_paths["standalone"], + source_summary_workflow_payload, + ) + source_summary_workflow_result = run( + summary_command(script, source_summary_workflow_paths), + repo, + ) + assert source_summary_workflow_result.returncode == 1 + source_summary_workflow_summary = parse_last_json( + source_summary_workflow_result.stdout + ) + assert source_summary_workflow_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow" + in source_summary_workflow_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketWorkflowVerifier" + in source_summary_workflow_summary["status"]["missingUsableBaselineEvidence"] + ) + + source_summary_set_workflow_paths = write_inputs( + root / "source-summary-packet-set-post-capture-workflow-evidence", + has_official_golden=False, + ) + source_summary_set_workflow_payload = json.loads( + source_summary_set_workflow_paths["standalone"].read_text() + ) + source_summary_set_workflow_payload[ + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + ] = False + source_summary_set_workflow_payload[ + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier" + ] = False + write_json( + source_summary_set_workflow_paths["standalone"], + source_summary_set_workflow_payload, + ) + source_summary_set_workflow_result = run( + summary_command(script, source_summary_set_workflow_paths), + repo, + ) + assert source_summary_set_workflow_result.returncode == 1 + source_summary_set_workflow_summary = parse_last_json( + source_summary_set_workflow_result.stdout + ) + assert source_summary_set_workflow_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + in source_summary_set_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier" + in source_summary_set_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + source_summary_strict_handoff_paths = write_inputs( + root / "source-summary-strict-handoff-evidence", + has_official_golden=False, + ) + source_summary_strict_handoff_payload = json.loads( + source_summary_strict_handoff_paths["standalone"].read_text() + ) + source_summary_strict_handoff_payload[ + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff" + ] = False + write_json( + source_summary_strict_handoff_paths["standalone"], + source_summary_strict_handoff_payload, + ) + source_summary_strict_handoff_result = run( + summary_command(script, source_summary_strict_handoff_paths), + repo, + ) + assert source_summary_strict_handoff_result.returncode == 1 + source_summary_strict_handoff_summary = parse_last_json( + source_summary_strict_handoff_result.stdout + ) + assert source_summary_strict_handoff_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff" + in source_summary_strict_handoff_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + source_summary_strict_expected_failure_paths = write_inputs( + root / "source-summary-strict-expected-failure-evidence", + has_official_golden=False, + ) + source_summary_strict_expected_failure_payload = json.loads( + source_summary_strict_expected_failure_paths["standalone"].read_text() + ) + source_summary_strict_expected_failure_payload[ + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff" + ] = False + write_json( + source_summary_strict_expected_failure_paths["standalone"], + source_summary_strict_expected_failure_payload, + ) + source_summary_strict_expected_failure_result = run( + summary_command(script, source_summary_strict_expected_failure_paths), + repo, + ) + assert source_summary_strict_expected_failure_result.returncode == 1 + source_summary_strict_expected_failure_summary = parse_last_json( + source_summary_strict_expected_failure_result.stdout + ) + assert ( + source_summary_strict_expected_failure_summary["status"]["usableBaseline"] + is False + ) + assert ( + "standaloneSkillRepo.checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff" + in source_summary_strict_expected_failure_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + manifest_contract_paths = write_inputs( + root / "manifest-contract", + has_official_golden=False, + ) + manifest_contract_standalone = json.loads( + manifest_contract_paths["standalone"].read_text() + ) + manifest_contract_npm = json.loads(manifest_contract_paths["npm"].read_text()) + manifest_contract_standalone["checkedManifestContract"] = False + manifest_contract_npm["checkedManifestContract"] = False + write_json(manifest_contract_paths["standalone"], manifest_contract_standalone) + write_json(manifest_contract_paths["npm"], manifest_contract_npm) + manifest_contract_result = run( + summary_command(script, manifest_contract_paths), + repo, + ) + assert manifest_contract_result.returncode == 1 + manifest_contract_summary = parse_last_json(manifest_contract_result.stdout) + assert manifest_contract_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedManifestContract" + in manifest_contract_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedManifestContract" + in manifest_contract_summary["status"]["missingUsableBaselineEvidence"] + ) + + readme_scenario_paths = write_inputs( + root / "readme-scenario-list", + has_official_golden=False, + ) + readme_standalone_payload = json.loads( + readme_scenario_paths["standalone"].read_text() + ) + readme_npm_payload = json.loads(readme_scenario_paths["npm"].read_text()) + readme_standalone_payload["checkedGeneratedReadmeScenarioList"] = False + readme_standalone_payload["checkedGeneratedReadmePrerequisites"] = False + readme_npm_payload["checkedGeneratedReadmeScenarioList"] = False + readme_npm_payload["checkedGeneratedReadmePrerequisites"] = False + write_json(readme_scenario_paths["standalone"], readme_standalone_payload) + write_json(readme_scenario_paths["npm"], readme_npm_payload) + readme_scenario_result = run(summary_command(script, readme_scenario_paths), repo) + assert readme_scenario_result.returncode == 1 + readme_scenario_summary = parse_last_json(readme_scenario_result.stdout) + assert readme_scenario_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedGeneratedReadmeScenarioList" + in readme_scenario_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedGeneratedReadmePrerequisites" + in readme_scenario_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedGeneratedReadmeScenarioList" + in readme_scenario_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedGeneratedReadmePrerequisites" + in readme_scenario_summary["status"]["missingUsableBaselineEvidence"] + ) + + readme_handoff_paths = write_inputs( + root / "readme-handoff-contract", + has_official_golden=False, + ) + readme_handoff_standalone = json.loads( + readme_handoff_paths["standalone"].read_text() + ) + readme_handoff_npm = json.loads(readme_handoff_paths["npm"].read_text()) + for key in [ + "checkedReadmeHandoffContract", + "checkedReadmeOfficialEvidenceHandoff", + "checkedReadmeOfficialGoldenGap", + "checkedReadmeWaitNotifyBoundary", + ]: + readme_handoff_standalone[key] = False + readme_handoff_npm[key] = False + write_json(readme_handoff_paths["standalone"], readme_handoff_standalone) + write_json(readme_handoff_paths["npm"], readme_handoff_npm) + readme_handoff_result = run(summary_command(script, readme_handoff_paths), repo) + assert readme_handoff_result.returncode == 1 + readme_handoff_summary = parse_last_json(readme_handoff_result.stdout) + assert readme_handoff_summary["status"]["usableBaseline"] is False + assert ( + "standaloneSkillRepo.checkedReadmeHandoffContract" + in readme_handoff_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "standaloneSkillRepo.checkedReadmeOfficialGoldenGap" + in readme_handoff_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedReadmeOfficialEvidenceHandoff" + in readme_handoff_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "npmStagedSkillRepo.checkedReadmeWaitNotifyBoundary" + in readme_handoff_summary["status"]["missingUsableBaselineEvidence"] + ) + + audit_target_paths = write_inputs( + root / "audit-targets", + has_official_golden=False, + ) + audit_target_payload = json.loads( + audit_target_paths["baseline_audit_targets"].read_text() + ) + audit_target_payload["checkedBaselineAuditMakeTarget"] = False + audit_target_payload["checkedBaselineAuditCustomSummaryPath"] = False + audit_target_payload["checkedBaselineAuditIgnoresStrictSummaryVar"] = False + audit_target_payload["checkedStrictOfficialGoldenAuditMakeTarget"] = False + audit_target_payload[ + "checkedStrictOfficialGoldenAuditIgnoresBaselineSummaryVar" + ] = False + audit_target_payload[ + "checkedStrictOfficialGoldenAuditSeparateSummaryPath" + ] = False + write_json(audit_target_paths["baseline_audit_targets"], audit_target_payload) + audit_target_result = run(summary_command(script, audit_target_paths), repo) + assert audit_target_result.returncode == 1 + audit_target_summary = parse_last_json(audit_target_result.stdout) + assert audit_target_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedBaselineAuditMakeTargets" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "preflightPipelines.checkedBaselineAuditCustomSummaryPath" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "preflightPipelines.checkedBaselineAuditIgnoresStrictSummaryVar" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "preflightPipelines.checkedBaselineAuditStrictOfficialGoldenTarget" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "preflightPipelines.checkedBaselineAuditStrictOfficialGoldenIgnoresBaselineSummaryVar" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + assert ( + "preflightPipelines.checkedBaselineAuditStrictOfficialGoldenSeparateSummaryPath" + in audit_target_summary["status"]["missingUsableBaselineEvidence"] + ) + + raw_timeout_paths = write_inputs(root / "raw-timeout", has_official_golden=False) + raw_timeout_payload = json.loads(raw_timeout_paths["raw_timeout"].read_text()) + raw_timeout_payload["checkedOfficialRawStartDoesNotReturnRecordingPaths"] = False + write_json(raw_timeout_paths["raw_timeout"], raw_timeout_payload) + raw_timeout_result = run(summary_command(script, raw_timeout_paths), repo) + assert raw_timeout_result.returncode == 1 + raw_timeout_summary = parse_last_json(raw_timeout_result.stdout) + assert raw_timeout_summary["status"]["usableBaseline"] is False + assert raw_timeout_summary["status"]["officialRawStartTimeoutBoundaryVerified"] is False + assert ( + "officialRawStartTimeout.checkedOfficialRawStartDoesNotReturnRecordingPaths" + in raw_timeout_summary["status"]["missingUsableBaselineEvidence"] + ) + + candidate_handoff_paths = write_inputs( + root / "candidate-handoff", + has_official_golden=False, + ) + candidate_handoff_payload = json.loads( + candidate_handoff_paths["candidate_ingest"].read_text() + ) + candidate_handoff_payload["checkedCandidateIngestHandoffCommands"] = False + write_json(candidate_handoff_paths["candidate_ingest"], candidate_handoff_payload) + candidate_handoff_result = run( + summary_command(script, candidate_handoff_paths), + repo, + ) + assert candidate_handoff_result.returncode == 1 + candidate_handoff_summary = parse_last_json(candidate_handoff_result.stdout) + assert candidate_handoff_summary["status"]["usableBaseline"] is False + assert ( + "fixtureIngestPipelines.checkedOcuCandidateIngestHandoffCommands" + in candidate_handoff_summary["status"]["missingUsableBaselineEvidence"] + ) + + official_session_dir_paths = write_inputs( + root / "official-session-directory-handoff", + has_official_golden=False, + ) + official_session_dir_payload = json.loads( + official_session_dir_paths["official_ingest"].read_text() + ) + official_session_dir_payload["checkedOfficialSessionDirectoryPathHandoff"] = False + write_json(official_session_dir_paths["official_ingest"], official_session_dir_payload) + official_session_dir_result = run( + summary_command(script, official_session_dir_paths), + repo, + ) + assert official_session_dir_result.returncode == 1 + official_session_dir_summary = parse_last_json(official_session_dir_result.stdout) + assert official_session_dir_summary["status"]["usableBaseline"] is False + assert ( + "fixtureIngestPipelines.checkedOfficialSessionDirectoryPathHandoff" + in official_session_dir_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_paths = write_inputs(root / "preflight", has_official_golden=False) + preflight_payload = json.loads(preflight_paths["official_capture_preflight"].read_text()) + preflight_payload["checkedCapturePacketSet"] = False + write_json(preflight_paths["official_capture_preflight"], preflight_payload) + preflight_result = run(summary_command(script, preflight_paths), repo) + assert preflight_result.returncode == 1 + preflight_summary = parse_last_json(preflight_result.stdout) + assert preflight_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSet" + in preflight_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_packet_no_transcript_paths = write_inputs( + root / "preflight-no-transcript-packet", + has_official_golden=False, + ) + preflight_packet_no_transcript_payload = json.loads( + preflight_packet_no_transcript_paths["official_capture_preflight"].read_text() + ) + preflight_packet_no_transcript_payload["checkedCapturePacketNoTranscript"] = False + write_json( + preflight_packet_no_transcript_paths["official_capture_preflight"], + preflight_packet_no_transcript_payload, + ) + preflight_packet_no_transcript_result = run( + summary_command(script, preflight_packet_no_transcript_paths), + repo, + ) + assert preflight_packet_no_transcript_result.returncode == 1 + preflight_packet_no_transcript_summary = parse_last_json( + preflight_packet_no_transcript_result.stdout + ) + assert preflight_packet_no_transcript_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketNoTranscript" + in preflight_packet_no_transcript_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_handoff_scripts_paths = write_inputs( + root / "preflight-handoff-scripts", + has_official_golden=False, + ) + preflight_handoff_scripts_payload = json.loads( + preflight_handoff_scripts_paths["official_capture_preflight"].read_text() + ) + preflight_handoff_scripts_payload["checkedCapturePacketHandoffScripts"] = False + write_json( + preflight_handoff_scripts_paths["official_capture_preflight"], + preflight_handoff_scripts_payload, + ) + preflight_handoff_scripts_result = run( + summary_command(script, preflight_handoff_scripts_paths), + repo, + ) + assert preflight_handoff_scripts_result.returncode == 1 + preflight_handoff_scripts_summary = parse_last_json( + preflight_handoff_scripts_result.stdout + ) + assert preflight_handoff_scripts_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketHandoffScripts" + in preflight_handoff_scripts_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_strict_audit_handoff_paths = write_inputs( + root / "preflight-strict-audit-handoff", + has_official_golden=False, + ) + preflight_strict_audit_handoff_payload = json.loads( + preflight_strict_audit_handoff_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_strict_audit_handoff_payload[ + "checkedCapturePacketStrictAuditHandoff" + ] = False + write_json( + preflight_strict_audit_handoff_paths["official_capture_preflight"], + preflight_strict_audit_handoff_payload, + ) + preflight_strict_audit_handoff_result = run( + summary_command(script, preflight_strict_audit_handoff_paths), + repo, + ) + assert preflight_strict_audit_handoff_result.returncode == 1 + preflight_strict_audit_handoff_summary = parse_last_json( + preflight_strict_audit_handoff_result.stdout + ) + assert ( + preflight_strict_audit_handoff_summary["status"]["usableBaseline"] + is False + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketStrictAuditHandoff" + in preflight_strict_audit_handoff_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_strict_expected_failure_handoff_paths = write_inputs( + root / "preflight-strict-expected-failure-handoff", + has_official_golden=False, + ) + preflight_strict_expected_failure_handoff_payload = json.loads( + preflight_strict_expected_failure_handoff_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_strict_expected_failure_handoff_payload[ + "checkedCapturePacketStrictExpectedFailureAuditHandoff" + ] = False + write_json( + preflight_strict_expected_failure_handoff_paths[ + "official_capture_preflight" + ], + preflight_strict_expected_failure_handoff_payload, + ) + preflight_strict_expected_failure_handoff_result = run( + summary_command(script, preflight_strict_expected_failure_handoff_paths), + repo, + ) + assert preflight_strict_expected_failure_handoff_result.returncode == 1 + preflight_strict_expected_failure_handoff_summary = parse_last_json( + preflight_strict_expected_failure_handoff_result.stdout + ) + assert ( + preflight_strict_expected_failure_handoff_summary["status"][ + "usableBaseline" + ] + is False + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + in preflight_strict_expected_failure_handoff_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_ocu_candidate_output_dir_paths = write_inputs( + root / "preflight-ocu-candidate-output-dir", + has_official_golden=False, + ) + preflight_ocu_candidate_output_dir_payload = json.loads( + preflight_ocu_candidate_output_dir_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_ocu_candidate_output_dir_payload[ + "checkedCapturePacketOcuCandidateOutputDir" + ] = False + write_json( + preflight_ocu_candidate_output_dir_paths["official_capture_preflight"], + preflight_ocu_candidate_output_dir_payload, + ) + preflight_ocu_candidate_output_dir_result = run( + summary_command(script, preflight_ocu_candidate_output_dir_paths), + repo, + ) + assert preflight_ocu_candidate_output_dir_result.returncode == 1 + preflight_ocu_candidate_output_dir_summary = parse_last_json( + preflight_ocu_candidate_output_dir_result.stdout + ) + assert ( + preflight_ocu_candidate_output_dir_summary["status"]["usableBaseline"] + is False + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketOcuCandidateOutputDir" + in preflight_ocu_candidate_output_dir_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_no_transcript_paths = write_inputs( + root / "preflight-no-transcript-packet-set", + has_official_golden=False, + ) + preflight_no_transcript_payload = json.loads( + preflight_no_transcript_paths["official_capture_preflight"].read_text() + ) + preflight_no_transcript_payload["checkedCapturePacketSetNoTranscript"] = False + write_json( + preflight_no_transcript_paths["official_capture_preflight"], + preflight_no_transcript_payload, + ) + preflight_no_transcript_result = run( + summary_command(script, preflight_no_transcript_paths), + repo, + ) + assert preflight_no_transcript_result.returncode == 1 + preflight_no_transcript_summary = parse_last_json(preflight_no_transcript_result.stdout) + assert preflight_no_transcript_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetNoTranscript" + in preflight_no_transcript_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_set_candidate_handoff_paths = write_inputs( + root / "preflight-set-candidate-handoff", + has_official_golden=False, + ) + preflight_set_candidate_handoff_payload = json.loads( + preflight_set_candidate_handoff_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_set_candidate_handoff_payload[ + "checkedCapturePacketSetOcuCandidateHandoff" + ] = False + write_json( + preflight_set_candidate_handoff_paths["official_capture_preflight"], + preflight_set_candidate_handoff_payload, + ) + preflight_set_candidate_handoff_result = run( + summary_command(script, preflight_set_candidate_handoff_paths), + repo, + ) + assert preflight_set_candidate_handoff_result.returncode == 1 + preflight_set_candidate_handoff_summary = parse_last_json( + preflight_set_candidate_handoff_result.stdout + ) + assert ( + preflight_set_candidate_handoff_summary["status"]["usableBaseline"] + is False + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetOcuCandidateHandoff" + in preflight_set_candidate_handoff_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_set_contract_paths = write_inputs( + root / "preflight-set-contract-manifest", + has_official_golden=False, + ) + preflight_set_contract_payload = json.loads( + preflight_set_contract_paths["official_capture_preflight"].read_text() + ) + preflight_set_contract_payload["checkedCapturePacketSetContractManifest"] = False + write_json( + preflight_set_contract_paths["official_capture_preflight"], + preflight_set_contract_payload, + ) + preflight_set_contract_result = run( + summary_command(script, preflight_set_contract_paths), + repo, + ) + assert preflight_set_contract_result.returncode == 1 + preflight_set_contract_summary = parse_last_json( + preflight_set_contract_result.stdout + ) + assert preflight_set_contract_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetContractManifest" + in preflight_set_contract_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_packet_workflow_paths = write_inputs( + root / "preflight-packet-workflow", + has_official_golden=False, + ) + preflight_packet_workflow_payload = json.loads( + preflight_packet_workflow_paths["official_capture_preflight"].read_text() + ) + preflight_packet_workflow_payload["checkedCapturePacketPostCaptureWorkflow"] = False + preflight_packet_workflow_payload["checkedCapturePacketWorkflowVerifier"] = False + write_json( + preflight_packet_workflow_paths["official_capture_preflight"], + preflight_packet_workflow_payload, + ) + preflight_packet_workflow_result = run( + summary_command(script, preflight_packet_workflow_paths), + repo, + ) + assert preflight_packet_workflow_result.returncode == 1 + preflight_packet_workflow_summary = parse_last_json( + preflight_packet_workflow_result.stdout + ) + assert preflight_packet_workflow_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketPostCaptureWorkflow" + in preflight_packet_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketWorkflowVerifier" + in preflight_packet_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_packet_set_workflow_paths = write_inputs( + root / "preflight-packet-set-workflow", + has_official_golden=False, + ) + preflight_packet_set_workflow_payload = json.loads( + preflight_packet_set_workflow_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_packet_set_workflow_payload[ + "checkedCapturePacketSetPostCaptureWorkflow" + ] = False + preflight_packet_set_workflow_payload[ + "checkedCapturePacketSetWorkflowVerifier" + ] = False + write_json( + preflight_packet_set_workflow_paths["official_capture_preflight"], + preflight_packet_set_workflow_payload, + ) + preflight_packet_set_workflow_result = run( + summary_command(script, preflight_packet_set_workflow_paths), + repo, + ) + assert preflight_packet_set_workflow_result.returncode == 1 + preflight_packet_set_workflow_summary = parse_last_json( + preflight_packet_set_workflow_result.stdout + ) + assert preflight_packet_set_workflow_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetPostCaptureWorkflow" + in preflight_packet_set_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetWorkflowVerifier" + in preflight_packet_set_workflow_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_manifest_paths = write_inputs( + root / "preflight-transcript-manifest", + has_official_golden=False, + ) + preflight_manifest_payload = json.loads( + preflight_manifest_paths["official_capture_preflight"].read_text() + ) + preflight_manifest_payload["checkedCapturePacketTranscriptManifest"] = False + write_json( + preflight_manifest_paths["official_capture_preflight"], + preflight_manifest_payload, + ) + preflight_manifest_result = run( + summary_command(script, preflight_manifest_paths), + repo, + ) + assert preflight_manifest_result.returncode == 1 + preflight_manifest_summary = parse_last_json(preflight_manifest_result.stdout) + assert preflight_manifest_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketTranscriptManifest" + in preflight_manifest_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_make_paths = write_inputs( + root / "preflight-make-targets", + has_official_golden=False, + ) + preflight_make_payload = json.loads( + preflight_make_paths["official_capture_preflight"].read_text() + ) + preflight_make_payload["checkedMakeCapturePacketTargets"] = False + write_json( + preflight_make_paths["official_capture_preflight"], + preflight_make_payload, + ) + preflight_make_result = run(summary_command(script, preflight_make_paths), repo) + assert preflight_make_result.returncode == 1 + preflight_make_summary = parse_last_json(preflight_make_result.stdout) + assert preflight_make_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketMakeTargets" + in preflight_make_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_semantic_paths = write_inputs( + root / "preflight-semantic-guard", + has_official_golden=False, + ) + preflight_semantic_payload = json.loads( + preflight_semantic_paths["official_capture_preflight"].read_text() + ) + preflight_semantic_payload["checkedCapturePacketInputSemanticGuard"] = False + write_json( + preflight_semantic_paths["official_capture_preflight"], + preflight_semantic_payload, + ) + preflight_semantic_result = run( + summary_command(script, preflight_semantic_paths), + repo, + ) + assert preflight_semantic_result.returncode == 1 + preflight_semantic_summary = parse_last_json(preflight_semantic_result.stdout) + assert preflight_semantic_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketInputSemanticGuard" + in preflight_semantic_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_verify_paths = write_inputs( + root / "preflight-verify-inputs", + has_official_golden=False, + ) + preflight_verify_payload = json.loads( + preflight_verify_paths["official_capture_preflight"].read_text() + ) + preflight_verify_payload["checkedCapturePacketVerifyInputs"] = False + write_json( + preflight_verify_paths["official_capture_preflight"], + preflight_verify_payload, + ) + preflight_verify_result = run( + summary_command(script, preflight_verify_paths), + repo, + ) + assert preflight_verify_result.returncode == 1 + preflight_verify_summary = parse_last_json(preflight_verify_result.stdout) + assert preflight_verify_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketVerifyInputs" + in preflight_verify_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_import_guard_paths = write_inputs( + root / "preflight-import-guard", + has_official_golden=False, + ) + preflight_import_guard_payload = json.loads( + preflight_import_guard_paths["official_capture_preflight"].read_text() + ) + preflight_import_guard_payload[ + "checkedCapturePacketImportPlaceholderGuard" + ] = False + write_json( + preflight_import_guard_paths["official_capture_preflight"], + preflight_import_guard_payload, + ) + preflight_import_guard_result = run( + summary_command(script, preflight_import_guard_paths), + repo, + ) + assert preflight_import_guard_result.returncode == 1 + preflight_import_guard_summary = parse_last_json( + preflight_import_guard_result.stdout + ) + assert preflight_import_guard_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketImportPlaceholderGuard" + in preflight_import_guard_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_set_guard_paths = write_inputs( + root / "preflight-set-guard", + has_official_golden=False, + ) + preflight_set_guard_payload = json.loads( + preflight_set_guard_paths["official_capture_preflight"].read_text() + ) + preflight_set_guard_payload["checkedCapturePacketSetRootPlaceholderGuard"] = False + write_json( + preflight_set_guard_paths["official_capture_preflight"], + preflight_set_guard_payload, + ) + preflight_set_guard_result = run( + summary_command(script, preflight_set_guard_paths), + repo, + ) + assert preflight_set_guard_result.returncode == 1 + preflight_set_guard_summary = parse_last_json(preflight_set_guard_result.stdout) + assert preflight_set_guard_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetRootPlaceholderGuard" + in preflight_set_guard_summary["status"]["missingUsableBaselineEvidence"] + ) + + preflight_set_preflight_guard_paths = write_inputs( + root / "preflight-set-preflight-guard", + has_official_golden=False, + ) + preflight_set_preflight_guard_payload = json.loads( + preflight_set_preflight_guard_paths[ + "official_capture_preflight" + ].read_text() + ) + preflight_set_preflight_guard_payload[ + "checkedCapturePacketSetRootPreflightPlaceholderGuard" + ] = False + write_json( + preflight_set_preflight_guard_paths["official_capture_preflight"], + preflight_set_preflight_guard_payload, + ) + preflight_set_preflight_guard_result = run( + summary_command(script, preflight_set_preflight_guard_paths), + repo, + ) + assert preflight_set_preflight_guard_result.returncode == 1 + preflight_set_preflight_guard_summary = parse_last_json( + preflight_set_preflight_guard_result.stdout + ) + assert ( + preflight_set_preflight_guard_summary["status"]["usableBaseline"] + is False + ) + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetRootPreflightPlaceholderGuard" + in preflight_set_preflight_guard_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + preflight_set_verify_paths = write_inputs( + root / "preflight-set-verify-all", + has_official_golden=False, + ) + preflight_set_verify_payload = json.loads( + preflight_set_verify_paths["official_capture_preflight"].read_text() + ) + preflight_set_verify_payload["checkedCapturePacketSetVerifyAll"] = False + write_json( + preflight_set_verify_paths["official_capture_preflight"], + preflight_set_verify_payload, + ) + preflight_set_verify_result = run( + summary_command(script, preflight_set_verify_paths), + repo, + ) + assert preflight_set_verify_result.returncode == 1 + preflight_set_verify_summary = parse_last_json( + preflight_set_verify_result.stdout + ) + assert preflight_set_verify_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOfficialCapturePacketSetVerifyAll" + in preflight_set_verify_summary["status"][ + "missingUsableBaselineEvidence" + ] + ) + + pairing_preflight_paths = write_inputs( + root / "pairing-preflight", + has_official_golden=False, + ) + pairing_preflight_payload = json.loads( + pairing_preflight_paths["ocu_pairing_preflight"].read_text() + ) + pairing_preflight_payload["checkedPairedCandidatePreflight"] = False + write_json(pairing_preflight_paths["ocu_pairing_preflight"], pairing_preflight_payload) + pairing_preflight_result = run(summary_command(script, pairing_preflight_paths), repo) + assert pairing_preflight_result.returncode == 1 + pairing_preflight_summary = parse_last_json(pairing_preflight_result.stdout) + assert pairing_preflight_summary["status"]["usableBaseline"] is False + assert ( + "preflightPipelines.checkedOcuPairingPairedCandidatePreflight" + in pairing_preflight_summary["status"]["missingUsableBaselineEvidence"] + ) + + not_ready_paths = write_inputs(root / "not-ready", has_official_golden=True) + coverage_payload = json.loads(not_ready_paths["coverage"].read_text()) + coverage_payload["coverageOk"] = False + coverage_payload["hasRequiredOfficialSuccessfulFixture"] = False + coverage_payload["requiredOfficialReadinessOk"] = False + coverage_payload["missingOfficialScenarios"] = [] + coverage_payload["notReadyOfficialScenarios"] = ["simple-action-stop"] + coverage_payload["availableOfficialScenarios"] = ["simple-action-stop"] + write_json(not_ready_paths["coverage"], coverage_payload) + strict_not_ready = run( + summary_command(script, not_ready_paths) + ["--require-official-golden"], + repo, + ) + assert strict_not_ready.returncode == 1 + strict_not_ready_summary = parse_last_json(strict_not_ready.stdout) + assert strict_not_ready_summary["status"]["officialSuccessfulRecordingGoldenComplete"] is False + assert strict_not_ready_summary["status"]["officialGoldenGatePassed"] is False + assert strict_not_ready_summary["status"][ + "notReadyRequiredOfficialSuccessfulRecordingScenarios" + ] == ["simple-action-stop"] + assert "notReady=simple-action-stop" in strict_not_ready.stderr + + coverage_error_paths = write_inputs(root / "coverage-error", has_official_golden=True) + coverage_error_payload = json.loads(coverage_error_paths["coverage"].read_text()) + coverage_error_payload["coverageOk"] = False + coverage_error_payload["scenarioCoverageOk"] = False + coverage_error_payload["hasRequiredOfficialSuccessfulFixture"] = False + coverage_error_payload["requiredOfficialReadinessOk"] = False + coverage_error_payload["missingOfficialScenarios"] = [] + coverage_error_payload["notReadyOfficialScenarios"] = [] + coverage_error_payload["availableOfficialScenarios"] = ["simple-action-stop"] + coverage_error_payload["errors"] = [ + "official-action: fixture-manifest.json scenarioRecipe does not match scenario 'simple-action-stop'" + ] + write_json(coverage_error_paths["coverage"], coverage_error_payload) + strict_coverage_error = run( + summary_command(script, coverage_error_paths) + ["--require-official-golden"], + repo, + ) + assert strict_coverage_error.returncode == 1 + strict_coverage_error_summary = parse_last_json(strict_coverage_error.stdout) + assert strict_coverage_error_summary["status"][ + "officialSuccessfulRecordingGoldenComplete" + ] is False + assert strict_coverage_error_summary["status"]["officialGoldenGatePassed"] is False + assert strict_coverage_error_summary["status"]["officialFixtureCoverageErrors"] == [ + "official-action: fixture-manifest.json scenarioRecipe does not match scenario 'simple-action-stop'" + ] + assert strict_coverage_error_summary["evidence"]["officialFixtureSetGate"][ + "coverageErrors" + ] == strict_coverage_error_summary["status"]["officialFixtureCoverageErrors"] + assert "coverageErrors=official-action: fixture-manifest.json scenarioRecipe" in ( + strict_coverage_error.stderr + ) + + covered_paths = write_inputs(root / "covered", has_official_golden=True) + strict_covered = run( + summary_command(script, covered_paths) + ["--require-official-golden"], + repo, + ) + assert strict_covered.returncode == 0, strict_covered.stderr + strict_covered_summary = parse_last_json(strict_covered.stdout) + assert strict_covered_summary["ok"] is True + assert strict_covered_summary["status"]["strictOfficialGoldenRequired"] is True + assert strict_covered_summary["status"]["officialGoldenRequirementSatisfied"] is True + assert strict_covered_summary["status"]["officialGoldenGatePassed"] is True + assert strict_covered_summary["status"]["officialSuccessfulRecordingGoldenComplete"] is True + assert strict_covered_summary["status"]["standaloneRepoBaselineReady"] is True + assert ( + strict_covered_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] + is True + ) + strict_covered_next_action_kinds = { + action.get("kind") for action in strict_covered_summary["nextActions"] + } + assert "capture-official-successful-recording-golden" not in ( + strict_covered_next_action_kinds + ) + assert "scaffold-standalone-record-and-replay-repo" in ( + strict_covered_next_action_kinds + ) + + print( + json.dumps( + { + "ok": True, + "checkedDefaultAllowsMissingOfficialGolden": True, + "checkedIncompleteUsableBaselineEvidenceFails": True, + "checkedRealInputActionConcreteEvidence": True, + "checkedWaitNotifyCallbackFailureEvidence": True, + "checkedWaitNotifyCallbackTimeoutEvidence": True, + "checkedRuntimeTimeoutDiagnosticsEvidence": True, + "checkedNpmPythonLauncherDiagnosticsEvidence": True, + "checkedGeneratedReadmePrerequisitesEvidence": True, + "checkedReadmeHandoffContractEvidence": True, + "checkedCapturePacketHandoffScriptsEvidence": True, + "checkedCapturePacketStrictAuditHandoffEvidence": True, + "checkedCapturePacketOcuCandidateOutputDirEvidence": True, + "checkedCapturePacketSetOcuCandidateHandoffEvidence": True, + "checkedCandidateIngestHandoffCommandsEvidence": True, + "checkedStandaloneLifecycleSemanticsEvidence": True, + "checkedStandaloneRecordingToSkillConcreteEvidence": True, + "checkedNpmRecordingToSkillConcreteEvidence": True, + "checkedBaselineAuditTargetEvidence": True, + "checkedStrictCoverageErrorsReported": True, + "checkedStrictNotReadyOfficialGoldenFails": True, + "checkedStrictMissingOfficialGoldenFails": True, + "checkedStrictCoveredOfficialGoldenPasses": True, + "checkedNextActionPacketCommandsGeneratePackets": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py b/scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py new file mode 100755 index 00000000..e7fa385b --- /dev/null +++ b/scripts/test-record-and-replay-ocu-candidate-pairing-preflight.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import DEFAULT_REQUIRED_SCENARIOS, scenario_recipe + + +def write_json(path, value): + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path, records): + path.write_text("\n".join(json.dumps(record, separators=(",", ":")) for record in records) + "\n") + + +def make_recording(root: pathlib.Path, session_id: str, source_label: str) -> pathlib.Path: + recording = root / session_id + recording.mkdir(parents=True) + metadata_path = recording / "metadata.json" + session_path = recording / "session.json" + events_path = recording / "events.jsonl" + suppressed_path = recording / "suppressed.jsonl" + events = [ + {"type": "session.started", "sessionId": session_id}, + { + "type": "AX.focusedWindowChanged", + "sessionId": session_id, + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "treeLines": [f"AXWindow {source_label}", "+ AXButton Pair"], + }, + }, + { + "type": "mouse.click", + "sessionId": session_id, + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": {"role": "AXButton", "title": "Pair"}, + }, + { + "type": "session.ended", + "sessionId": session_id, + "endReason": "recording_controls_stopped", + }, + ] + write_jsonl(events_path, events) + write_jsonl(suppressed_path, []) + metadata = { + "sessionId": session_id, + "state": "stopped", + "active": False, + "endReason": "recording_controls_stopped", + "eventCount": len(events), + "suppressedEventCount": 0, + "metadataPath": str(metadata_path), + "sessionPath": str(session_path), + "eventsPath": str(events_path), + "suppressedEventsPath": str(suppressed_path), + } + write_json(metadata_path, metadata) + write_json(session_path, metadata) + shape = { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps(metadata), + } + ] + } + } + write_json( + recording / "mcp-transcript.json", + { + "startResponseShape": shape, + "repeatStartResponseShape": shape, + "statusResponseShape": shape, + "stopResponseShape": shape, + "repeatStopResponseShape": shape, + "finalStatusResponseShape": shape, + }, + ) + return recording + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def import_fixture(repo: pathlib.Path, recording: pathlib.Path, output: pathlib.Path, source: str): + completed = run( + [ + sys.executable, + str(repo / "scripts/import-event-stream-fixture.py"), + str(recording), + "--name", + f"{source}-simple-action", + "--output-dir", + str(output), + "--source", + source, + "--scenario", + "simple-action-stop", + "--official-plugin-version", + "record-and-replay 1.0.857", + "--mcp-transcript", + str(recording / "mcp-transcript.json"), + ], + repo, + ) + assert completed.returncode == 0, completed.stderr + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + script = repo / "scripts/prepare-record-and-replay-ocu-candidate-pairing.py" + default_required_scenario = DEFAULT_REQUIRED_SCENARIOS[0] + with tempfile.TemporaryDirectory() as tmp: + root = pathlib.Path(tmp) + official_root = root / "official" + candidate_root = root / "candidates" + + missing = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert missing.returncode == 0, missing.stderr + missing_json = json.loads(missing.stdout) + assert missing_json["scenarioStatus"]["officialFixtureReady"] is False + assert missing_json["scenarioRecipe"]["scenario"] == default_required_scenario + assert missing_json["scenarioRecipe"]["expectedActionEvents"] == ["mouse.click"] + assert missing_json["scenarioRecipe"]["ocuCandidateSourceKind"] == "run-action-smoke" + assert "Import a ready official fixture" in missing_json["nextActions"][0] + assert missing_json["commands"]["sourceKind"] == "run-action-smoke" + assert "--run-action-smoke" in missing_json["commands"]["ingestCandidate"] + + official_recording = make_recording(root / "recordings", "official-session", "Official") + import_fixture(repo, official_recording, official_root, "official") + no_candidate = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert no_candidate.returncode == 0, no_candidate.stderr + no_candidate_json = json.loads(no_candidate.stdout) + assert no_candidate_json["scenarioStatus"]["officialFixtureReady"] is True + assert no_candidate_json["scenarioStatus"]["candidateFixturePresent"] is False + assert "commands.ingestCandidate" in no_candidate_json["nextActions"][0] + + candidate_recording = make_recording(root / "recordings", "candidate-session", "Official") + import_fixture(repo, candidate_recording, candidate_root, "ocu") + paired = run( + [ + sys.executable, + str(script), + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + "--require-candidate-ready", + ], + repo, + ) + assert paired.returncode == 0, paired.stderr + paired_json = json.loads(paired.stdout) + assert paired_json["scenarioStatus"]["candidateFixtureReady"] is True + assert paired_json["scenarioStatus"]["candidateComparePassed"] is True + + keyboard = run( + [ + sys.executable, + str(script), + "--scenario", + "keyboard-input-stop", + "--official-root", + str(official_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert keyboard.returncode == 0, keyboard.stderr + keyboard_json = json.loads(keyboard.stdout) + assert keyboard_json["commands"]["sourceKind"] == "recording-required" + assert keyboard_json["scenarioRecipe"]["expectedActionEvents"] == ["keyboard.text_input"] + assert keyboard_json["scenarioRecipe"]["ocuCandidateSourceKind"] == "recording-required" + assert "cannot use synthetic --run-action-smoke" in keyboard_json["nextActions"][-1] + + recipe_drift_root = root / "recipe-drift-official" + recipe_drift_fixture = recipe_drift_root / "official-action" + recipe_drift_fixture.mkdir(parents=True) + recipe = scenario_recipe("simple-action-stop") + recipe["expectedActionEvents"] = ["keyboard.text_input"] + write_json( + recipe_drift_fixture / "fixture-manifest.json", + { + "fixtureFormatVersion": 1, + "name": "official-action", + "source": "official", + "scenario": "simple-action-stop", + "scenarioRecipe": recipe, + "officialPluginVersion": "record-and-replay 1.0.857", + "eventCount": 4, + }, + ) + coverage_error = run( + [ + sys.executable, + str(script), + "--official-root", + str(recipe_drift_root), + "--candidate-root", + str(candidate_root), + ], + repo, + ) + assert coverage_error.returncode == 0, coverage_error.stderr + coverage_error_json = json.loads(coverage_error.stdout) + assert coverage_error_json["ok"] is False + assert coverage_error_json["officialCoverageErrors"] + assert any( + "scenarioRecipe does not match scenario" in error + for error in coverage_error_json["officialCoverageErrors"] + ) + assert "Fix official fixture coverage errors" in coverage_error_json["nextActions"][0] + + print( + json.dumps( + { + "ok": True, + "checkedMissingOfficialPreflight": True, + "checkedNoCandidatePreflight": True, + "checkedPairedCandidatePreflight": True, + "checkedKeyboardRecordingRequiredScenario": True, + "checkedCoverageErrorReport": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-official-capture-finalizer.py b/scripts/test-record-and-replay-official-capture-finalizer.py new file mode 100755 index 00000000..5e9fb709 --- /dev/null +++ b/scripts/test-record-and-replay-official-capture-finalizer.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + + +def write_json(path: pathlib.Path, value) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value, indent=2, sort_keys=True) + "\n") + + +def write_jsonl(path: pathlib.Path, values) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(json.dumps(value, sort_keys=True) + "\n" for value in values)) + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def hosted_tool_json(payload): + return { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps(payload, sort_keys=True), + } + ] + } + } + + +def make_recording(root: pathlib.Path) -> dict[str, pathlib.Path]: + recording = root / "official-recording" + metadata = recording / "metadata.json" + session = recording / "session.json" + events = recording / "events.jsonl" + suppressed = recording / "suppressed.jsonl" + event_records = [ + {"type": "session.started", "sessionId": "official-secret-session"}, + { + "type": "mouse.click", + "sessionId": "official-secret-session", + "button": "left", + "location": {"x": 10, "y": 20}, + "targetAccessibilityElement": { + "role": "AXButton", + "title": "Safe Button", + }, + }, + { + "type": "AX.focusedWindowChanged", + "sessionId": "official-secret-session", + "accessibilityInspectorPayload": { + "kind": "full", + "diffFromPrevious": False, + "treeLines": ["AXWindow Safe"], + "renderedText": "AXWindow Safe", + }, + }, + { + "type": "session.ended", + "sessionId": "official-secret-session", + "endReason": "recording_controls_stopped", + }, + ] + suppressed_records = [ + { + "type": "AX.snapshot.suppressed", + "reason": "snapshotTooLarge", + "subsystem": "accessibility", + } + ] + metadata_payload = { + "sessionId": "official-secret-session", + "state": "stopped", + "endReason": "recording_controls_stopped", + "eventCount": len(event_records), + "suppressedEventCount": len(suppressed_records), + "metadataPath": str(metadata), + "sessionPath": str(session), + "eventsPath": str(events), + "suppressedEventsPath": str(suppressed), + } + write_json(metadata, metadata_payload) + write_json(session, metadata_payload) + write_jsonl(events, event_records) + write_jsonl(suppressed, suppressed_records) + return { + "recording": recording, + "metadata": metadata, + "session": session, + "events": events, + "suppressed": suppressed, + } + + +def main() -> int: + repo = pathlib.Path(__file__).resolve().parents[1] + prepare = repo / "scripts/prepare-record-and-replay-official-golden-capture.py" + finalizer = repo / "scripts/finalize-record-and-replay-official-capture-packet.py" + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fixture_root = tmp_path / "fixtures" + plugin_root = tmp_path / "record-and-replay" + (plugin_root / "1.0.857/skills/record-and-replay").mkdir(parents=True) + (plugin_root / "1.0.857/skills/record-and-replay/SKILL.md").write_text( + "---\nname: record-and-replay\n---\n" + ) + packet_dir = tmp_path / "packet" + prepared = run( + [ + sys.executable, + str(prepare), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--capture-packet-dir", + str(packet_dir), + ], + repo, + ) + assert prepared.returncode == 0, prepared.stderr + paths = make_recording(tmp_path) + start_json = tmp_path / "start.json" + status_json = tmp_path / "status.json" + stop_json = tmp_path / "stop.json" + final_status_json = tmp_path / "final-status.json" + start_payload = { + "isRecording": True, + "sessionId": "official-secret-session", + "metadataPath": str(paths["metadata"]), + "eventsPath": str(paths["events"]), + } + status_payload = { + "isRecording": True, + "sessionId": "official-secret-session", + "metadataPath": str(paths["metadata"]), + "eventsPath": str(paths["events"]), + } + stop_payload = { + "isRecording": False, + "sessionId": "official-secret-session", + "metadataPath": str(paths["metadata"]), + "sessionPath": str(paths["session"]), + "eventsPath": str(paths["events"]), + } + write_json(start_json, hosted_tool_json(start_payload)) + write_json(status_json, hosted_tool_json(status_payload)) + write_json(stop_json, hosted_tool_json(stop_payload)) + write_json(final_status_json, hosted_tool_json(stop_payload)) + + finalized = run( + [ + sys.executable, + str(finalizer), + "--packet-dir", + str(packet_dir), + "--start-json", + str(start_json), + "--status-json", + str(status_json), + "--stop-json", + str(stop_json), + "--final-status-json", + str(final_status_json), + ], + repo, + ) + assert finalized.returncode == 0, finalized.stderr + finalized_payload = json.loads(finalized.stdout) + assert finalized_payload["ok"] is True + assert finalized_payload["checkedHostedResponseShapes"] == [ + "startResponseShape", + "statusResponseShape", + "stopResponseShape", + "finalStatusResponseShape", + ] + assert finalized_payload["verifyWorkflow"]["ok"] is True + assert finalized_payload["verifyInputs"]["ok"] is True + transcript = json.loads((packet_dir / "inputs/mcp-transcript.json").read_text()) + assert transcript["kind"] == "record-and-replay-hosted-capture-transcript" + assert transcript["startResponseShape"]["result"]["content"][0]["type"] == "text" + assert transcript["statusResponseShape"]["result"]["content"][0]["type"] == "text" + assert transcript["stopResponseShape"]["result"]["content"][0]["type"] == "text" + assert transcript["finalStatusResponseShape"]["result"]["content"][0]["type"] == "text" + status_input = json.loads((packet_dir / "inputs/event_stream_stop-response.json").read_text()) + status_input_text = status_input["result"]["content"][0]["text"] + assert "eventsPath" in status_input_text + assert "metadataPath" in status_input_text + assert "suppressedEventsPath" not in status_input_text + + missing_final = run( + [ + sys.executable, + str(finalizer), + "--packet-dir", + str(packet_dir), + "--start-json", + str(start_json), + "--status-json", + str(status_json), + "--stop-json", + str(stop_json), + ], + repo, + ) + assert missing_final.returncode == 1 + assert "finalStatusResponseShape" in missing_final.stderr + + no_handoff = tmp_path / "no-handoff.json" + write_json(no_handoff, hosted_tool_json({"isRecording": False})) + no_handoff_result = run( + [ + sys.executable, + str(finalizer), + "--packet-dir", + str(packet_dir), + "--start-json", + str(start_json), + "--status-json", + str(status_json), + "--stop-json", + str(no_handoff), + "--final-status-json", + str(final_status_json), + ], + repo, + ) + assert no_handoff_result.returncode == 1 + assert "handoff paths" in no_handoff_result.stderr + + print(json.dumps({"ok": True, "checkedHostedCaptureFinalizer": True}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test-record-and-replay-official-golden-capture-preflight.py b/scripts/test-record-and-replay-official-golden-capture-preflight.py new file mode 100755 index 00000000..ac44406a --- /dev/null +++ b/scripts/test-record-and-replay-official-golden-capture-preflight.py @@ -0,0 +1,965 @@ +#!/usr/bin/env python3 + +import json +import pathlib +import subprocess +import sys +import tempfile + +from record_and_replay_scenarios import DEFAULT_REQUIRED_SCENARIOS, scenario_recipe + + +def run(command, cwd): + return subprocess.run( + command, + cwd=cwd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + +def main(): + repo = pathlib.Path(__file__).resolve().parents[1] + script = repo / "scripts/prepare-record-and-replay-official-golden-capture.py" + default_required_scenario = DEFAULT_REQUIRED_SCENARIOS[0] + with tempfile.TemporaryDirectory() as tmp: + tmp_path = pathlib.Path(tmp) + fixture_root = tmp_path / "fixtures" + plugin_root = tmp_path / "record-and-replay" + skill_dir = plugin_root / "1.0.857/skills/record-and-replay" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\nname: record-and-replay\n---\n") + + missing = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--name", + "official-simple-action-stop-1.0.857", + ], + repo, + ) + assert missing.returncode == 0, missing.stderr + missing_json = json.loads(missing.stdout) + assert missing_json["ok"] is True + assert missing_json["stage"] == "preflight" + assert missing_json["officialPlugin"]["available"] is True + assert missing_json["officialPlugin"]["versions"][0]["version"] == "1.0.857" + assert missing_json["scenarioStatus"] == { + "available": False, + "missing": True, + "notReady": False, + "ready": False, + "scenario": default_required_scenario, + } + assert missing_json["coverage"]["missingOfficialScenarios"] == [default_required_scenario] + assert missing_json["scenarioRecipe"]["priority"] == "required" + assert missing_json["scenarioRecipe"]["expectedActionEvents"] == ["mouse.click"] + assert missing_json["scenarioRecipe"]["expectedEndReason"] == "recording_controls_stopped" + assert missing_json["scenarioRecipe"]["ocuCandidateSourceKind"] == "run-action-smoke" + assert "commands.inspectOnly" in " ".join(missing_json["nextActions"]) + inspect_cmd = missing_json["commands"]["inspectOnly"] + assert "--inspect-only" in inspect_cmd + assert "--mcp-transcript" in inspect_cmd + assert missing_json["commands"]["strictOfficialGoldenGate"] == [ + "make", + "record-and-replay-official-golden-gate-audit", + ] + assert ( + missing_json["commands"]["strictOfficialGoldenGateShell"] + == "make record-and-replay-official-golden-gate-audit" + ) + assert missing_json["commands"]["strictOfficialGoldenExpectedFailureAudit"] == [ + "python3", + "scripts/check-record-and-replay-baseline-summary.py", + "dist/record-and-replay-official-golden-gate-summary.json", + "--allow-strict-official-golden-missing", + ] + assert "--allow-strict-official-golden-missing" in ( + missing_json["commands"]["strictOfficialGoldenExpectedFailureAuditShell"] + ) + assert missing_json["commands"]["inspectOnlyShell"].startswith("python3 ") + assert missing_json["commands"]["ocuCandidate"][1] == "scripts/ingest-ocu-record-and-replay-candidate.py" + assert "--run-action-smoke" in missing_json["commands"]["ocuCandidate"] + assert "--output-dir" in missing_json["commands"]["ocuCandidate"] + assert str(fixture_root / "ocu-candidates") in missing_json["commands"]["ocuCandidate"] + assert "--official-root" in missing_json["commands"]["ocuCandidate"] + assert str(fixture_root) in missing_json["commands"]["ocuCandidate"] + + packet_dir = tmp_path / "capture-packet" + capture_packet = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--name", + "official-simple-action-stop-1.0.857", + "--capture-packet-dir", + str(packet_dir), + ], + repo, + ) + assert capture_packet.returncode == 0, capture_packet.stderr + capture_packet_json = json.loads(capture_packet.stdout) + assert pathlib.Path(capture_packet_json["capturePacket"]["path"]).resolve() == packet_dir.resolve() + assert capture_packet_json["capturePacket"]["includeTranscript"] is True + assert capture_packet_json["capturePacket"]["requiresMcpTranscriptInput"] is True + assert capture_packet_json["capturePacket"]["statusResponseInputPath"].endswith( + "inputs/event_stream_stop-response.json" + ) + assert capture_packet_json["capturePacket"]["mcpTranscriptInputPath"].endswith( + "inputs/mcp-transcript.json" + ) + assert any( + "Replace capture packet input placeholders" in action + for action in capture_packet_json["nextActions"] + ) + assert any( + "capturePacket.verifyInputsShell" in action + for action in capture_packet_json["nextActions"] + ) + assert "verifyInputsShell" in capture_packet_json["capturePacket"] + assert capture_packet_json["capturePacket"]["verifyWorkflowShell"].endswith( + "verify-workflow.sh" + ) + assert capture_packet_json["capturePacket"]["checkFixtureSetShell"].endswith( + "check-fixture-set.sh" + ) + assert capture_packet_json["capturePacket"]["strictGoldenGateShell"].endswith( + "strict-golden-gate.sh" + ) + assert capture_packet_json["capturePacket"][ + "strictExpectedFailureAuditShell" + ].endswith("strict-expected-failure-audit.sh") + assert capture_packet_json["capturePacket"]["ingestOcuCandidateShell"].endswith( + "ingest-ocu-candidate.sh" + ) + assert "--output-dir" in capture_packet_json["commands"]["ocuCandidate"] + assert str(fixture_root / "ocu-candidates") in capture_packet_json["commands"]["ocuCandidate"] + assert (packet_dir / "README.md").exists() + assert (packet_dir / "preflight.json").exists() + assert (packet_dir / "capture-contract.json").exists() + assert (packet_dir / "scenario-recipe.json").exists() + assert (packet_dir / "inputs/event_stream_stop-response.json").exists() + assert (packet_dir / "inputs/mcp-transcript.json").exists() + assert (packet_dir / "verify-inputs.sh").exists() + assert (packet_dir / "verify-workflow.sh").exists() + assert (packet_dir / "inspect-only.sh").exists() + assert (packet_dir / "import-fixture.sh").exists() + assert (packet_dir / "check-coverage.sh").exists() + assert (packet_dir / "check-fixture-set.sh").exists() + assert (packet_dir / "strict-golden-gate.sh").exists() + assert (packet_dir / "strict-expected-failure-audit.sh").exists() + assert (packet_dir / "ingest-ocu-candidate.sh").exists() + assert (packet_dir / "verify-inputs.sh").stat().st_mode & 0o111 + assert (packet_dir / "verify-workflow.sh").stat().st_mode & 0o111 + assert (packet_dir / "inspect-only.sh").stat().st_mode & 0o111 + packet_readme = (packet_dir / "README.md").read_text() + assert "Record & Replay Official Capture Packet" in packet_readme + assert "MCP transcript input required: `yes`" in packet_readme + assert "`capture-contract.json` records the expected scenario" in packet_readme + assert "`postCaptureWorkflow`" in packet_readme + assert "`inputs/mcp-transcript.json`: required same-session MCP transcript evidence." in ( + packet_readme + ) + assert "Do not commit raw official recording JSON" in packet_readme + assert "./verify-inputs.sh" in packet_readme + assert "./verify-workflow.sh" in packet_readme + assert "finalize-record-and-replay-official-capture-packet.py" in packet_readme + assert "--final-status-json" in packet_readme + assert "`verify-workflow.sh` checks" in packet_readme + assert "./check-fixture-set.sh" in packet_readme + assert "./strict-golden-gate.sh" in packet_readme + assert "./strict-expected-failure-audit.sh" in packet_readme + assert "./ingest-ocu-candidate.sh" in packet_readme + assert "Equivalent fixture-set gate" in packet_readme + assert "Equivalent strict golden gate audit" in packet_readme + assert "Equivalent strict expected-failure audit" in packet_readme + assert "make record-and-replay-official-golden-gate-audit" in packet_readme + assert "--allow-strict-official-golden-missing" in packet_readme + assert "embed the repository path captured when this packet was created" in packet_readme + assert "REPO_ROOT=/path/to/repo" in packet_readme + assert "git rev-parse" not in packet_readme + packet_recipe = json.loads((packet_dir / "scenario-recipe.json").read_text()) + assert packet_recipe == scenario_recipe(default_required_scenario) + packet_contract = json.loads((packet_dir / "capture-contract.json").read_text()) + assert packet_contract["formatVersion"] == 1 + assert packet_contract["scenario"] == default_required_scenario + assert packet_contract["expectedActionEvents"] == ["mouse.click"] + assert packet_contract["expectedEndReason"] == "recording_controls_stopped" + assert packet_contract["requiresMcpTranscriptInput"] is True + assert packet_contract["requiredStatusHandoffPathKeys"] == ["eventsPath"] + assert packet_contract["requiredStatusHandoffAnyOf"] == [ + ["metadataPath", "sessionPath"] + ] + assert "suppressedEventsPath" in packet_contract["optionalStatusHandoffPathKeys"] + assert "startResponseShape" in packet_contract["requiredTranscriptEvidenceKeys"] + assert "statusResponseShape" in packet_contract["requiredTranscriptEvidenceKeys"] + assert "stopResponseShape" in packet_contract["requiredTranscriptEvidenceKeys"] + assert "finalStatusResponseShape" in packet_contract["requiredTranscriptEvidenceKeys"] + assert "repeatStartResponseShape" not in packet_contract["requiredTranscriptEvidenceKeys"] + workflow_ids = [step["id"] for step in packet_contract["postCaptureWorkflow"]] + assert workflow_ids == [ + "replace-status-json", + "replace-mcp-transcript", + "verify-inputs", + "inspect-only", + "import-fixture", + "check-coverage", + "check-fixture-set", + "ingest-ocu-candidate", + "strict-golden-gate", + "strict-expected-failure-audit", + ] + assert capture_packet_json["capturePacket"]["postCaptureWorkflow"] == ( + packet_contract["postCaptureWorkflow"] + ) + assert capture_packet_json["capturePacket"]["captureContractPath"].endswith( + "capture-contract.json" + ) + packet_placeholder = json.loads( + (packet_dir / "inputs/event_stream_stop-response.json").read_text() + ) + assert packet_placeholder["_placeholder"] is True + verify_script = (packet_dir / "verify-inputs.sh").read_text() + assert "Invalid packet JSON" in verify_script + assert "Packet input still contains placeholder" in verify_script + workflow_verify = run([str(packet_dir / "verify-workflow.sh")], repo) + assert workflow_verify.returncode == 0, workflow_verify.stderr + workflow_verify_json = json.loads(workflow_verify.stdout) + assert workflow_verify_json["ok"] is True + assert workflow_verify_json["checkedWorkflow"] is True + assert workflow_verify_json["checkedWorkflowCommands"] is True + assert workflow_verify_json["checkedWorkflowInputs"] is True + mutated_contract = json.loads((packet_dir / "capture-contract.json").read_text()) + for step in mutated_contract["postCaptureWorkflow"]: + if step["id"] == "inspect-only": + step["command"] = "./missing-inspect-only.sh" + (packet_dir / "capture-contract.json").write_text( + json.dumps(mutated_contract, indent=2, sort_keys=True) + "\n" + ) + workflow_drift = run([str(packet_dir / "verify-workflow.sh")], repo) + assert workflow_drift.returncode == 2 + assert "Workflow command" in workflow_drift.stderr + assert "missing" in workflow_drift.stderr + (packet_dir / "capture-contract.json").write_text( + json.dumps(packet_contract, indent=2, sort_keys=True) + "\n" + ) + inspect_script = (packet_dir / "inspect-only.sh").read_text() + assert 'default_repo_root=' in inspect_script + assert '"${packet_dir}/inputs/event_stream_stop-response.json"' in inspect_script + assert '"${packet_dir}/inputs/mcp-transcript.json"' in inspect_script + assert "check_replaced_input" in inspect_script + import_script = (packet_dir / "import-fixture.sh").read_text() + assert "check_replaced_input" in import_script + strict_gate_script = (packet_dir / "strict-golden-gate.sh").read_text() + assert "record-and-replay-official-golden-gate-audit" in strict_gate_script + assert "record-and-replay-official-golden-gate\n" not in strict_gate_script + strict_expected_failure_script = ( + packet_dir / "strict-expected-failure-audit.sh" + ).read_text() + assert "check-record-and-replay-baseline-summary.py" in strict_expected_failure_script + assert "--allow-strict-official-golden-missing" in strict_expected_failure_script + placeholder_verify = run([str(packet_dir / "verify-inputs.sh")], repo) + assert placeholder_verify.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_verify.stderr + ) + placeholder_inspect = run([str(packet_dir / "inspect-only.sh")], repo) + assert placeholder_inspect.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_inspect.stderr + ) + assert "event_stream_stop-response.json" in placeholder_inspect.stderr + placeholder_import = run([str(packet_dir / "import-fixture.sh")], repo) + assert placeholder_import.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_import.stderr + ) + assert "event_stream_stop-response.json" in placeholder_import.stderr + (packet_dir / "inputs/event_stream_stop-response.json").write_text("{}\n") + placeholder_transcript_verify = run([str(packet_dir / "verify-inputs.sh")], repo) + assert placeholder_transcript_verify.returncode == 2 + assert "Packet input still contains placeholder: MCP transcript" in ( + placeholder_transcript_verify.stderr + ) + placeholder_transcript_inspect = run([str(packet_dir / "inspect-only.sh")], repo) + assert placeholder_transcript_inspect.returncode == 2 + assert "Packet input still contains placeholder: MCP transcript" in ( + placeholder_transcript_inspect.stderr + ) + assert "mcp-transcript.json" in placeholder_transcript_inspect.stderr + placeholder_transcript_import = run([str(packet_dir / "import-fixture.sh")], repo) + assert placeholder_transcript_import.returncode == 2 + assert "Packet input still contains placeholder: MCP transcript" in ( + placeholder_transcript_import.stderr + ) + assert "mcp-transcript.json" in placeholder_transcript_import.stderr + (packet_dir / "inputs/mcp-transcript.json").write_text("{}\n") + semantic_status_verify = run([str(packet_dir / "verify-inputs.sh")], repo) + assert semantic_status_verify.returncode == 2 + assert "Packet status JSON does not contain Record & Replay handoff path evidence" in ( + semantic_status_verify.stderr + ) + (packet_dir / "inputs/event_stream_stop-response.json").write_text( + json.dumps( + { + "result": { + "content": [ + { + "type": "text", + "text": json.dumps( + { + "metadataPath": "/tmp/official-session/metadata.json", + "sessionPath": "/tmp/official-session/session.json", + "eventsPath": "/tmp/official-session/events.jsonl", + } + ), + } + ] + } + } + ) + + "\n" + ) + semantic_transcript_verify = run([str(packet_dir / "verify-inputs.sh")], repo) + assert semantic_transcript_verify.returncode == 2 + assert "Packet MCP transcript does not contain MCP transcript evidence" in ( + semantic_transcript_verify.stderr + ) + (packet_dir / "inputs/mcp-transcript.json").write_text( + json.dumps({"stopResponseShape": {"result": {"content": []}}}) + "\n" + ) + replaced_inputs_verify = run([str(packet_dir / "verify-inputs.sh")], repo) + assert replaced_inputs_verify.returncode == 0, replaced_inputs_verify.stderr + replaced_inputs_verify_json = json.loads(replaced_inputs_verify.stdout) + assert replaced_inputs_verify_json["ok"] is True + assert replaced_inputs_verify_json["stage"] == "verify-inputs" + assert replaced_inputs_verify_json["foundHandoffPathKeys"] == [ + "eventsPath", + "metadataPath", + "sessionPath", + ] + assert replaced_inputs_verify_json["requiredStatusHandoffPathKeys"] == ["eventsPath"] + assert replaced_inputs_verify_json["requiredStatusHandoffAnyOf"] == [ + ["metadataPath", "sessionPath"] + ] + assert replaced_inputs_verify_json["foundTranscriptEvidenceKeys"] == [ + "stopResponseShape" + ] + coverage_script = (packet_dir / "check-coverage.sh").read_text() + assert "check_replaced_input" not in coverage_script + + no_transcript_packet_dir = tmp_path / "capture-packet-no-transcript" + no_transcript_packet = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--name", + "official-simple-action-stop-1.0.857", + "--capture-packet-dir", + str(no_transcript_packet_dir), + "--no-include-transcript", + ], + repo, + ) + assert no_transcript_packet.returncode == 0, no_transcript_packet.stderr + no_transcript_packet_json = json.loads(no_transcript_packet.stdout) + assert no_transcript_packet_json["capturePacket"]["includeTranscript"] is False + assert no_transcript_packet_json["capturePacket"]["requiresMcpTranscriptInput"] is False + no_transcript_workflow_ids = [ + step["id"] for step in no_transcript_packet_json["capturePacket"]["postCaptureWorkflow"] + ] + assert "replace-mcp-transcript" not in no_transcript_workflow_ids + assert no_transcript_workflow_ids[0] == "replace-status-json" + assert no_transcript_workflow_ids[-1] == "strict-expected-failure-audit" + assert ( + no_transcript_packet_json["capturePacket"]["statusResponseInputPath"].endswith( + "inputs/event_stream_stop-response.json" + ) + ) + assert no_transcript_packet_json["capturePacket"]["mcpTranscriptInputPath"] is None + assert no_transcript_packet_json["capturePacket"]["checkFixtureSetShell"].endswith( + "check-fixture-set.sh" + ) + assert no_transcript_packet_json["capturePacket"]["strictGoldenGateShell"].endswith( + "strict-golden-gate.sh" + ) + assert no_transcript_packet_json["capturePacket"][ + "strictExpectedFailureAuditShell" + ].endswith("strict-expected-failure-audit.sh") + assert (no_transcript_packet_dir / "inputs/event_stream_stop-response.json").exists() + assert not (no_transcript_packet_dir / "inputs/mcp-transcript.json").exists() + assert (no_transcript_packet_dir / "capture-contract.json").exists() + assert (no_transcript_packet_dir / "verify-workflow.sh").exists() + no_transcript_workflow_verify = run( + [str(no_transcript_packet_dir / "verify-workflow.sh")], + repo, + ) + assert no_transcript_workflow_verify.returncode == 0, ( + no_transcript_workflow_verify.stderr + ) + no_transcript_packet_readme = (no_transcript_packet_dir / "README.md").read_text() + assert "MCP transcript input required: `no`" in no_transcript_packet_readme + assert "does not create `inputs/mcp-transcript.json`" in no_transcript_packet_readme + assert "--mcp-transcript" not in (no_transcript_packet_dir / "inspect-only.sh").read_text() + assert "--mcp-transcript" not in (no_transcript_packet_dir / "import-fixture.sh").read_text() + no_transcript_placeholder_verify = run( + [str(no_transcript_packet_dir / "verify-inputs.sh")], + repo, + ) + assert no_transcript_placeholder_verify.returncode == 2 + assert "event_stream_stop response" in no_transcript_placeholder_verify.stderr + assert "MCP transcript" not in no_transcript_placeholder_verify.stderr + (no_transcript_packet_dir / "inputs/event_stream_stop-response.json").write_text( + json.dumps( + { + "metadataPath": "/tmp/official-session/metadata.json", + "eventsPath": "/tmp/official-session/events.jsonl", + } + ) + + "\n" + ) + no_transcript_replaced_verify = run( + [str(no_transcript_packet_dir / "verify-inputs.sh")], + repo, + ) + assert no_transcript_replaced_verify.returncode == 0, ( + no_transcript_replaced_verify.stderr + ) + + packet_set_dir = tmp_path / "capture-packet-set" + capture_packet_set = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--capture-packet-dir", + str(packet_set_dir), + "--capture-packet-recommended-scenarios", + ], + repo, + ) + assert capture_packet_set.returncode == 0, capture_packet_set.stderr + capture_packet_set_json = json.loads(capture_packet_set.stdout) + assert capture_packet_set_json["stage"] == "capture-packet-set" + assert capture_packet_set_json["scenarios"] == [ + "simple-action-stop", + "keyboard-input-stop", + "drag-stop", + "cancel", + "timeout", + ] + assert (packet_set_dir / "README.md").exists() + assert (packet_set_dir / "capture-packets.json").exists() + assert (packet_set_dir / "verify-all.sh").exists() + assert (packet_set_dir / "verify-workflow.sh").exists() + assert (packet_set_dir / "inspect-all.sh").exists() + assert (packet_set_dir / "import-all.sh").exists() + assert (packet_set_dir / "check-all.sh").exists() + assert (packet_set_dir / "ingest-ocu-candidates.sh").exists() + assert (packet_set_dir / "strict-expected-failure-audit.sh").exists() + assert (packet_set_dir / "verify-all.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "verify-workflow.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "inspect-all.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "import-all.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "check-all.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "ingest-ocu-candidates.sh").stat().st_mode & 0o111 + assert (packet_set_dir / "strict-expected-failure-audit.sh").stat().st_mode & 0o111 + for scenario in capture_packet_set_json["scenarios"]: + scenario_dir = packet_set_dir / scenario + assert (scenario_dir / "README.md").exists() + assert (scenario_dir / "preflight.json").exists() + assert (scenario_dir / "capture-contract.json").exists() + assert (scenario_dir / "scenario-recipe.json").exists() + assert (scenario_dir / "inputs/event_stream_stop-response.json").exists() + assert (scenario_dir / "verify-inputs.sh").exists() + assert (scenario_dir / "inspect-only.sh").exists() + assert (scenario_dir / "import-fixture.sh").exists() + assert (packet_set_dir / "simple-action-stop/ingest-ocu-candidate.sh").exists() + assert (packet_set_dir / "drag-stop/ingest-ocu-candidate.sh").exists() + assert not (packet_set_dir / "keyboard-input-stop/ingest-ocu-candidate.sh").exists() + assert not (packet_set_dir / "cancel/ingest-ocu-candidate.sh").exists() + simple_preflight = json.loads((packet_set_dir / "simple-action-stop/preflight.json").read_text()) + assert "--output-dir" in simple_preflight["commands"]["ocuCandidate"] + assert str(fixture_root / "ocu-candidates") in simple_preflight["commands"]["ocuCandidate"] + drag_preflight = json.loads((packet_set_dir / "drag-stop/preflight.json").read_text()) + assert "--output-dir" in drag_preflight["commands"]["ocuCandidate"] + assert str(fixture_root / "ocu-candidates") in drag_preflight["commands"]["ocuCandidate"] + set_manifest = json.loads((packet_set_dir / "capture-packets.json").read_text()) + assert set_manifest["stage"] == "capture-packet-set" + assert set_manifest["includeTranscript"] is True + assert set_manifest["requiresMcpTranscriptInput"] is True + assert set_manifest["capturePackets"]["simple-action-stop"]["path"] + assert set_manifest["capturePackets"]["simple-action-stop"]["includeTranscript"] is True + assert set_manifest["captureContractPaths"]["simple-action-stop"].endswith( + "simple-action-stop/capture-contract.json" + ) + assert set_manifest["captureContracts"]["simple-action-stop"]["scenario"] == ( + "simple-action-stop" + ) + assert set_manifest["postCaptureWorkflow"]["simple-action-stop"] == ( + set_manifest["captureContracts"]["simple-action-stop"]["postCaptureWorkflow"] + ) + assert set_manifest["postCaptureWorkflow"]["keyboard-input-stop"] == ( + set_manifest["captureContracts"]["keyboard-input-stop"]["postCaptureWorkflow"] + ) + assert any( + step["id"] == "ingest-ocu-candidate" + for step in set_manifest["postCaptureWorkflow"]["simple-action-stop"] + ) + assert not any( + step["id"] == "ingest-ocu-candidate" + for step in set_manifest["postCaptureWorkflow"]["keyboard-input-stop"] + ) + assert set_manifest["captureContracts"]["simple-action-stop"][ + "expectedActionEvents" + ] == ["mouse.click"] + assert ( + set_manifest["captureContracts"]["simple-action-stop"]["expectedEndReason"] + == "recording_controls_stopped" + ) + assert ( + set_manifest["captureContracts"]["simple-action-stop"][ + "requiresMcpTranscriptInput" + ] + is True + ) + assert set_manifest["captureContracts"]["drag-stop"]["expectedActionEvents"] == [ + "mouse.drag" + ] + assert ( + set_manifest["capturePackets"]["simple-action-stop"]["requiresMcpTranscriptInput"] + is True + ) + assert set_manifest["scripts"]["verifyAll"].endswith("verify-all.sh") + assert set_manifest["scripts"]["verifyWorkflow"].endswith("verify-workflow.sh") + assert set_manifest["scripts"]["inspectAll"].endswith("inspect-all.sh") + assert set_manifest["scripts"]["importAll"].endswith("import-all.sh") + assert set_manifest["scripts"]["checkAll"].endswith("check-all.sh") + assert set_manifest["scripts"]["ingestOcuCandidates"].endswith( + "ingest-ocu-candidates.sh" + ) + assert set_manifest["scripts"]["strictExpectedFailureAudit"].endswith( + "strict-expected-failure-audit.sh" + ) + assert capture_packet_set_json["capturePacketSet"][ + "verifyWorkflowShell" + ].endswith("verify-workflow.sh") + assert capture_packet_set_json["capturePacketSet"][ + "ingestOcuCandidatesShell" + ].endswith("ingest-ocu-candidates.sh") + assert capture_packet_set_json["capturePacketSet"][ + "strictExpectedFailureAuditShell" + ].endswith("strict-expected-failure-audit.sh") + set_readme = (packet_set_dir / "README.md").read_text() + assert "MCP transcript input required: `yes`" in set_readme + assert "`capture-packets.json` includes `captureContracts`" in set_readme + assert "`postCaptureWorkflow`" in set_readme + assert "./verify-workflow.sh" in set_readme + assert "`verify-workflow.sh` checks" in set_readme + assert ( + "Replace `inputs/mcp-transcript.json` with same-session transcript evidence." + in set_readme + ) + assert "./ingest-ocu-candidates.sh" in set_readme + assert "./strict-expected-failure-audit.sh" in set_readme + assert "skips scenarios without a generated per-scenario OCU candidate wrapper" in ( + set_readme + ) + verify_all_script = (packet_set_dir / "verify-all.sh").read_text() + assert "simple-action-stop" in verify_all_script + assert "verify-inputs.sh" in verify_all_script + set_workflow_verify = run([str(packet_set_dir / "verify-workflow.sh")], repo) + assert set_workflow_verify.returncode == 0, set_workflow_verify.stderr + set_workflow_verify_json = json.loads(set_workflow_verify.stdout) + assert set_workflow_verify_json["ok"] is True + assert set_workflow_verify_json["checkedPacketSetWorkflow"] is True + mutated_manifest = json.loads((packet_set_dir / "capture-packets.json").read_text()) + mutated_manifest["scripts"]["verifyAll"] = "drifted.sh" + (packet_set_dir / "capture-packets.json").write_text( + json.dumps(mutated_manifest, indent=2, sort_keys=True) + "\n" + ) + set_workflow_drift = run([str(packet_set_dir / "verify-workflow.sh")], repo) + assert set_workflow_drift.returncode == 2 + assert "Packet set scripts manifest drift for verifyAll" in ( + set_workflow_drift.stderr + ) + (packet_set_dir / "capture-packets.json").write_text( + json.dumps(set_manifest, indent=2, sort_keys=True) + "\n" + ) + inspect_all_script = (packet_set_dir / "inspect-all.sh").read_text() + assert "simple-action-stop" in inspect_all_script + assert "inspect-only.sh" in inspect_all_script + assert "check_replaced_input" in inspect_all_script + import_all_script = (packet_set_dir / "import-all.sh").read_text() + assert "check_replaced_input" in import_all_script + check_all_script = (packet_set_dir / "check-all.sh").read_text() + assert "check_replaced_input" not in check_all_script + ingest_ocus_script = (packet_set_dir / "ingest-ocu-candidates.sh").read_text() + assert "ingest-ocu-candidate.sh" in ingest_ocus_script + assert "skipping; no generated ingest-ocu-candidate.sh" in ingest_ocus_script + strict_expected_failure_all = ( + packet_set_dir / "strict-expected-failure-audit.sh" + ).read_text() + assert "--allow-strict-official-golden-missing" in strict_expected_failure_all + placeholder_set_verify = run([str(packet_set_dir / "verify-all.sh")], repo) + assert placeholder_set_verify.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_set_verify.stderr + ) + placeholder_set_inspect = run([str(packet_set_dir / "inspect-all.sh")], repo) + assert placeholder_set_inspect.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_set_inspect.stderr + ) + placeholder_set_import = run([str(packet_set_dir / "import-all.sh")], repo) + assert placeholder_set_import.returncode == 2 + assert "Packet input still contains placeholder: event_stream_stop response" in ( + placeholder_set_import.stderr + ) + assert "==> simple-action-stop" not in placeholder_set_import.stdout + (packet_set_dir / "simple-action-stop/inputs/event_stream_stop-response.json").write_text( + json.dumps( + { + "metadataPath": "/tmp/official-session/metadata.json", + "eventsPath": "/tmp/official-session/events.jsonl", + } + ) + + "\n" + ) + (packet_set_dir / "simple-action-stop/inputs/mcp-transcript.json").write_text( + json.dumps({"stopResponseShape": {"result": {"content": []}}}) + "\n" + ) + partial_placeholder_verify = run([str(packet_set_dir / "verify-all.sh")], repo) + assert partial_placeholder_verify.returncode == 2 + assert "keyboard-input-stop/inputs/event_stream_stop-response.json" in ( + partial_placeholder_verify.stderr + ) + partial_placeholder_import = run([str(packet_set_dir / "import-all.sh")], repo) + assert partial_placeholder_import.returncode == 2 + assert "keyboard-input-stop/inputs/event_stream_stop-response.json" in ( + partial_placeholder_import.stderr + ) + assert "==> simple-action-stop" not in partial_placeholder_import.stdout + + make_packet_dir = tmp_path / "make-capture-packet" + make_packet = run( + [ + "make", + "record-and-replay-official-golden-capture-packet", + f"RNR_PACKET_DIR={make_packet_dir}", + "RNR_SCENARIO=drag-stop", + f"RNR_FIXTURE_ROOT={fixture_root}", + f"RNR_OFFICIAL_PLUGIN_ROOT={tmp_path / 'missing-plugin-for-make'}", + "RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1", + ], + repo, + ) + assert make_packet.returncode == 0, make_packet.stderr + make_packet_json = json.loads(make_packet.stdout) + assert make_packet_json["scenario"] == "drag-stop" + assert make_packet_json["officialPlugin"]["allowedMissing"] is True + assert (make_packet_dir / "README.md").exists() + assert (make_packet_dir / "scenario-recipe.json").exists() + assert (make_packet_dir / "capture-contract.json").exists() + assert (make_packet_dir / "inputs/event_stream_stop-response.json").exists() + assert (make_packet_dir / "verify-inputs.sh").exists() + assert (make_packet_dir / "verify-workflow.sh").exists() + assert (make_packet_dir / "inspect-only.sh").exists() + assert (make_packet_dir / "ingest-ocu-candidate.sh").exists() + assert json.loads((make_packet_dir / "scenario-recipe.json").read_text()) == scenario_recipe( + "drag-stop" + ) + + make_packet_set_dir = tmp_path / "make-capture-packet-set" + make_packet_set = run( + [ + "make", + "record-and-replay-official-golden-capture-packet-set", + f"RNR_PACKET_DIR={make_packet_set_dir}", + f"RNR_FIXTURE_ROOT={fixture_root}", + f"RNR_OFFICIAL_PLUGIN_ROOT={tmp_path / 'missing-plugin-for-make'}", + "RNR_ALLOW_MISSING_OFFICIAL_PLUGIN=1", + ], + repo, + ) + assert make_packet_set.returncode == 0, make_packet_set.stderr + make_packet_set_json = json.loads(make_packet_set.stdout) + assert make_packet_set_json["stage"] == "capture-packet-set" + assert make_packet_set_json["scenarios"] == capture_packet_set_json["scenarios"] + assert (make_packet_set_dir / "capture-packets.json").exists() + assert (make_packet_set_dir / "verify-all.sh").exists() + assert (make_packet_set_dir / "verify-workflow.sh").exists() + assert (make_packet_set_dir / "inspect-all.sh").exists() + assert (make_packet_set_dir / "strict-expected-failure-audit.sh").exists() + assert (make_packet_set_dir / "drag-stop/ingest-ocu-candidate.sh").exists() + assert not (make_packet_set_dir / "keyboard-input-stop/ingest-ocu-candidate.sh").exists() + + no_transcript_packet_set_dir = tmp_path / "capture-packet-set-no-transcript" + no_transcript_packet_set = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--capture-packet-dir", + str(no_transcript_packet_set_dir), + "--capture-packet-recommended-scenarios", + "--no-include-transcript", + ], + repo, + ) + assert no_transcript_packet_set.returncode == 0, no_transcript_packet_set.stderr + no_transcript_json = json.loads(no_transcript_packet_set.stdout) + assert no_transcript_json["stage"] == "capture-packet-set" + assert no_transcript_json["capturePacketSet"]["includeTranscript"] is False + assert no_transcript_json["capturePacketSet"]["requiresMcpTranscriptInput"] is False + assert ( + no_transcript_json["capturePacketSet"]["ingestOcuCandidatesShell"].endswith( + "ingest-ocu-candidates.sh" + ) + ) + assert no_transcript_json["capturePacketSet"][ + "strictExpectedFailureAuditShell" + ].endswith("strict-expected-failure-audit.sh") + no_transcript_manifest = json.loads( + (no_transcript_packet_set_dir / "capture-packets.json").read_text() + ) + assert no_transcript_manifest["includeTranscript"] is False + assert no_transcript_manifest["requiresMcpTranscriptInput"] is False + assert ( + no_transcript_manifest["captureContracts"]["simple-action-stop"][ + "requiresMcpTranscriptInput" + ] + is False + ) + assert ( + no_transcript_manifest["captureContracts"]["drag-stop"]["expectedActionEvents"] + == ["mouse.drag"] + ) + assert no_transcript_manifest["scripts"]["ingestOcuCandidates"].endswith( + "ingest-ocu-candidates.sh" + ) + assert no_transcript_manifest["scripts"]["verifyWorkflow"].endswith( + "verify-workflow.sh" + ) + assert no_transcript_manifest["scripts"]["strictExpectedFailureAudit"].endswith( + "strict-expected-failure-audit.sh" + ) + assert (no_transcript_packet_set_dir / "ingest-ocu-candidates.sh").exists() + assert (no_transcript_packet_set_dir / "verify-workflow.sh").exists() + assert ( + no_transcript_packet_set_dir / "strict-expected-failure-audit.sh" + ).exists() + no_transcript_set_workflow_verify = run( + [str(no_transcript_packet_set_dir / "verify-workflow.sh")], + repo, + ) + assert no_transcript_set_workflow_verify.returncode == 0, ( + no_transcript_set_workflow_verify.stderr + ) + for scenario in no_transcript_json["scenarios"]: + scenario_dir = no_transcript_packet_set_dir / scenario + assert (scenario_dir / "inputs/event_stream_stop-response.json").exists() + assert not (scenario_dir / "inputs/mcp-transcript.json").exists() + assert (scenario_dir / "capture-contract.json").exists() + assert "--mcp-transcript" not in (scenario_dir / "inspect-only.sh").read_text() + assert "--mcp-transcript" not in (scenario_dir / "import-fixture.sh").read_text() + assert ( + no_transcript_manifest["capturePackets"][scenario]["includeTranscript"] is False + ) + assert ( + no_transcript_manifest["capturePackets"][scenario][ + "requiresMcpTranscriptInput" + ] + is False + ) + assert no_transcript_manifest["capturePackets"][scenario]["mcpTranscriptInputPath"] is None + no_transcript_readme = (scenario_dir / "README.md").read_text() + assert "MCP transcript input required: `no`" in no_transcript_readme + assert "does not create `inputs/mcp-transcript.json`" in no_transcript_readme + no_transcript_inspect_all = (no_transcript_packet_set_dir / "inspect-all.sh").read_text() + no_transcript_import_all = (no_transcript_packet_set_dir / "import-all.sh").read_text() + assert "mcp-transcript.json" not in no_transcript_inspect_all + assert "mcp-transcript.json" not in no_transcript_import_all + no_transcript_set_readme = (no_transcript_packet_set_dir / "README.md").read_text() + assert "MCP transcript input required: `no`" in no_transcript_set_readme + assert "does not create or require `inputs/mcp-transcript.json`" in no_transcript_set_readme + assert "./ingest-ocu-candidates.sh" in no_transcript_set_readme + for scenario in no_transcript_json["scenarios"]: + ( + no_transcript_packet_set_dir + / scenario + / "inputs/event_stream_stop-response.json" + ).write_text( + json.dumps( + { + "metadataPath": "/tmp/official-session/metadata.json", + "eventsPath": "/tmp/official-session/events.jsonl", + } + ) + + "\n" + ) + no_transcript_verify_all = run( + [str(no_transcript_packet_set_dir / "verify-all.sh")], + repo, + ) + assert no_transcript_verify_all.returncode == 0, no_transcript_verify_all.stderr + + require_ready = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(plugin_root), + "--require-ready", + ], + repo, + ) + assert require_ready.returncode == 1 + require_ready_json = json.loads(require_ready.stderr) + assert require_ready_json["requireReady"] is True + assert require_ready_json["scenarioStatus"]["ready"] is False + + missing_plugin = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(tmp_path / "missing-plugin"), + ], + repo, + ) + assert missing_plugin.returncode == 1 + missing_plugin_json = json.loads(missing_plugin.stderr) + assert missing_plugin_json["officialPlugin"]["available"] is False + + allowed_missing_plugin = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(fixture_root), + "--official-plugin-root", + str(tmp_path / "missing-plugin"), + "--allow-missing-official-plugin", + "--scenario", + "keyboard-input-stop", + "--no-include-transcript", + ], + repo, + ) + assert allowed_missing_plugin.returncode == 0, allowed_missing_plugin.stderr + allowed_json = json.loads(allowed_missing_plugin.stdout) + assert allowed_json["officialPlugin"]["allowedMissing"] is True + assert allowed_json["scenarioRecipe"]["scenario"] == "keyboard-input-stop" + assert allowed_json["scenarioRecipe"]["expectedActionEvents"] == ["keyboard.text_input"] + assert allowed_json["scenarioRecipe"]["ocuCandidateSourceKind"] == "recording-required" + assert allowed_json["commands"]["ocuCandidate"] is None + assert "synthetic --run-action-smoke" in allowed_json["commands"]["ocuCandidateNote"] + assert "--mcp-transcript" not in allowed_json["commands"]["inspectOnly"] + + recipe_drift_root = tmp_path / "recipe-drift-fixtures" + recipe_drift_fixture = recipe_drift_root / "official-action" + recipe_drift_fixture.mkdir(parents=True) + recipe = scenario_recipe("simple-action-stop") + recipe["expectedActionEvents"] = ["keyboard.text_input"] + (recipe_drift_fixture / "fixture-manifest.json").write_text( + json.dumps( + { + "fixtureFormatVersion": 1, + "name": "official-action", + "source": "official", + "scenario": "simple-action-stop", + "scenarioRecipe": recipe, + "officialPluginVersion": "record-and-replay 1.0.857", + "eventCount": 4, + }, + indent=2, + sort_keys=True, + ) + + "\n" + ) + coverage_error = run( + [ + sys.executable, + str(script), + "--fixture-root", + str(recipe_drift_root), + "--official-plugin-root", + str(plugin_root), + ], + repo, + ) + assert coverage_error.returncode == 1 + coverage_error_json = json.loads(coverage_error.stderr) + assert coverage_error_json["coverageErrors"] + assert any( + "scenarioRecipe does not match scenario" in error + for error in coverage_error_json["coverageErrors"] + ) + assert any("Fix official fixture coverage errors" in action for action in coverage_error_json["nextActions"]) + + print( + json.dumps( + { + "ok": True, + "checkedMissingScenarioPreflight": True, + "checkedCapturePacket": True, + "checkedCapturePacketPostCaptureWorkflow": True, + "checkedCapturePacketWorkflowVerifier": True, + "checkedCapturePacketHandoffScripts": True, + "checkedCapturePacketStrictAuditHandoff": True, + "checkedCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedCapturePacketOcuCandidateOutputDir": True, + "checkedCapturePacketNoTranscript": True, + "checkedCapturePacketSet": True, + "checkedCapturePacketSetPostCaptureWorkflow": True, + "checkedCapturePacketSetWorkflowVerifier": True, + "checkedCapturePacketSetNoTranscript": True, + "checkedCapturePacketSetOcuCandidateHandoff": True, + "checkedCapturePacketSetContractManifest": True, + "checkedCapturePacketTranscriptManifest": True, + "checkedCapturePacketInputSemanticGuard": True, + "checkedCapturePacketVerifyInputs": True, + "checkedCapturePacketSetVerifyAll": True, + "checkedCapturePacketPlaceholderGuard": True, + "checkedCapturePacketImportPlaceholderGuard": True, + "checkedCapturePacketTranscriptPlaceholderGuard": True, + "checkedCapturePacketSetRootPlaceholderGuard": True, + "checkedCapturePacketSetRootPreflightPlaceholderGuard": True, + "checkedMakeCapturePacketTargets": True, + "checkedRequireReadyFailure": True, + "checkedMissingPluginFailure": True, + "checkedAllowedMissingPluginKeyboardScenario": True, + "checkedCoverageErrorFailure": True, + }, + sort_keys=True, + ) + ) + + +if __name__ == "__main__": + main() diff --git a/scripts/test-record-and-replay-skill-repo-scaffold.py b/scripts/test-record-and-replay-skill-repo-scaffold.py new file mode 100755 index 00000000..f7187cda --- /dev/null +++ b/scripts/test-record-and-replay-skill-repo-scaffold.py @@ -0,0 +1,1525 @@ +#!/usr/bin/env python3 + +import json +import os +import pathlib +import re +import subprocess +import sys +import tempfile +import atexit + +from record_and_replay_scenarios import ( + DEFAULT_RECOMMENDED_SCENARIOS, + DEFAULT_REQUIRED_SCENARIOS, + scenario_recipe, +) + + +def main() -> int: + repo = pathlib.Path(__file__).resolve().parents[1] + baseline_summary_path = repo / "dist/record-and-replay-baseline-summary.json" + baseline_summary_original = baseline_summary_path.read_text() + atexit.register(lambda: baseline_summary_path.write_text(baseline_summary_original)) + baseline_summary = json.loads(baseline_summary_original) + baseline_summary.setdefault("evidence", {}).setdefault( + "fixtureIngestPipelines", + {}, + )["checkedOfficialSessionDirectoryPathHandoff"] = True + baseline_summary_path.write_text(json.dumps(baseline_summary, indent=2, sort_keys=True) + "\n") + with tempfile.TemporaryDirectory() as tmp: + output_dir = pathlib.Path(tmp) / "standalone-rnr-skill" + result = subprocess.run( + [ + sys.executable, + str(repo / "scripts/scaffold-record-and-replay-skill-repo.py"), + "--output-dir", + str(output_dir), + ], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + payload = json.loads(result.stdout) + assert payload["ok"] is True + + skill_path = output_dir / "skills/open-computer-use-record-and-replay/SKILL.md" + readme_path = output_dir / "README.md" + package_script = output_dir / "scripts/package-skill.sh" + check_script = output_dir / "scripts/check.sh" + verify_package_script = output_dir / "scripts/verify-package-artifact.py" + verify_manifest_script = output_dir / "scripts/verify-manifest.py" + verify_source_summary_script = output_dir / "scripts/verify-source-baseline-summary.py" + verify_readme_script = output_dir / "scripts/verify-readme-handoff.py" + verify_runtime_script = output_dir / "scripts/verify-runtime.py" + verify_skill_workflow_script = output_dir / "scripts/verify-skill-workflow.py" + wait_notify_smoke_script = output_dir / "scripts/wait-notify-contract-smoke.py" + recording_to_skill_smoke_script = output_dir / "scripts/recording-to-skill-smoke.py" + lifecycle_smoke_script = output_dir / "scripts/recording-lifecycle-smoke.py" + workflow_path = output_dir / ".github/workflows/ci.yml" + manifest_path = output_dir / "record-and-replay-skill-repo.json" + source_summary_path = output_dir / "evidence/source-baseline-summary.json" + assert skill_path.exists() + assert readme_path.exists() + assert package_script.exists() + assert check_script.exists() + assert verify_package_script.exists() + assert verify_manifest_script.exists() + assert verify_source_summary_script.exists() + assert verify_readme_script.exists() + assert verify_runtime_script.exists() + assert verify_skill_workflow_script.exists() + assert wait_notify_smoke_script.exists() + assert recording_to_skill_smoke_script.exists() + assert lifecycle_smoke_script.exists() + assert workflow_path.exists() + assert manifest_path.exists() + assert source_summary_path.exists() + + skill_text = skill_path.read_text() + assert "name: open-computer-use-record-and-replay" in skill_text + assert "event_stream_start" in skill_text + assert "open-computer-use event-stream scaffold-skill" in skill_text + assert "ask whether the user wants to use that active recording or wait" in skill_text + assert "safetySignals" in skill_text + assert "skillEvidence.hasSafetySignals=true" in skill_text + assert "explicit user confirmation before replay" in skill_text + + combined = "\n".join(path.read_text() for path in output_dir.rglob("*") if path.is_file()) + assert str(repo) not in combined + assert "open-computer-use install-codex-record-and-replay-mcp" in combined + assert "./scripts/check.sh" in combined + assert "./scripts/verify-runtime.py" in combined + assert "./scripts/verify-skill-workflow.py" in combined + assert "scripts/verify-readme-handoff.py" in combined + assert "scripts/verify-source-baseline-summary.py" in combined + assert "evidence/source-baseline-summary.json" in combined + assert "scripts/recording-to-skill-smoke.py" in combined + assert "./scripts/recording-lifecycle-smoke.py" in combined + assert "npm i -g open-computer-use" in combined + assert "Python 3 must be available as `python3`" in combined + assert "PYTHON=/path/to/python3" in combined + assert "frontmatter description is required" in combined + assert "frontmatter name must be" in combined + assert "Record & Replay tool metadata drifted" in combined + assert "checkedSkillWorkflow" in combined + assert "checkedSourceBaselineSummaryEvidence" in combined + assert "checkedStatusNotUsedAsWaitLoopGuard" in combined + assert "checkedMcpNoDirectEventContentsGuard" in combined + assert "Do not poll while the user is recording." in combined + assert "Use `event_stream_status` only when the user asks for status" in combined + assert "The MCP server does not expose event-stream contents directly." in combined + assert "forbiddenSnippets" in combined + assert "EXPECTED_INITIALIZE_CAPABILITIES" in combined + assert "notifications/initialized" in combined + assert "listChanged" in combined + assert "destructiveHint" in combined + assert "readOnlyHint" in combined + assert "--strict-ocu --require-skill-draft " in combined + assert "--require-skill-draft " in combined + assert "session.ended.endReason=recording_controls_cancelled" in combined + assert "single `session.started` opening event" in combined + assert "single `session.ended`" in combined + assert "final-event closure" in combined + assert "metadata/session alias consistency" in combined + assert "declared `metadataPath`" in combined + assert "checkedDeclaredHandoffPaths" in combined + assert "checkedScreenshotPathContainment" in combined + assert "checks the declared handoff path evidence" in combined + assert "Cancelled, incomplete, or" in combined + assert "must not be used to" in combined + assert "create or update a skill" in combined + assert "event-stream scaffold-skill --json " in combined + assert "The scaffold command also runs the skill-draft validation gate" in combined + assert "checkedEventsOnlyValidation" in combined + assert "checkedScaffoldSkill" in combined + assert "checkedCancelledRecordingRejected" in combined + assert "checkedSkillCreatorHandoff" in combined + assert "checkedWaitNotifyContract" in combined + assert "checkedNoActiveStatusStop" in combined + assert "EXPECTED_NO_ACTIVE_RESPONSE" in combined + assert "synthetic-recording-workflow" in combined + assert "Complete the `skill-creator` workflow, including validation" in combined + assert "not a standalone runbook or replay plan" in combined + assert "does not start a recording" in combined + assert "Independent Wait / Notify Integration" in combined + assert "event-stream wait --json --session-id --notify-command" in combined + assert "`wait --json` adds `waitTimedOut` and `waitSessionMatched`." in combined + assert "`--notify-command` receives the final status JSON on stdin" in combined + assert "OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON" in combined + assert "Keep this listener path outside the official-compatible MCP tool surface." in combined + assert "## Official Evidence" in combined + assert "record-and-replay-event-stream-surface-1.0.857.json" in combined + assert "record-and-replay-official-no-active-status-stop-1.0.857.json" in combined + assert "record-and-replay-official-raw-start-timeout-1.0.857.json" in combined + assert "baseline contract smoke" in combined + assert "official-golden-capture-preflight-smoke" in combined + assert "ocu-candidate-pairing-preflight-smoke" in combined + assert "source checks and preflight" in combined + assert "scripts are not copied into this" in combined + assert "default standalone" in combined + assert "self-check still does not" in combined + assert "make record-and-replay-baseline-audit" in combined + assert "make record-and-replay-official-golden-gate-audit" in combined + assert "dist/record-and-replay-baseline-summary.json" in combined + assert "dist/record-and-replay-official-golden-gate-summary.json" in combined + assert "Successful recording fixtures are still required" in combined + assert "The minimum required official successful recording scenario is" in combined + assert "officialEvidence.scenarioRecipes" in combined + assert "OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS" in combined + assert "runtimeVersion=" in combined + assert ( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime" + in combined + ) + + verify_runtime_text = verify_runtime_script.read_text() + assert "OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS" in verify_runtime_text + assert "runtimeVersion=" in verify_runtime_text + assert ( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime" + in verify_runtime_text + ) + + manifest = json.loads(manifest_path.read_text()) + required_scenarios = list(DEFAULT_REQUIRED_SCENARIOS) + recommended_scenarios = list(DEFAULT_RECOMMENDED_SCENARIOS) + required_readme_list = ", ".join(f"`{scenario}`" for scenario in required_scenarios) + recommended_readme_list = ( + ", ".join(f"`{scenario}`" for scenario in recommended_scenarios[:-1]) + + f", and `{recommended_scenarios[-1]}`" + ) + readme_text = readme_path.read_text() + assert ( + "The minimum required official successful recording scenario is\n" + f"{required_readme_list}. The recommended calibration set is\n" + f"{recommended_readme_list}." + ) in readme_text + for scenario in recommended_scenarios: + assert f"`{scenario}`" in combined + assert manifest["runtimeDependency"] == "open-computer-use" + source_baseline_summary = json.loads( + (repo / "dist/record-and-replay-baseline-summary.json").read_text() + ) + source_has_successful_recording_golden = ( + source_baseline_summary["status"]["officialSuccessfulRecordingGoldenComplete"] + is True + ) + assert manifest["officialEvidence"] == { + "baselineVersion": "record-and-replay/1.0.857", + "nonRecordingSurfaceFixture": "record-and-replay-event-stream-surface-1.0.857.json", + "noActiveStatusStopFixture": ( + "record-and-replay-official-no-active-status-stop-1.0.857.json" + ), + "hostlessRawStartTimeoutFixture": ( + "record-and-replay-official-raw-start-timeout-1.0.857.json" + ), + "sourceRepoBaselineChecks": { + "baselineContract": "baseline-contract-smoke", + "officialRawStartTimeout": "official-raw-start-timeout-fixture-smoke", + "officialFixtureSetGate": { + "check": "official-fixture-set-smoke", + "sameScenarioComparePolicy": { + "requiresAxDiffEvidence": True, + "requiresSameAxDiffMarkers": True, + "requiresSameSuppressedEventSequence": True, + "requiresSameSuppressedSchema": True, + }, + }, + "officialFixtureIngest": { + "check": "official-fixture-ingest-smoke", + "requiredEvidence": [ + "checkedOfficialSessionDirectoryPathHandoff", + ], + }, + "officialGoldenCapturePreflight": { + "check": "official-golden-capture-preflight-smoke", + "requiredEvidence": [ + "checkedOfficialCapturePacketInputSemanticGuard", + "checkedOfficialCapturePacketSetContractManifest", + "checkedOfficialCapturePacketPostCaptureWorkflow", + "checkedOfficialCapturePacketWorkflowVerifier", + "checkedOfficialCapturePacketSetPostCaptureWorkflow", + "checkedOfficialCapturePacketSetWorkflowVerifier", + "checkedOfficialCapturePacketStrictAuditHandoff", + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff", + ], + }, + "ocuCandidatePairingPreflight": ( + "ocu-candidate-pairing-preflight-smoke" + ), + }, + "sourceRepoBaselineAudit": { + "usableBaseline": "make record-and-replay-baseline-audit", + "strictOfficialGoldenGate": ( + "make record-and-replay-official-golden-gate-audit" + ), + "auditTargetDryRunSmoke": ( + "make record-and-replay-baseline-audit-targets-smoke" + ), + "baselineSummaryArtifact": ( + "dist/record-and-replay-baseline-summary.json" + ), + "copiedBaselineSummaryEvidence": ( + "evidence/source-baseline-summary.json" + ), + "strictOfficialGoldenSummaryArtifact": ( + "dist/record-and-replay-official-golden-gate-summary.json" + ), + "baselineSummaryEnvVar": "RNR_BASELINE_SUMMARY_JSON", + "strictOfficialGoldenSummaryEnvVar": ( + "RNR_OFFICIAL_GOLDEN_SUMMARY_JSON" + ), + "strictOfficialGoldenExpectedFailureAudit": ( + "scripts/check-record-and-replay-baseline-summary.py " + "dist/record-and-replay-official-golden-gate-summary.json " + "--allow-strict-official-golden-missing" + ), + "verifiesSummaryArtifactSeparation": True, + "verifiesSummaryEnvVarIsolation": True, + }, + "standaloneRepoBoundary": { + "defaultChecksDoNotStartOfficialRecording": True, + "preflightScriptsRemainInOpenComputerUseRepo": True, + "doesNotCopyOpenComputerUseRuntimeSource": True, + }, + "hasSuccessfulRecordingGolden": source_has_successful_recording_golden, + "requiredSuccessfulRecordingScenarios": required_scenarios, + "recommendedSuccessfulRecordingScenarios": recommended_scenarios, + "scenarioRecipes": { + scenario: scenario_recipe(scenario) for scenario in recommended_scenarios + }, + "successfulRecordingGoldenRequiredFor": [ + "session file schema equivalence", + "event field schema equivalence", + "AX compact diff algorithm equivalence", + "screenshot trigger equivalence", + "timeout endReason equivalence", + ], + } + assert manifest["mcpServer"]["args"] == ["event-stream", "mcp"] + assert manifest["mcpServer"]["capabilities"] == {"tools": {"listChanged": False}} + assert manifest["checks"]["packageSkill"] == "scripts/package-skill.sh" + assert manifest["checks"]["packageArtifact"] == "scripts/verify-package-artifact.py" + assert manifest["checks"]["manifestContract"] == "scripts/verify-manifest.py" + assert manifest["checks"]["sourceBaselineSummaryEvidence"] == ( + "scripts/verify-source-baseline-summary.py" + ) + assert manifest["checks"]["readmeHandoffContract"] == "scripts/verify-readme-handoff.py" + assert manifest["checks"]["runtimeContract"] == "scripts/verify-runtime.py" + assert manifest["checks"]["skillWorkflow"] == "scripts/verify-skill-workflow.py" + assert manifest["checks"]["waitNotifyContractSmoke"] == ( + "scripts/wait-notify-contract-smoke.py" + ) + assert manifest["checks"]["recordingToSkillSmoke"] == "scripts/recording-to-skill-smoke.py" + assert manifest["checks"]["selfCheck"] == "scripts/check.sh" + assert manifest["optionalChecks"]["recordingLifecycleSmoke"] == ( + "scripts/recording-lifecycle-smoke.py" + ) + assert manifest["mcpServer"]["tools"] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ] + assert manifest["mcpServer"]["requiresObjectParams"] is True + assert manifest["mcpServer"]["requiresStringToolName"] is True + assert manifest["mcpServer"]["requiresObjectArguments"] is True + assert manifest["mcpServer"]["rejectsUnexpectedArguments"] is True + assert manifest["mcpServer"]["rejectedRequestsDoNotCreateSessionFiles"] is True + assert manifest["mcpServer"]["noActiveResponse"] == { + "event_stream_status": { + "isRecording": False, + "maxDurationSeconds": 1800, + }, + "event_stream_stop": { + "isRecording": False, + "maxDurationSeconds": 1800, + }, + } + assert [tool["name"] for tool in manifest["mcpServer"]["toolMetadata"]] == manifest["mcpServer"]["tools"] + assert manifest["mcpServer"]["toolMetadata"][0]["description"].startswith( + "Start recording the user's actions" + ) + assert manifest["mcpServer"]["toolMetadata"][1]["annotations"]["readOnlyHint"] is True + assert manifest["mcpServer"]["toolMetadata"][2]["inputSchema"] == { + "additionalProperties": False, + "properties": {}, + "type": "object", + } + assert manifest["extensionLayer"]["officialCompatibleMcpSurface"] is False + assert "event-stream wait --json --session-id --notify-command " in ( + manifest["extensionLayer"]["commands"] + ) + assert manifest["extensionLayer"]["waitNotify"]["resultFields"] == [ + "waitTimedOut", + "waitSessionMatched", + ] + assert manifest["extensionLayer"]["waitNotify"]["stdin"] == "final status JSON" + assert ( + "OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH" + in manifest["extensionLayer"]["waitNotify"]["environmentVariables"] + ) + assert ( + "OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH" + in manifest["extensionLayer"]["waitNotify"]["environmentVariables"] + ) + assert ( + manifest["extensionLayer"]["waitNotify"][ + "callbackSkippedWhenWaitSessionMatchedFalse" + ] + is True + ) + assert ( + manifest["extensionLayer"]["waitNotify"]["callbackFailureMakesCliFail"] + is True + ) + assert ( + manifest["extensionLayer"]["waitNotify"]["callbackTimeoutMakesCliFail"] + is True + ) + assert manifest["recordingToSkill"]["strictValidation"]["command"] == ( + "event-stream validate --json --strict-ocu --require-skill-draft " + ) + assert manifest["recordingToSkill"]["strictValidation"]["requiresDeclaredHandoffPaths"] == [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", + ] + assert ( + manifest["recordingToSkill"]["strictValidation"]["requiresMetadataSessionAlias"] + is True + ) + assert ( + manifest["recordingToSkill"]["strictValidation"][ + "requiresScreenshotPathsInsideSession" + ] + is True + ) + assert manifest["recordingToSkill"]["strictValidation"]["requiresSkillDraftReady"] is True + assert manifest["recordingToSkill"]["eventsOnlyValidation"] == { + "command": "event-stream validate --json --require-skill-draft ", + "provesMetadataSessionAlias": False, + "provesDeclaredHandoffPaths": False, + } + assert manifest["recordingToSkill"]["scaffoldSkill"] == { + "command": "event-stream scaffold-skill --json ", + "runsSkillDraftValidationGate": True, + } + assert manifest["recordingToSkill"]["rejectsCancelledRecordings"] is True + + workflow_text = workflow_path.read_text() + assert "run: ./scripts/check.sh" in workflow_text + for line in workflow_text.splitlines(): + if "uses:" in line: + assert re.search(r"@[0-9a-f]{40}(\\s|$)", line), line + + subprocess.run( + ["bash", "-n", str(package_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["bash", "-n", str(check_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(verify_manifest_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(verify_source_summary_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(verify_readme_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(verify_runtime_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(lifecycle_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(recording_to_skill_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(verify_skill_workflow_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + subprocess.run( + ["python3", "-m", "py_compile", str(wait_notify_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + original_skill_text = skill_path.read_text() + try: + mutated_skill_lines = [ + "description: " if line.startswith("description: ") else line + for line in original_skill_text.splitlines() + ] + skill_path.write_text("\n".join(mutated_skill_lines) + "\n") + invalid_package = subprocess.run( + [str(package_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_package.returncode != 0 + assert "frontmatter description is required" in invalid_package.stderr + finally: + skill_path.write_text(original_skill_text) + + manifest_check = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + manifest_payload = json.loads(manifest_check.stdout) + assert manifest_payload["ok"] is True + assert manifest_payload["checkedManifestContract"] is True + assert manifest_payload["checkedStrictExpectedFailureAudit"] is True + assert manifest_payload["checkedOfficialFixtureSetComparePolicy"] is True + assert manifest_payload["checkedSourceBaselineSummaryEvidenceCheck"] is True + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineAudit"][ + "strictOfficialGoldenExpectedFailureAudit" + ] = "missing" + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "strict expected-failure audit command drifted" in invalid_manifest.stderr + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + source_summary_check = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + source_summary_payload = json.loads(source_summary_check.stdout) + assert source_summary_payload["ok"] is True + assert source_summary_payload["checkedSourceBaselineSummaryEvidence"] is True + assert source_summary_payload["checkedSourceBaselineSummaryOfficialGoldenState"] is True + assert ( + source_summary_payload["checkedSourceBaselineSummaryOfficialGoldenGap"] + is not source_has_successful_recording_golden + ) + assert ( + source_summary_payload["checkedSourceBaselineSummaryOfficialGoldenComplete"] + is source_has_successful_recording_golden + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketSetContractManifest" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff" + ] + is True + ) + assert ( + source_summary_payload[ + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff" + ] + is True + ) + source_summary = json.loads(source_summary_path.read_text()) + assert source_summary["status"]["usableBaseline"] is True + assert source_summary["status"]["standaloneRepoBaselineReady"] is True + assert ( + source_summary["status"]["officialSuccessfulRecordingGoldenComplete"] + is source_has_successful_recording_golden + ) + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetContractManifest" + ] is True + assert source_summary["evidence"]["fixtureIngestPipelines"][ + "checkedOfficialSessionDirectoryPathHandoff" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketPostCaptureWorkflow" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketWorkflowVerifier" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetPostCaptureWorkflow" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetWorkflowVerifier" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictAuditHandoff" + ] is True + assert source_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + ] is True + assert source_summary["evidence"]["standaloneSkillRepo"][ + "checkedOfficialFixtureSetComparePolicyManifest" + ] is True + assert source_summary["evidence"]["npmStagedSkillRepo"][ + "checkedOfficialFixtureSetComparePolicyManifest" + ] is True + if not source_has_successful_recording_golden: + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["status"]["officialGoldenGatePassed"] = True + mutated_summary["status"]["officialSuccessfulRecordingGoldenComplete"] = True + mutated_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] = True + mutated_summary["status"]["requiresOfficialGoldenCapture"] = False + mutated_summary["status"][ + "missingRequiredOfficialSuccessfulRecordingScenarios" + ] = [] + mutated_summary["status"][ + "notReadyRequiredOfficialSuccessfulRecordingScenarios" + ] = [] + mutated_summary["status"]["officialFixtureCoverageErrors"] = [] + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + completed_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + completed_payload = json.loads(completed_source_summary.stdout) + assert completed_payload["checkedSourceBaselineSummaryOfficialGoldenState"] is True + assert completed_payload["checkedSourceBaselineSummaryOfficialGoldenGap"] is False + assert ( + completed_payload["checkedSourceBaselineSummaryOfficialGoldenComplete"] + is True + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["status"]["officialSuccessfulRecordingEquivalenceReady"] = ( + not source_has_successful_recording_golden + ) + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert ( + "official golden state must be either current required gap or completed equivalence" + in invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["fixtureIngestPipelines"][ + "checkedOfficialSessionDirectoryPathHandoff" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "official sessionDirectoryPath handoff evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetContractManifest" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet set contract manifest evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketPostCaptureWorkflow" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet post-capture workflow evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetPostCaptureWorkflow" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet set post-capture workflow evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketWorkflowVerifier" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet workflow verifier evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketSetWorkflowVerifier" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet set workflow verifier evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictAuditHandoff" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert "capture packet strict audit handoff evidence missing" in ( + invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_summary = json.loads(source_summary_path.read_text()) + mutated_summary["evidence"]["preflightPipelines"][ + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + ] = False + source_summary_path.write_text(json.dumps(mutated_summary, indent=2) + "\n") + invalid_source_summary = subprocess.run( + [str(verify_source_summary_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_source_summary.returncode != 0 + assert ( + "capture packet strict expected-failure audit handoff evidence missing" + in invalid_source_summary.stderr + ) + finally: + source_summary_path.write_text(json.dumps(source_summary, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialFixtureSetGate" + ]["sameScenarioComparePolicy"]["requiresSameSuppressedSchema"] = False + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official fixture set suppressed schema policy missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketStrictExpectedFailureAuditHandoff" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert ( + "official capture packet strict expected-failure audit handoff evidence missing" + in invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketSetContractManifest" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official capture packet set contract manifest evidence missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketPostCaptureWorkflow" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official capture packet post-capture workflow evidence missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketWorkflowVerifier" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official capture packet workflow verifier evidence missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketSetPostCaptureWorkflow" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert ( + invalid_manifest.returncode != 0 + ) + assert ( + "official capture packet set post-capture workflow evidence missing" + in invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketSetWorkflowVerifier" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official capture packet set workflow verifier evidence missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + try: + mutated_manifest = json.loads(manifest_path.read_text()) + mutated_manifest["officialEvidence"]["sourceRepoBaselineChecks"][ + "officialGoldenCapturePreflight" + ]["requiredEvidence"].remove( + "checkedOfficialCapturePacketStrictAuditHandoff" + ) + manifest_path.write_text(json.dumps(mutated_manifest, indent=2) + "\n") + invalid_manifest = subprocess.run( + [str(verify_manifest_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_manifest.returncode != 0 + assert "official capture packet strict audit handoff evidence missing" in ( + invalid_manifest.stderr + ) + finally: + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + + readme_check = subprocess.run( + [str(verify_readme_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + readme_payload = json.loads(readme_check.stdout) + assert readme_payload["ok"] is True + assert readme_payload["checkedReadmeHandoffContract"] is True + assert readme_payload["checkedReadmeOfficialEvidenceHandoff"] is True + assert readme_payload["checkedReadmeOfficialGoldenGap"] is True + assert readme_payload["checkedReadmeWaitNotifyBoundary"] is True + original_readme_text = readme_path.read_text() + try: + readme_path.write_text( + original_readme_text.replace( + "make record-and-replay-official-golden-gate-audit", + "missing-strict-gate", + ) + ) + invalid_readme = subprocess.run( + [str(verify_readme_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_readme.returncode != 0 + assert "missingRequiredSnippets" in invalid_readme.stderr + finally: + readme_path.write_text(original_readme_text) + + workflow_check = subprocess.run( + [str(verify_skill_workflow_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + workflow_payload = json.loads(workflow_check.stdout) + assert workflow_payload["ok"] is True + assert workflow_payload["checkedSkillWorkflow"] is True + assert workflow_payload["checkedStatusNotUsedAsWaitLoopGuard"] is True + assert workflow_payload["checkedMcpNoDirectEventContentsGuard"] is True + try: + skill_path.write_text(original_skill_text.replace("Do not poll while the user is recording.", "")) + invalid_workflow = subprocess.run( + [str(verify_skill_workflow_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_workflow.returncode != 0 + assert "missingRequiredSnippets" in invalid_workflow.stderr + finally: + skill_path.write_text(original_skill_text) + try: + skill_path.write_text( + original_skill_text.replace( + "Use `event_stream_status` only when the user asks for status or returns after recording; do not use it to wait for completion.", + "", + ) + ) + invalid_status_workflow = subprocess.run( + [str(verify_skill_workflow_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_status_workflow.returncode != 0 + assert "missingRequiredSnippets" in invalid_status_workflow.stderr + finally: + skill_path.write_text(original_skill_text) + try: + skill_path.write_text( + original_skill_text.replace( + "The MCP server does not expose event-stream contents directly.", + "", + ) + ) + invalid_handoff_workflow = subprocess.run( + [str(verify_skill_workflow_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_handoff_workflow.returncode != 0 + assert "missingRequiredSnippets" in invalid_handoff_workflow.stderr + finally: + skill_path.write_text(original_skill_text) + try: + skill_path.write_text( + original_skill_text.replace( + "OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON", + "REMOVED_STATUS_JSON_ENV", + ) + ) + invalid_wait_workflow = subprocess.run( + [str(verify_skill_workflow_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_wait_workflow.returncode != 0 + assert "missingRequiredSnippets" in invalid_wait_workflow.stderr + finally: + skill_path.write_text(original_skill_text) + + subprocess.run( + [str(package_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert (output_dir / "dist/skills/open-computer-use-record-and-replay.skill").exists() + package_check = subprocess.run( + [str(verify_package_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + package_payload = json.loads(package_check.stdout) + assert package_payload["ok"] is True + assert package_payload["checkedPackageArtifact"] is True + assert package_payload["checkedPackageSkillArchive"] is True + assert package_payload["checkedPackageSkillFrontmatter"] is True + assert package_payload["checkedPackageSkillHandoff"] is True + packaged_skill = output_dir / "dist/skills/open-computer-use-record-and-replay.skill" + original_packaged_skill = packaged_skill.read_bytes() + try: + packaged_skill.write_bytes(b"not a zip") + invalid_package = subprocess.run( + [str(verify_package_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + assert invalid_package.returncode != 0 + assert "failed to read skill package" in invalid_package.stderr + finally: + packaged_skill.write_bytes(original_packaged_skill) + + subprocess.run( + ["swift", "build", "--product", "OpenComputerUse"], + cwd=repo, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + fake_runtime = pathlib.Path(tmp) / "stale-open-computer-use" + fake_runtime.write_text( + "#!/usr/bin/env bash\n" + "if [[ \"${1:-}\" == \"--version\" ]]; then\n" + " echo '0.1.51'\n" + " exit 0\n" + "fi\n" + "cat >/dev/null\n", + ) + fake_runtime.chmod(0o755) + timed_out_runtime = subprocess.run( + [str(verify_runtime_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(fake_runtime), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS": "0.2", + }, + ) + assert timed_out_runtime.returncode != 0 + assert "runtimeVersion={'ok': True" in timed_out_runtime.stderr + assert "0.1.51" in timed_out_runtime.stderr + assert ( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime" + in timed_out_runtime.stderr + ) + timed_out_lifecycle = subprocess.run( + [str(lifecycle_smoke_script)], + cwd=output_dir, + check=False, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(fake_runtime), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + "OPEN_COMPUTER_USE_RUNTIME_VERIFY_TIMEOUT_SECONDS": "0.2", + }, + ) + assert timed_out_lifecycle.returncode != 0 + assert "runtimeVersion={'ok': True" in timed_out_lifecycle.stderr + assert "0.1.51" in timed_out_lifecycle.stderr + assert ( + "Set OPEN_COMPUTER_USE_CLI to the current open-computer-use runtime" + in timed_out_lifecycle.stderr + ) + runtime_check = subprocess.run( + [str(verify_runtime_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(repo / ".build/debug/OpenComputerUse"), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + runtime_payload = json.loads(runtime_check.stdout) + assert runtime_payload["ok"] is True + assert runtime_payload["tools"] == [ + "event_stream_start", + "event_stream_status", + "event_stream_stop", + ] + assert runtime_payload["checkedInitializeSurfaceContract"] is True + assert runtime_payload["checkedToolMetadataContract"] is True + assert runtime_payload["checkedToolInputSchemaNoArguments"] is True + assert runtime_payload["checkedNoActiveStatusStop"] is True + assert runtime_payload["checkedRejectsUnexpectedArguments"] is True + assert runtime_payload["checkedRejectsNonObjectArguments"] is True + assert runtime_payload["checkedRequiresObjectParams"] is True + assert runtime_payload["checkedRequiresStringToolName"] is True + assert runtime_payload["checkedRequiresObjectArguments"] is True + assert runtime_payload["checkedRejectedRequestsDoNotCreateSessionFiles"] is True + wait_notify_check = subprocess.run( + [str(wait_notify_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(repo / ".build/debug/OpenComputerUse"), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + wait_notify_payload = json.loads(wait_notify_check.stdout) + assert wait_notify_payload["ok"] is True + assert wait_notify_payload["checkedWaitNotifyContract"] is True + assert wait_notify_payload["checkedNotifySuppressedEventsPathEnv"] is True + assert wait_notify_payload["checkedNotifyCallbackFailureExit"] is True + assert wait_notify_payload["checkedNotifyCallbackFailureReason"] is True + assert wait_notify_payload["checkedNotifyCallbackTimeoutFailureExit"] is True + assert wait_notify_payload["checkedNotifyCallbackTimeoutReason"] is True + assert wait_notify_payload["waitTimedOut"] is True + assert wait_notify_payload["waitSessionMatched"] is False + assert wait_notify_payload["callbackSkipped"] is True + assert wait_notify_payload["callbackFailureDetected"] is True + assert wait_notify_payload["callbackFailureReason"] == "nonZeroExit" + assert wait_notify_payload["callbackTimeoutDetected"] is True + assert wait_notify_payload["callbackTimeoutReason"] == "timeout" + recording_to_skill_check = subprocess.run( + [str(recording_to_skill_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(repo / ".build/debug/OpenComputerUse"), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + recording_to_skill_payload = json.loads(recording_to_skill_check.stdout) + assert recording_to_skill_payload["ok"] is True + assert recording_to_skill_payload["checkedStrictValidation"] is True + assert recording_to_skill_payload["checkedScreenshotPathContainment"] is True + assert recording_to_skill_payload["checkedEventsOnlyValidation"] is True + assert recording_to_skill_payload["checkedScaffoldSkill"] is True + assert recording_to_skill_payload["checkedScaffoldSkillFailureExit"] is True + assert recording_to_skill_payload["checkedCancelledRecordingRejected"] is True + assert recording_to_skill_payload["checkedSkillCreatorHandoff"] is True + lifecycle_check = subprocess.run( + [str(lifecycle_smoke_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(repo / ".build/debug/OpenComputerUse"), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + lifecycle_payload = json.loads(lifecycle_check.stdout) + assert lifecycle_payload["ok"] is True + assert lifecycle_payload["checkedOneActive"] is True + assert lifecycle_payload["checkedIdempotentStop"] is True + assert lifecycle_payload["checkedFinalStatus"] is True + assert "session.started" in lifecycle_payload["eventTypes"] + assert "session.ended" in lifecycle_payload["eventTypes"] + check_run = subprocess.run( + [str(check_script)], + cwd=output_dir, + check=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env={ + **os.environ, + "OPEN_COMPUTER_USE_CLI": str(repo / ".build/debug/OpenComputerUse"), + "OPEN_COMPUTER_USE_DISABLE_APP_AGENT_PROXY": "1", + }, + ) + assert "open-computer-use-record-and-replay.skill" in check_run.stdout + check_lines = [line for line in check_run.stdout.splitlines() if line.startswith("{")] + assert check_lines + check_payloads = [json.loads(line) for line in check_lines] + assert any(payload.get("checkedManifestContract") is True for payload in check_payloads) + assert any(payload.get("checkedPackageArtifact") is True for payload in check_payloads) + assert any( + payload.get("checkedSourceBaselineSummaryEvidence") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedSourceBaselineSummaryOfficialGoldenState") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff") + is True + for payload in check_payloads + ) + assert any( + payload.get("checkedReadmeHandoffContract") is True + for payload in check_payloads + ) + assert any(payload.get("checkedSkillWorkflow") is True for payload in check_payloads) + assert any( + payload.get("checkedStatusNotUsedAsWaitLoopGuard") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedMcpNoDirectEventContentsGuard") is True + for payload in check_payloads + ) + assert any(payload.get("checkedWaitNotifyContract") is True for payload in check_payloads) + assert any( + payload.get("checkedNotifyCallbackFailureExit") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedNotifyCallbackFailureReason") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedNotifyCallbackTimeoutFailureExit") is True + for payload in check_payloads + ) + assert any( + payload.get("checkedNotifyCallbackTimeoutReason") is True + for payload in check_payloads + ) + assert any(payload.get("checkedScaffoldSkill") is True for payload in check_payloads) + assert check_payloads[-1]["ok"] is True + + print( + json.dumps( + { + "ok": True, + "checkedStandaloneRepoScaffold": True, + "checkedManifestContract": True, + "checkedReadmeHandoffContract": True, + "checkedReadmeOfficialEvidenceHandoff": True, + "checkedReadmeOfficialGoldenGap": True, + "checkedReadmeWaitNotifyBoundary": True, + "checkedGeneratedReadmePrerequisites": True, + "checkedGeneratedReadmeScenarioList": True, + "checkedOfficialEvidenceManifest": True, + "checkedOfficialEvidenceScenarioManifest": True, + "checkedOfficialEvidencePreflightManifest": True, + "checkedOfficialEvidenceAuditManifest": True, + "checkedOfficialFixtureSetComparePolicyManifest": True, + "checkedSourceBaselineSummaryEvidence": True, + "checkedSourceBaselineSummaryOfficialGoldenState": True, + "checkedSourceBaselineSummaryOfficialSessionDirectoryPathHandoff": True, + "checkedSourceBaselineSummaryCapturePacketSetContractManifest": True, + "checkedSourceBaselineSummaryCapturePacketPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketSetPostCaptureWorkflow": True, + "checkedSourceBaselineSummaryCapturePacketSetWorkflowVerifier": True, + "checkedSourceBaselineSummaryCapturePacketStrictAuditHandoff": True, + "checkedSourceBaselineSummaryCapturePacketStrictExpectedFailureAuditHandoff": True, + "checkedPackageArtifact": True, + "checkedSkillPackaging": True, + "checkedRuntimeTimeoutDiagnostics": True, + "checkedRuntimeContract": True, + "checkedInitializeSurfaceContract": True, + "checkedToolMetadataContract": True, + "checkedToolInputSchemaNoArguments": True, + "checkedNoActiveStatusStop": True, + "checkedRejectsUnexpectedArguments": True, + "checkedRejectsNonObjectArguments": True, + "checkedRequiresObjectParams": True, + "checkedRequiresStringToolName": True, + "checkedRequiresObjectArguments": True, + "checkedRejectedRequestsDoNotCreateSessionFiles": True, + "checkedSkillWorkflow": True, + "checkedStatusNotUsedAsWaitLoopGuard": True, + "checkedMcpNoDirectEventContentsGuard": True, + "checkedWaitNotifyContract": True, + "checkedNotifySuppressedEventsPathEnv": True, + "checkedNotifyCallbackFailureExit": True, + "checkedNotifyCallbackFailureReason": True, + "checkedNotifyCallbackTimeoutFailureExit": True, + "checkedNotifyCallbackTimeoutReason": True, + "checkedRecordingToSkillHandoff": True, + "checkedStrictValidation": True, + "checkedDeclaredHandoffPaths": True, + "checkedScreenshotPathContainment": True, + "checkedEventsOnlyValidation": True, + "checkedScaffoldSkill": True, + "checkedScaffoldSkillFailureExit": True, + "checkedCancelledRecordingContract": True, + "checkedCancelledRecordingRejected": True, + "checkedSkillCreatorHandoff": True, + "checkedLifecycleSmoke": True, + "checkedOneActive": True, + "checkedIdempotentStop": True, + "checkedFinalStatus": True, + "checkedGeneratedRepoSelfCheck": True, + }, + sort_keys=True, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validate-event-stream-recording.py b/scripts/validate-event-stream-recording.py new file mode 100755 index 00000000..1471c00c --- /dev/null +++ b/scripts/validate-event-stream-recording.py @@ -0,0 +1,543 @@ +#!/usr/bin/env python3 + +import argparse +import json +import pathlib +import sys +from collections import Counter + + +FINAL_STATES = {"stopped", "cancelled"} +SKILL_DRAFT_ACTION_EVENT_TYPES = { + "mouse.click", + "mouse.context_menu", + "mouse.drag", + "keyboard.text_input", + "keyboard.submit", + "keyboard.shortcut", + "terminal.value_changed", + "selection.changed", +} +BLOCKING_DIAGNOSTIC_REASONS = {"inputMonitorsUnavailable"} +HANDOFF_PATH_KEYS = [ + "metadataPath", + "sessionPath", + "eventsPath", + "suppressedEventsPath", +] + + +def load_json(path): + try: + value = json.loads(path.read_text()) + except FileNotFoundError: + raise ValueError(f"missing JSON file: {path}") + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSON in {path}:{error.lineno}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}") + return value + + +def read_jsonl(path): + records = [] + if not path.exists(): + raise ValueError(f"missing JSONL file: {path}") + for index, line in enumerate(path.read_text().splitlines(), start=1): + if not line.strip(): + continue + try: + value = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError(f"invalid JSONL in {path}:{index}:{error.colno}: {error.msg}") + if not isinstance(value, dict): + raise ValueError(f"expected JSON object in {path}:{index}") + records.append(value) + return records + + +def event_kind(event): + kind = event.get("kind") + if isinstance(kind, str): + return kind + event_type = event.get("type") + if isinstance(event_type, str): + return event_type + return None + + +def resolve_input(path): + if path.is_dir(): + metadata_path = path / "metadata.json" + session_path = path / "session.json" + if metadata_path.exists(): + return path, metadata_path, None + if session_path.exists(): + return path, session_path, None + raise ValueError(f"session directory has no metadata.json or session.json: {path}") + if not path.exists(): + raise ValueError(f"path does not exist: {path}") + if path.name == "metadata.json": + return path.parent, path, None + if path.name == "session.json": + metadata_path = path.parent / "metadata.json" + return path.parent, metadata_path if metadata_path.exists() else path, None + if path.name == "events.jsonl": + return path.parent, None, path + raise ValueError("input must be a session directory, metadata.json, session.json, or events.jsonl") + + +def relative_path(base, raw): + candidate = pathlib.Path(raw) + if candidate.is_absolute(): + return candidate + return base / candidate + + +def path_from_metadata(value, key, fallback, base): + raw = value.get(key) + if isinstance(raw, str) and raw: + return relative_path(base, raw) + return fallback + + +def optional_path_from_metadata(value, key, base): + raw = value.get(key) + if isinstance(raw, str) and raw: + return relative_path(base, raw) + return None + + +def declared_handoff_path_evidence(metadata, base): + evidence = {} + for key in HANDOFF_PATH_KEYS: + raw = metadata.get(key) + entry = { + "value": raw, + "resolvedPath": None, + "exists": None, + } + if isinstance(raw, str) and raw: + resolved = relative_path(base, raw) + entry["resolvedPath"] = str(resolved) + entry["exists"] = resolved.exists() + evidence[key] = entry + return evidence + + +def declared_handoff_path_errors(evidence, require_all_paths): + errors = [] + for key in HANDOFF_PATH_KEYS: + entry = evidence.get(key, {}) + value = entry.get("value") + if isinstance(value, str) and value: + if entry.get("exists") is False: + errors.append(f"{key} does not exist: {value}") + continue + if value is not None and not (isinstance(value, str) and not value): + errors.append(f"{key} must be a non-empty string") + elif require_all_paths: + errors.append(f"strict OCU validation requires {key}") + return errors + + +def first_string(value, keys): + for key in keys: + candidate = value.get(key) + if isinstance(candidate, str) and candidate: + return candidate + return None + + +def session_alias_compatible(metadata, session_alias): + if metadata == session_alias: + return True + + metadata_session_id = first_string(metadata, ["sessionId", "sessionID", "id"]) + session_id = first_string(session_alias, ["id", "sessionId", "sessionID"]) + if metadata_session_id is None or session_id is None or metadata_session_id != session_id: + return False + + metadata_started_at = metadata.get("startedAt") + if isinstance(metadata_started_at, str) and metadata_started_at != session_alias.get("startedAt"): + return False + + for key in ["endedAt", "endReason"]: + session_value = session_alias.get(key) + if isinstance(session_value, str) and session_value != metadata.get(key): + return False + + session_events_path = session_alias.get("eventsPath") + metadata_events_path = metadata.get("eventsPath") + if not isinstance(session_events_path, str) or not session_events_path: + return False + if not isinstance(metadata_events_path, str) or not metadata_events_path: + return False + if ( + session_events_path != metadata_events_path + and pathlib.Path(session_events_path).name != pathlib.Path(metadata_events_path).name + ): + return False + + return True + + +def collect_screenshot_paths(value): + paths = [] + if isinstance(value, dict): + for key, child in value.items(): + if key == "screenshotPath" and isinstance(child, str) and child: + paths.append(child) + else: + paths.extend(collect_screenshot_paths(child)) + elif isinstance(value, list): + for child in value: + paths.extend(collect_screenshot_paths(child)) + return paths + + +def is_inside_directory(path, directory): + candidate = path.resolve(strict=False) + base = directory.resolve(strict=False) + return candidate == base or base in candidate.parents + + +def blocking_diagnostics(events): + diagnostics = [] + for index, event in enumerate(events, start=1): + if event_kind(event) != "debug.error": + continue + if event.get("reason") not in BLOCKING_DIAGNOSTIC_REASONS: + continue + diagnostic = {"line": index} + for key in ["subsystem", "reason", "errorType"]: + if key in event: + diagnostic[key] = event[key] + diagnostics.append(diagnostic) + return diagnostics + + +def recording_incomplete(state, active, ended_events): + if state == "recording" or active is True: + return True + return not ended_events + + +def skill_draft_blocking_reasons( + event_type_counts, + state, + end_reason, + incomplete, + blocking_diagnostics, + session_started_not_first, + session_started_count_invalid, + session_ended_not_final, + session_ended_count_invalid, +): + has_action = any(event_type_counts[event_type] > 0 for event_type in SKILL_DRAFT_ACTION_EVENT_TYPES) + recording_was_cancelled = state == "cancelled" or end_reason == "recording_controls_cancelled" + reasons = [] + if not has_action: + reasons.append("recording has no high-level user action events") + if recording_was_cancelled: + reasons.append("recording was cancelled; do not create or update a skill from this event stream") + if incomplete: + reasons.append("recording is not complete; stop the recording before creating a skill") + if blocking_diagnostics: + reasons.append("recording has blocking diagnostics; fix recording permissions and re-record before creating a skill") + if session_started_count_invalid: + reasons.append("recording must contain exactly one session.started event before creating a skill") + if session_started_not_first: + reasons.append("recording has events before session.started; start must be the first event before creating a skill") + if session_ended_not_final: + reasons.append( + "recording has events after session.ended; stop or cancel must be the final event before creating a skill" + ) + if session_ended_count_invalid: + reasons.append( + "recording has multiple session.ended events; stop or cancel must close the event stream exactly once" + ) + return reasons + + +def validate(path, strict_ocu, required_event_types, require_skill_draft=False): + session_dir, metadata_path, input_events_path = resolve_input(path) + metadata = load_json(metadata_path) if metadata_path is not None else {} + warnings = [] + errors = [] + + metadata_json_path = session_dir / "metadata.json" + session_alias_path = session_dir / "session.json" + if metadata_path is None: + if strict_ocu: + errors.append("strict OCU validation requires metadata.json or session.json") + else: + warnings.append("metadata/session files not available; validating events.jsonl only") + elif metadata_json_path.exists() and session_alias_path.exists(): + try: + metadata_json = load_json(metadata_json_path) + session_json = load_json(session_alias_path) + if not session_alias_compatible(metadata_json, session_json): + errors.append("metadata.json and session.json differ") + except ValueError as error: + errors.append(str(error)) + elif strict_ocu: + errors.append("strict OCU validation requires both metadata.json and session.json") + else: + warnings.append("metadata/session alias pair is incomplete") + + declared_paths = declared_handoff_path_evidence(metadata, session_dir) + errors.extend(declared_handoff_path_errors(declared_paths, strict_ocu)) + + state = metadata.get("state") + current_segment_events_path = optional_path_from_metadata( + metadata, + "currentSegmentEventsPath", + session_dir, + ) + current_segment_metadata_path = optional_path_from_metadata( + metadata, + "currentSegmentMetadataPath", + session_dir, + ) + if current_segment_events_path is not None and not current_segment_events_path.exists(): + errors.append(f"currentSegmentEventsPath does not exist: {metadata['currentSegmentEventsPath']}") + if current_segment_metadata_path is not None and not current_segment_metadata_path.exists(): + errors.append(f"currentSegmentMetadataPath does not exist: {metadata['currentSegmentMetadataPath']}") + if strict_ocu: + if state == "recording": + if current_segment_events_path is None: + errors.append("recording state requires currentSegmentEventsPath") + if current_segment_metadata_path is None: + errors.append("recording state requires currentSegmentMetadataPath") + elif state in FINAL_STATES: + if current_segment_events_path is not None: + errors.append("final state must not include currentSegmentEventsPath") + if current_segment_metadata_path is not None: + errors.append("final state must not include currentSegmentMetadataPath") + + events_path = input_events_path or path_from_metadata(metadata, "eventsPath", session_dir / "events.jsonl", session_dir) + suppressed_path = path_from_metadata( + metadata, + "suppressedEventsPath", + session_dir / "suppressed.jsonl", + session_dir, + ) + + try: + events = read_jsonl(events_path) + except ValueError as error: + errors.append(str(error)) + events = [] + + suppressed_events = [] + if suppressed_path.exists(): + try: + suppressed_events = read_jsonl(suppressed_path) + except ValueError as error: + errors.append(str(error)) + elif strict_ocu or metadata.get("suppressedEventsPath"): + errors.append(f"suppressedEventsPath does not exist: {suppressed_path}") + else: + warnings.append(f"suppressed events file not found: {suppressed_path}") + + event_count = metadata.get("eventCount") + if isinstance(event_count, int) and event_count != len(events): + errors.append(f"eventCount={event_count} does not match events.jsonl lines={len(events)}") + elif event_count is None: + warnings.append("metadata has no eventCount") + + suppressed_count = metadata.get("suppressedEventCount") + if isinstance(suppressed_count, int) and suppressed_count != len(suppressed_events): + errors.append( + f"suppressedEventCount={suppressed_count} does not match suppressed.jsonl lines={len(suppressed_events)}" + ) + elif suppressed_count is None: + warnings.append("metadata has no suppressedEventCount") + + event_types = [kind for event in events if (kind := event_kind(event))] + event_type_counts = Counter(event_types) + for event_type in required_event_types: + if event_type_counts[event_type] == 0: + errors.append(f"missing required event type: {event_type}") + + end_reason = metadata.get("endReason") + started_events = [event for event in events if event_kind(event) == "session.started"] + session_started_count_invalid = len(started_events) != 1 + first_event_type = event_kind(events[0]) if events else None + session_started_is_first = first_event_type == "session.started" + session_started_not_first = bool(started_events) and not session_started_is_first + ended_events = [event for event in events if event_kind(event) == "session.ended"] + ended_reasons = { + event.get("endReason") + for event in ended_events + if isinstance(event.get("endReason"), str) + } + session_ended_count_invalid = len(ended_events) > 1 + final_event_type = event_kind(events[-1]) if events else None + session_ended_is_final = final_event_type == "session.ended" + session_ended_not_final = bool(ended_events) and not session_ended_is_final + inferred_end_reason = end_reason or (next(iter(ended_reasons)) if len(ended_reasons) == 1 else None) + incomplete = recording_incomplete(state, metadata.get("active"), ended_events) + blocking_diagnostic_events = blocking_diagnostics(events) + skill_draft_reasons = skill_draft_blocking_reasons( + event_type_counts, + state, + inferred_end_reason, + incomplete, + blocking_diagnostic_events, + session_started_not_first, + session_started_count_invalid, + session_ended_not_final, + session_ended_count_invalid, + ) + if require_skill_draft: + errors.extend(skill_draft_reasons) + + if session_started_count_invalid: + message = "recording has no session.started event" if not started_events else "recording has multiple session.started events" + if strict_ocu: + errors.append(message) + elif not require_skill_draft: + warnings.append(message) + + if session_started_not_first: + message = "session.started is not the first event" + if strict_ocu: + errors.append(message) + elif not require_skill_draft: + warnings.append(message) + + if session_ended_count_invalid: + message = "recording has multiple session.ended events" + if strict_ocu: + errors.append(message) + elif not require_skill_draft: + warnings.append(message) + + if session_ended_not_final: + message = "session.ended is not the final event" + if strict_ocu: + errors.append(message) + elif not require_skill_draft: + warnings.append(message) + + if state in FINAL_STATES: + if not ended_events: + message = f"final state {state} has no session.ended event" + if strict_ocu: + errors.append(message) + else: + warnings.append(message) + if end_reason: + if ended_reasons and end_reason not in ended_reasons: + errors.append( + f"metadata endReason={end_reason} not present in session.ended events: {sorted(ended_reasons)}" + ) + + session_id = metadata.get("sessionId") or metadata.get("sessionID") + if isinstance(session_id, str) and session_id: + mismatched_events = [ + index + for index, event in enumerate(events, start=1) + if isinstance(event.get("sessionId"), str) and event.get("sessionId") != session_id + ] + if mismatched_events: + errors.append(f"event sessionId mismatch at JSONL lines: {mismatched_events[:10]}") + + for event_index, event in enumerate(events, start=1): + for screenshot_path in collect_screenshot_paths(event): + candidate = pathlib.Path(screenshot_path) + if not candidate.is_absolute(): + candidate = session_dir / candidate + if not is_inside_directory(candidate, session_dir): + errors.append( + f"screenshotPath from event line {event_index} must stay inside session directory: {screenshot_path}" + ) + continue + if not candidate.exists(): + errors.append(f"screenshotPath from event line {event_index} does not exist: {screenshot_path}") + + if strict_ocu: + if metadata.get("active") is not (state == "recording"): + errors.append("metadata active flag does not match state") + if state != "recording" and (session_dir.parent / "active-session.json").exists(): + errors.append("active-session.json exists after final session state") + + return { + "ok": not errors, + "sessionDir": str(session_dir), + "metadataPath": str(metadata_path) if metadata_path is not None else None, + "sessionPath": str(session_alias_path), + "eventsPath": str(events_path), + "suppressedEventsPath": str(suppressed_path), + "currentSegmentEventsPath": str(current_segment_events_path) if current_segment_events_path else None, + "currentSegmentMetadataPath": str(current_segment_metadata_path) if current_segment_metadata_path else None, + "declaredPaths": declared_paths, + "state": state, + "active": metadata.get("active"), + "endReason": inferred_end_reason, + "sessionId": session_id, + "eventCount": len(events), + "metadataEventCount": event_count, + "suppressedEventCount": len(suppressed_events), + "metadataSuppressedEventCount": suppressed_count, + "eventTypes": dict(sorted(event_type_counts.items())), + "sessionStartedCount": len(started_events), + "sessionStartedCountInvalid": session_started_count_invalid, + "firstEventType": first_event_type, + "sessionStartedIsFirst": session_started_is_first if first_event_type is not None else None, + "sessionEndedCount": len(ended_events), + "sessionEndedCountInvalid": session_ended_count_invalid, + "finalEventType": final_event_type, + "sessionEndedIsFinal": session_ended_is_final if final_event_type is not None else None, + "requireSkillDraft": require_skill_draft, + "skillDraftReady": not skill_draft_reasons, + "skillDraftReasons": skill_draft_reasons, + "recordingIncomplete": incomplete, + "blockingDiagnostics": blocking_diagnostic_events, + "warnings": warnings, + "errors": errors, + } + + +def main(): + parser = argparse.ArgumentParser(description="Validate a Record & Replay event-stream recording directory.") + parser.add_argument("path", type=pathlib.Path, help="Session directory, metadata.json, session.json, or events.jsonl") + parser.add_argument( + "--strict-ocu", + action="store_true", + help="Require OCU baseline files and final-state invariants.", + ) + parser.add_argument( + "--require-event-type", + action="append", + default=[], + help="Require at least one event of this type. May be passed multiple times.", + ) + parser.add_argument( + "--require-skill-draft", + action="store_true", + help="Require enough evidence to create a first-draft skill: at least one high-level action and no cancelled end state.", + ) + args = parser.parse_args() + + try: + result = validate(args.path, args.strict_ocu, args.require_event_type, args.require_skill_draft) + except ValueError as error: + result = { + "ok": False, + "errors": [str(error)], + "warnings": [], + } + + output = json.dumps(result, indent=2, sort_keys=True) + if result["ok"]: + print(output) + return 0 + print(output, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/open-computer-use-record-and-replay/SKILL.md b/skills/open-computer-use-record-and-replay/SKILL.md new file mode 100644 index 00000000..b5ec4c7b --- /dev/null +++ b/skills/open-computer-use-record-and-replay/SKILL.md @@ -0,0 +1,146 @@ +--- +name: open-computer-use-record-and-replay +description: Record a user's macOS workflow with Open Computer Use Record & Replay and turn the captured event stream into a reusable skill or automation. Use when the user wants to demonstrate steps, capture a workflow, replay a process, or create a skill from recorded desktop actions without depending on Codex.app-specific UI. +--- + +# Open Computer Use Record & Replay + +## Overview + +Use this skill to record a user's macOS workflow through Open Computer Use and convert the captured event stream into reusable automation. This workflow is independent of Codex.app-specific plugin UI; Open Computer Use owns the start approval, recording controls, session files, wait listener, and notification handoff. + +Current scope is macOS only. + +## Required MCP Server + +Configure the Record & Replay MCP surface separately from the normal Computer Use MCP server. For Codex, prefer the installer: + +```sh +open-computer-use install-codex-record-and-replay-mcp +``` + +Equivalent manual config: + +```toml +[mcp_servers.record_and_replay] +command = "open-computer-use" +args = ["event-stream", "mcp"] +``` + +This server exposes only the official-compatible tools: + +```text +event_stream_start +event_stream_status +event_stream_stop +``` + +Do not add cancel, wait, callback, webhook, or extra arguments to this MCP surface. Those are Open Computer Use extension-layer CLI features. + +## Recording Workflow + +1. Confirm the user is ready to record. Starting may show an Open Computer Use approval dialog and recording control bar. +2. Call `event_stream_start` once. If it reports an already active recording, do not start another one. Explain that a recording is already in progress and ask whether the user wants to use that active recording or wait until it is stopped. +3. End the turn and let the user perform the workflow. Ask the user to tell you when they are done, and tell them the recording can last up to 30 minutes. Do not poll while the user is recording. +4. Use `event_stream_status` only when the user asks for status or returns after recording; do not use it to wait for completion. +5. When the user says they are done, call `event_stream_stop`. +6. Parse the returned text JSON and read `eventsPath`, `metadataPath`, and, when present, `sessionPath`. The MCP server does not expose event-stream contents directly. +7. Treat `events.jsonl` as the primary evidence. Use `metadata.json` or `session.json` only for session timing, state, counts, paths, and `endReason`. +8. Validate and summarize the recording before generating any reusable asset: + +```sh +open-computer-use event-stream validate --json --strict-ocu --require-skill-draft +open-computer-use event-stream validate --json --require-skill-draft +open-computer-use event-stream summarize --json +``` + +Use the strict validation command for complete OCU session output. It proves metadata/session alias consistency and that the declared `metadataPath`, `sessionPath`, `eventsPath`, and `suppressedEventsPath` handoff paths resolve to files. If only `eventsPath` is available, run the non-strict events-only validator; it can still check JSONL readability, completion, whether `session.started` appears exactly once and is the first event, whether `session.ended` appears exactly once and is the final event, high-level action evidence, cancellation from `session.ended.endReason=recording_controls_cancelled`, and blocking diagnostics, but it cannot prove metadata/session alias consistency or declared handoff paths. + +For a repeatable first draft, use the installed runtime CLI: + +```sh +open-computer-use event-stream scaffold-skill --json \ + --skill-name \ + --description "" \ + --output-dir +``` + +When developing inside this source checkout, the Python scaffold helper should +produce the same draft shape and can be used as a cross-check: + +```sh +scripts/scaffold-event-stream-skill.py \ + --skill-name \ + --description "" \ + --output-dir +``` + +The scaffold runs the same skill-draft validator gate before writing files, then writes `SKILL.md` and `references/recording-summary.json`. The generated `SKILL.md` includes inferred runtime inputs, an agent replay procedure that prefers `get_app_state` and semantic `element_index` actions, observed replay steps, and a verification checklist. It intentionally produces a draft, not a fully verified automation: review the original `events.jsonl`, replace user-specific values with inputs, and tighten the replay instructions before packaging or installing the generated skill. + +The summary and generated draft include `skillReadiness` with `ready`, `needsReview`, or `insufficient` status. Treat `needsReview` as a signal to inspect `events.jsonl`, replace placeholders, and verify missing context before finalizing. Treat `insufficient` as a failed or incomplete demonstration: stop the recording if it is still active, or ask the user to re-record or provide the missing workflow information instead of inventing steps. Cancelled, incomplete, and malformed recordings are always insufficient for skill creation; scaffold commands should reject them instead of writing a draft, including events-only streams whose `session.ended.endReason` is `recording_controls_cancelled`, whose `session.started` is missing, duplicated, or not the first event, whose `session.ended` appears more than once, or whose `session.ended` is not the final event. + +The summary and generated draft can also include `runtimeInputs`, `safetySignals`, `summaryLimits`, and `blockingDiagnostics`: + +- Use `runtimeInputs` to identify text values, selected-content semantics, paths, names, dates, or other values that must become explicit runtime inputs instead of copied from the recording. +- Use `safetySignals` to find recorded submit/send/delete/save/upload/publish-style actions that require explicit user confirmation before replay. These signals are hints from the summary helper, not a replacement for reading `events.jsonl`; still inspect neighboring events and app/window context before finalizing safety instructions. +- If `skillEvidence.hasSafetySignals=true`, keep the generated Confirmation Signals or equivalent wording in the final skill and make the replay procedure pause before those steps. +- If `summaryLimits.hasTruncatedSummary=true`, the summary omitted some high-volume fields such as later replay steps, targets, runtime inputs, or diagnostics. Do not finalize the skill from the draft alone; inspect the original `events.jsonl` for omitted steps and update the final skill accordingly. +- If `skillEvidence.recordingIncomplete=true`, do not create or update a skill from that recording. Stop the active recording first, then validate and summarize the completed session. +- If `skillEvidence.sessionStartedCountInvalid=true` or `sessionStartedCount` is not 1, do not create or update a skill from that recording. Ask the user to re-record so the start event is written exactly once. +- If `skillEvidence.sessionStartedNotFirst=true` or `sessionStartedIsFirst=false`, do not create or update a skill from that recording. Ask the user to re-record so the start event opens the stream. +- If `skillEvidence.sessionEndedNotFinal=true` or `sessionEndedIsFinal=false`, do not create or update a skill from that recording. Ask the user to re-record so the stop/cancel event closes the event stream. +- If `skillEvidence.sessionEndedCountInvalid=true` or `sessionEndedCount` is greater than 1, do not create or update a skill from that recording. Ask the user to re-record so the stop/cancel event is written exactly once. +- If `skillEvidence.hasBlockingDiagnostics=true` or `blockingDiagnostics` is non-empty, do not create or update a skill from that recording. Fix the reported recording permission or input monitoring issue, then ask the user to re-record. + +If the recording contains enough information to identify a reusable workflow, do not stop at a summary, runbook, or replay plan. Use the scaffold as the starting artifact, then follow the `skill-creator` workflow to refine, validate, package, or install an actual discoverable skill. + +If the user says they clicked Discard or cancelled recording, do not call `event_stream_stop` again. Read `sessionPath` or `session.json` only if you need to confirm `endReason=recording_controls_cancelled`, then acknowledge the cancellation and ask the user to re-record before creating or updating a skill. + +## Creating The Reusable Skill + +After validation, inspect the summary first, then inspect `events.jsonl` for the action details that matter. Build a reusable skill from the observed workflow rather than only summarizing the recording. + +Before finalizing the skill, read and follow the `skill-creator` skill. Complete the `skill-creator` workflow, including validation, before reporting that the skill was created. The generated scaffold is a first draft; the final deliverable should be a usable, discoverable skill directory with clear triggers, prerequisites, runtime inputs, safety confirmations, replay instructions, and verification steps. + +When creating the skill: + +- Preserve the user's goal and decision points, not every raw mouse coordinate. +- Check whether an available connector, API, or dedicated tool can perform stable semantic operations more reliably than UI replay; use Computer Use for unsupported UI interactions, visually dependent verification, or when manipulating the interface is itself the task. +- Prefer app names, visible labels, element roles, text entry targets, and keyboard shortcuts over brittle coordinates. +- Use placeholders for user-specific values such as names, emails, paths, account ids, message text, or dates. +- Include prerequisites such as required apps, login state, permissions, or files. +- Call out destructive or externally visible steps that require confirmation before execution, using `safetySignals` as the first checklist and `events.jsonl` as the final evidence. +- Keep the resulting skill independent of Codex.app; reference `open-computer-use` CLI and MCP commands instead. + +If the recording is too sparse, mostly empty, or contains only diagnostics such as `inputMonitorsUnavailable`, ask the user to retry after fixing permissions instead of inventing workflow steps. + +## Independent Wait / Notify Integration + +For a plugin or wrapper that needs to resume automatically after the user clicks Done or Discard, use the Open Computer Use CLI extension layer: + +```sh +open-computer-use event-stream start --json +open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' +``` + +`wait --json` adds `waitTimedOut` and `waitSessionMatched`. `waitTimedOut=false` means the waiter was released by stop, cancel, or automatic end for the requested session. `waitSessionMatched=false` means the requested session id was not found or the current active session does not match; in that case `waitTimedOut=true` and notification callbacks are skipped. + +`--notify-command` receives the final status JSON on stdin and these environment variables when present: + +- `OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID` +- `OPEN_COMPUTER_USE_EVENT_STREAM_STATE` +- `OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON` +- `OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` + +Keep this listener path outside the official-compatible MCP tool surface. + +## Safety + +- Do not record passwords, OTPs, API keys, financial account numbers, private medical/legal/HR content, or unrelated private content. +- Ask before creating automation that sends, deletes, purchases, approves, uploads, or changes external state. +- Secure input and obvious terminal buffers may be redacted; do not reconstruct redacted secrets from context. +- If screenshots are present, treat them as sensitive evidence and avoid copying their content into the generated skill unless it is essential and sanitized. diff --git a/skills/open-computer-use/SKILL.md b/skills/open-computer-use/SKILL.md index 42f98981..9e094680 100644 --- a/skills/open-computer-use/SKILL.md +++ b/skills/open-computer-use/SKILL.md @@ -23,7 +23,8 @@ It supports the same core tool surface across macOS, Linux, and Windows: 6. Prefer element-targeted actions using `element_index` from the latest `get_app_state` result. 7. For multi-step CLI work, use `open-computer-use call --calls ''` so one process can reuse the latest element index mapping. 8. For agent runtimes that support local MCP servers, configure `open-computer-use mcp` or `ocu mcp` and call the exposed Computer Use tools directly. Read [references/usage.md](references/usage.md). -9. If communication, permission, or desktop-session access fails, read [references/troubleshooting.md](references/troubleshooting.md). +9. When a user wants to demonstrate a macOS workflow for reusable automation, read [references/record-and-replay.md](references/record-and-replay.md). +10. If communication, permission, or desktop-session access fails, read [references/troubleshooting.md](references/troubleshooting.md). ## Operating Rules @@ -74,4 +75,5 @@ Read [references/usage.md](references/usage.md) for JSON config examples, direct - [references/installation.md](references/installation.md): one-time CLI install, agent MCP install commands, and macOS permissions. - [references/usage.md](references/usage.md): MCP config, direct CLI calls, sequencing, and platform behavior. +- [references/record-and-replay.md](references/record-and-replay.md): macOS Record & Replay-compatible event-stream MCP, session files, and OCU wait/cancel extensions. - [references/troubleshooting.md](references/troubleshooting.md): permission, desktop-session, app discovery, and action failures. diff --git a/skills/open-computer-use/references/record-and-replay.md b/skills/open-computer-use/references/record-and-replay.md new file mode 100644 index 00000000..172fea0a --- /dev/null +++ b/skills/open-computer-use/references/record-and-replay.md @@ -0,0 +1,200 @@ +# Open Computer Use Record & Replay + +Read this reference when the user wants to demonstrate a macOS workflow and turn the captured event stream into reusable automation. + +Record & Replay support is macOS-only in the current baseline. It exposes an official-compatible event-stream MCP surface plus OCU-only CLI lifecycle helpers for independent integrations. + +## MCP Server + +For Codex, install the separate Record & Replay MCP entry with: + +```sh +open-computer-use install-codex-record-and-replay-mcp +``` + +For other MCP clients that support stdio servers: + +```toml +[mcp_servers.record_and_replay] +command = "open-computer-use" +args = ["event-stream", "mcp"] +``` + +Equivalent JSON shape: + +```json +{ + "mcpServers": { + "record-and-replay": { + "command": "open-computer-use", + "args": ["event-stream", "mcp"] + } + } +} +``` + +This MCP server is a separate surface from `open-computer-use mcp`. It exposes only the official-compatible tools: + +```text +event_stream_start +event_stream_status +event_stream_stop +``` + +Do not add `cancel`, `wait`, callback, or webhook arguments to this MCP surface. Those are OCU extension-layer concerns. + +If the MCP host supports elicitation and should own the approval prompt, launch the server with: + +```sh +OPEN_COMPUTER_USE_EVENT_STREAM_START_APPROVAL=mcp open-computer-use event-stream mcp +``` + +The server will request approval with MCP `elicitation/create` only when the host declares `capabilities.elicitation` during `initialize`. Accept starts the recording; decline, cancel, or unsupported elicitation returns an approval error without creating a session. OCU sends a host-compatible `mode: "form"` request with an empty object schema; the exact official Record & Replay business message / metadata still needs golden recording calibration. This path does not change the official-compatible three-tool surface. + +## Recording Workflow + +1. Confirm the user is ready before starting. Starting may show a local approval UI. +2. Call `event_stream_start` once. If it reports an already active recording, do not start another one. Explain that a recording is already in progress and ask whether the user wants to use that active recording or wait until it is stopped. +3. Let the user perform the workflow. Ask the user to tell you when they are done, and tell them the recording can last up to 30 minutes. Do not poll in a loop while they are recording. +4. When the user says they are done, call `event_stream_stop`. +5. Read `eventsPath` and `metadataPath` from the returned status. +6. Treat `events.jsonl` as the primary evidence. `metadata.json` and `session.json` provide session timing, paths, counts, state, and `endReason`. + +If the user says they cancelled recording, do not call `event_stream_stop` again. Read `session.json` only when needed to confirm `endReason`. OCU status JSON also includes `sessionPath` as a convenience alias path; official-compatible workflows should still treat `metadataPath` and `eventsPath` as the primary returned paths. + +## Session Files + +Each recording writes a session directory containing: + +```text +events.jsonl +metadata.json +session.json +suppressed.jsonl +screenshots/ +``` + +`session.json` is an alias of `metadata.json`. `suppressed.jsonl` contains events or diagnostics omitted from the main stream because of limits, privacy policy, or degraded context collection. + +OCU status and metadata include `sessionPath` as a convenience field pointing at the `session.json` alias. This is an OCU integration aid, not an extra MCP tool or parameter. + +The main event stream can include mouse, keyboard, window, selection, terminal, debug, and AX payload events. AX payloads may be full trees or compact diffs using `~`, `+`, and `-`. + +## OCU CLI Extensions + +Independent integrations can control the same app-agent recorder with: + +```sh +open-computer-use event-stream start --json +open-computer-use event-stream status --json +open-computer-use event-stream stop --json +open-computer-use event-stream cancel --json +open-computer-use event-stream wait --json --session-id --timeout +open-computer-use event-stream wait --json --session-id --notify-command '["/path/to/hook"]' +open-computer-use event-stream validate --json +open-computer-use event-stream summarize --json +open-computer-use event-stream scaffold-skill --json --skill-name --output-dir +``` + +`wait --json` returns `waitTimedOut` and `waitSessionMatched`, which are OCU extension fields. `waitTimedOut=false` means the waiter was released by stop/cancel/end for the requested session. `waitSessionMatched=false` means the requested session id was not found or the current active session does not match; in that case `waitTimedOut=true` and notification callbacks are skipped. + +`--notify-command` is also an OCU extension. It accepts a JSON argv array, runs only when `waitTimedOut=false`, and receives the final status JSON on stdin plus these environment variables when present: + +- `OPEN_COMPUTER_USE_EVENT_STREAM_STATUS_JSON` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_ID` +- `OPEN_COMPUTER_USE_EVENT_STREAM_STATE` +- `OPEN_COMPUTER_USE_EVENT_STREAM_END_REASON` +- `OPEN_COMPUTER_USE_EVENT_STREAM_METADATA_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SESSION_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_EVENTS_PATH` +- `OPEN_COMPUTER_USE_EVENT_STREAM_SUPPRESSED_EVENTS_PATH` + +The callback is synchronous and bounded by `OPEN_COMPUTER_USE_EVENT_STREAM_NOTIFY_TIMEOUT_SECONDS` or 10 seconds by default. A non-zero exit or timeout marks the CLI result as an error and adds a `notification` object to the JSON response. + +Use `wait` for independent plugin or skill integrations that need to resume work when the user clicks Done or Discard in OCU's recording controls. Keep `wait` out of the official-compatible MCP tools. + +## Validation + +Useful local checks: + +```sh +make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_NO_ACTIVE=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_MCP_ELICITATION=1 make event-stream-smoke +OPEN_COMPUTER_USE_EVENT_STREAM_SMOKE_OFFICIAL=1 make event-stream-smoke +``` + +The official compare smoke only checks `initialize` and `tools/list`; it does not start an official recording. + +To validate an existing recording directory or returned `metadataPath`: + +```sh +open-computer-use event-stream validate --json +open-computer-use event-stream validate --json --strict-ocu --require-skill-draft +open-computer-use event-stream validate --json --require-skill-draft +scripts/validate-event-stream-recording.py +scripts/validate-event-stream-recording.py --strict-ocu --require-skill-draft +scripts/validate-event-stream-recording.py --require-skill-draft +scripts/summarize-event-stream-recording.py +open-computer-use event-stream summarize --json +``` + +Use `--strict-ocu` for complete recordings produced by Open Computer Use. It checks metadata/session alias consistency and verifies that the declared `metadataPath`, `sessionPath`, `eventsPath`, and `suppressedEventsPath` handoff paths resolve to files. Leave it off for newly captured official golden recordings until their exact alias/index behavior has been confirmed. Non-strict validation can also take `events.jsonl` directly; in that mode it validates the event stream, confirms `session.started` appears exactly once and opens the stream, confirms `session.ended` appears exactly once and closes the stream when present, checks skill-draft evidence, infers cancellation from `session.ended.endReason=recording_controls_cancelled`, and only warns that metadata/session files are unavailable. + +Use the runtime CLI as the preferred integration point for an installed OCU binary. `event-stream validate` checks session file consistency when metadata is present and returns `ok=false` as a tool-style error when required structure is missing; add `--require-skill-draft` before scaffold generation to require at least one high-level action and reject incomplete recordings, cancelled recordings, recordings where `session.started` is missing, duplicated, or not the first event, recordings where `session.ended` appears more than once or is not the final event, or recordings with blocking diagnostics such as unavailable input monitoring. `event-stream scaffold-skill` and the source Python scaffold also run this validator gate before writing files, so malformed metadata/session/events cannot be turned into a draft just because the stream can be summarized. The Python validator remains a source-checkout helper for OCU development and official golden recording calibration. + +Use the summary output to quickly inspect event counts, windows, action sequence, target/focused elements, selection signals, debug errors, and redaction signals before creating a skill. It is a triage aid; `events.jsonl` remains the primary evidence. Both summary paths omit raw AX `value` and `selectedText` by default and report lengths instead; pass `--include-text` only when reviewing a deliberately sanitized recording. + +Summary output can include: + +- `runtimeInputs`: text-entry and selection-derived values that should become explicit runtime inputs in the final skill. +- `safetySignals`: submit/send/delete/save/upload/publish-style confirmation hints extracted from action events and semantic target titles. Treat them as a first checklist for confirmation-sensitive steps, then verify against `events.jsonl` and nearby app/window context before finalizing the skill. +- `summaryLimits`: caps, stored counts, source counts, and omitted counts for high-volume summary fields. If `summaryLimits.hasTruncatedSummary=true`, the draft may omit later replay steps or evidence; read the original `events.jsonl` before finalizing. +- `skillEvidence.recordingIncomplete`: the recording is still active or lacks `session.ended`. Stop the recording and use the completed session before creating a skill. +- `skillEvidence.sessionStartedCountInvalid`: `session.started` is missing or duplicated. Treat the recording as malformed and ask the user to re-record before creating a skill. +- `skillEvidence.sessionStartedNotFirst`: events appear before `session.started`. Treat the recording as malformed and ask the user to re-record before creating a skill. +- `skillEvidence.sessionEndedNotFinal`: events appear after `session.ended`. Treat the recording as malformed and ask the user to re-record before creating a skill. +- `skillEvidence.sessionEndedCountInvalid`: multiple `session.ended` events appear. Treat the recording as malformed and ask the user to re-record before creating a skill. +- `blockingDiagnostics`: recording failures that make the demonstration unfit for skill creation, currently including `inputMonitorsUnavailable`. If present, fix permissions or recording setup and re-record instead of generating a skill. +- `skillEvidence.hasSafetySignals`: a boolean that should make the generated or final skill preserve Confirmation Signals and pause for explicit user approval before those steps. + +Summary output includes `skillReadiness.status`: + +- `ready`: enough structural evidence was captured to create a reusable skill draft. +- `needsReview`: a draft can be created, but missing context, diagnostics, or redaction signals require human review before finalizing. +- `insufficient`: do not invent a workflow; ask the user to re-record or provide the missing information. + +Incomplete, cancelled, and malformed recordings are always `insufficient` for skill creation. If metadata state is `recording`, `active=true`, or `session.ended` is missing, stop the recording and re-run validation on the completed session. If metadata state is `cancelled`, metadata `endReason=recording_controls_cancelled`, an events-only stream contains `session.ended.endReason=recording_controls_cancelled`, `session.started` is missing, duplicated, or not first, `session.ended` appears more than once, or `session.ended` is not the final event, summary/scaffold tooling should refuse to create a draft and the agent should ask the user to re-record. + +To create a repeatable first-draft skill directory from a validated recording with an installed OCU runtime: + +```sh +open-computer-use event-stream scaffold-skill --json \ + --skill-name \ + --description "" \ + --output-dir +``` + +When working in this source checkout, the Python helper is still useful as a cross-check: + +```sh +scripts/scaffold-event-stream-skill.py \ + --skill-name \ + --description "" \ + --output-dir +``` + +The scaffold writes a `SKILL.md` draft plus `references/recording-summary.json` only after the validator gate succeeds. The draft includes workflow readiness, inferred runtime inputs, confirmation signals, an agent replay procedure that first checks connectors/APIs/dedicated tools and then prefers `get_app_state` plus semantic `element_index` actions for UI-dependent work, observed replay steps, a verification checklist, and a finalization checklist that points back to `skill-creator`. It sanitizes local file paths and avoids raw AX text by default. Treat the result as an editable skill draft, not a standalone runbook: inspect `events.jsonl`, replace user-specific values with explicit inputs, tighten any destructive or externally visible steps, then complete the `skill-creator` workflow and validation before reporting the actual discoverable skill as created. + +For automated tests, use: + +```sh +swift test --filter EventStream +``` + +## Safety + +- Do not record passwords, OTPs, API keys, financial account numbers, private medical/legal/HR content, or other unrelated sensitive information. +- Secure or protected text input may be redacted in the event stream. +- Obvious terminal buffers are treated conservatively and should not be summarized as raw user content unless the user explicitly asks. +- Ask before creating or running a reusable skill that would submit, send, delete, purchase, approve, upload, publish, save externally visible state, or otherwise act on the user's behalf; preserve any generated `safetySignals` / Confirmation Signals until the final skill has an equivalent explicit approval step.