diff --git a/.github/workflows/README.md b/.github/workflows/README.md index b4809ec54..de7009d83 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -205,7 +205,7 @@ The required Rust + Binding RC run IDs are an input contract for this workflow only. They do **not** make the cascade a close gate for child implementation or construction issues; those close on outcomes (see `AGENTS.md` § Issue close). -### `binding-release-candidate.yml`, `release-credential-preflight.yml`, and `publish.yaml` +### `binding-release-candidate.yml`, `publish-track.yml`, `release-credential-preflight.yml`, and `publish.yaml` The exact-SHA Binding RC retains tested release bytes and their partitioned v2 candidate manifest for 30 days. Credential preflight verifies the npm/crates.io @@ -219,6 +219,15 @@ bytes only. Skip re-RC when a complete unexpired candidate for the current ≤35m cold; publish-track ≤35m p50 / ≤50m cold (see TESTING.md). M1, checkpoint, and m20/m21 are **not** required on this path. +`publish-track.yml` schedules exact-main Binding RC dispatch every six hours. +It reassembles and validates every retained partition before deciding a +candidate is reusable. Schedule runs never tag or publish. A maintainer must +set both `create_release` and `confirm_registry_publish` and supply the exact +root-version tag to create a published GitHub Release; that event triggers +`publish.yaml`. Mixed SHA, incomplete/expired partitions, tag disagreement, +existing Release identity, and all `publish.yaml` registry conflict checks fail +closed. + ### `clean-env-verify.yml` Maintainer `workflow_dispatch` after section 6 publication. Installs from **public** diff --git a/.github/workflows/publish-track.yml b/.github/workflows/publish-track.yml new file mode 100644 index 000000000..d5a28a5be --- /dev/null +++ b/.github/workflows/publish-track.yml @@ -0,0 +1,239 @@ +name: Publish Track + +on: + schedule: + # Certify the reviewed main tip without coupling frequent publication to M1. + - cron: "17 */6 * * *" + workflow_dispatch: + inputs: + commit_sha: + description: Current 40-character main SHA to certify (blank resolves main) + required: false + type: string + create_release: + description: Create the immutable GitHub Release identity after candidate validation + required: true + default: false + type: boolean + confirm_registry_publish: + description: Required with create_release; the published Release triggers publish.yaml + required: true + default: false + type: boolean + release_tag: + description: Exact v tag; required only when creating the Release + required: false + type: string + +permissions: + actions: read + contents: read + +concurrency: + group: publish-track-${{ inputs.commit_sha || github.run_id }} + cancel-in-progress: false + +jobs: + resolve_source: + name: Resolve exact main source + runs-on: blacksmith-4vcpu-ubuntu-2404 + outputs: + release_sha: ${{ steps.source.outputs.release_sha }} + release_version: ${{ steps.source.outputs.release_version }} + steps: + - uses: actions/checkout@v7 + with: + ref: main + fetch-depth: 1 + - name: Require current main SHA and aligned release version + id: source + shell: bash + env: + REQUESTED_SHA: ${{ inputs.commit_sha || '' }} + run: | + git fetch --no-tags --depth=1 origin \ + +refs/heads/main:refs/remotes/origin/main + main_sha="$(git rev-parse refs/remotes/origin/main)" + if test -n "$REQUESTED_SHA"; then + [[ "$REQUESTED_SHA" =~ ^[0-9a-f]{40}$ ]] + test "$REQUESTED_SHA" = "$main_sha" + fi + git checkout --detach "$main_sha" + python3 scripts/set_release_version.py --check + release_version="$(python3 - <<'PY' + import re + from pathlib import Path + text = Path("Cargo.toml").read_text(encoding="utf-8") + match = re.search(r'(?m)^version = "([^"]+)"$', text) + if match is None: + raise SystemExit("workspace release version is missing") + print(match.group(1)) + PY + )" + case "$release_version" in + *dev*) echo "publish-track requires a non-development version" >&2; exit 1 ;; + esac + printf 'release_sha=%s\n' "$main_sha" >> "$GITHUB_OUTPUT" + printf 'release_version=%s\n' "$release_version" >> "$GITHUB_OUTPUT" + + locate_candidate: + name: Locate complete retained candidate + needs: resolve_source + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + actions: read + contents: read + outputs: + candidate_run_id: ${{ steps.candidate.outputs.run_id }} + steps: + - name: Require one unexpired partition per candidate group + id: candidate + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ needs.resolve_source.outputs.release_sha }} + run: | + run_id="$(gh api \ + "repos/$GITHUB_REPOSITORY/actions/workflows/binding-release-candidate.yml/runs?event=workflow_dispatch&status=success&head_sha=$RELEASE_SHA&per_page=100" \ + --jq '.workflow_runs | map(select(.head_sha == env.RELEASE_SHA)) | first | .id // empty')" + if test -z "$run_id"; then + exit 0 + fi + for group in manifest python npm crates evidence; do + artifact_name="M1-Release-Candidate-$group-$RELEASE_SHA" + count="$(gh api "repos/$GITHUB_REPOSITORY/actions/runs/$run_id/artifacts" \ + --jq "[.artifacts[] | select(.name == \"$artifact_name\" and .expired == false)] | length")" + if test "$count" != 1; then + printf 'candidate_state=incomplete group=%s count=%s\n' "$group" "$count" + exit 0 + fi + done + printf 'run_id=%s\n' "$run_id" >> "$GITHUB_OUTPUT" + + validate_candidate: + name: Validate retained candidate bytes + needs: [resolve_source, locate_candidate] + if: needs.locate_candidate.outputs.candidate_run_id != '' + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + actions: read + contents: read + steps: + - uses: actions/checkout@v7 + with: + ref: ${{ needs.resolve_source.outputs.release_sha }} + - name: Download candidate manifest + uses: actions/download-artifact@v8 + with: + name: M1-Release-Candidate-manifest-${{ needs.resolve_source.outputs.release_sha }} + path: candidate + run-id: ${{ needs.locate_candidate.outputs.candidate_run_id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + - name: Download Python partition + uses: actions/download-artifact@v8 + with: + name: M1-Release-Candidate-python-${{ needs.resolve_source.outputs.release_sha }} + path: candidate/release-artifacts/python + run-id: ${{ needs.locate_candidate.outputs.candidate_run_id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + - name: Download npm partition + uses: actions/download-artifact@v8 + with: + name: M1-Release-Candidate-npm-${{ needs.resolve_source.outputs.release_sha }} + path: candidate/release-artifacts/npm + run-id: ${{ needs.locate_candidate.outputs.candidate_run_id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + - name: Download crates partition + uses: actions/download-artifact@v8 + with: + name: M1-Release-Candidate-crates-${{ needs.resolve_source.outputs.release_sha }} + path: candidate/release-artifacts/crates + run-id: ${{ needs.locate_candidate.outputs.candidate_run_id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + - name: Download evidence partition + uses: actions/download-artifact@v8 + with: + name: M1-Release-Candidate-evidence-${{ needs.resolve_source.outputs.release_sha }} + path: candidate/release-artifacts + run-id: ${{ needs.locate_candidate.outputs.candidate_run_id }} + github-token: ${{ github.token }} + repository: ${{ github.repository }} + - name: Reassemble and revalidate exact candidate + shell: bash + env: + RELEASE_SHA: ${{ needs.resolve_source.outputs.release_sha }} + RELEASE_VERSION: ${{ needs.resolve_source.outputs.release_version }} + run: | + python3 scripts/ci/release-candidate.py validate \ + --record "candidate/v${RELEASE_VERSION}-artifacts.json" \ + --artifacts-dir candidate/release-artifacts \ + --expected-sha "$RELEASE_SHA" \ + --version "$RELEASE_VERSION" + + dispatch_binding_rc: + name: Dispatch Binding RC when needed + needs: [resolve_source, locate_candidate, validate_candidate] + if: >- + always() && + needs.locate_candidate.result == 'success' && + needs.validate_candidate.result != 'success' && + !(github.event_name == 'workflow_dispatch' && inputs.create_release) + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + actions: write + contents: read + steps: + - name: Dispatch exact-main Binding RC + env: + GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ needs.resolve_source.outputs.release_sha }} + run: | + gh workflow run binding-release-candidate.yml \ + --ref main \ + -f "commit_sha=$RELEASE_SHA" + + create_release: + name: Create identity and trigger publish + needs: [resolve_source, validate_candidate] + if: >- + github.event_name == 'workflow_dispatch' && + inputs.create_release && + inputs.confirm_registry_publish && + needs.validate_candidate.result == 'success' + runs-on: blacksmith-4vcpu-ubuntu-2404 + permissions: + actions: read + contents: write + steps: + - uses: actions/checkout@v7 + with: + ref: main + fetch-depth: 0 + - name: Create only the exact immutable Release identity + shell: bash + env: + GH_TOKEN: ${{ github.token }} + RELEASE_SHA: ${{ needs.resolve_source.outputs.release_sha }} + RELEASE_VERSION: ${{ needs.resolve_source.outputs.release_version }} + RELEASE_TAG: ${{ inputs.release_tag }} + run: | + test "$RELEASE_TAG" = "v$RELEASE_VERSION" + git fetch --tags --force origin + if git rev-parse -q --verify "refs/tags/$RELEASE_TAG^{}" >/dev/null; then + test "$(git rev-parse "$RELEASE_TAG^{}")" = "$RELEASE_SHA" + fi + if gh release view "$RELEASE_TAG" >/dev/null 2>&1; then + echo "release identity already exists; use publish.yaml recovery dispatch" >&2 + exit 1 + fi + gh release create "$RELEASE_TAG" \ + --target "$RELEASE_SHA" \ + --generate-notes \ + --title "GraphForge $RELEASE_TAG" + printf '%s\n' \ + "Published release identity triggers publish.yaml; its retained-byte preflight remains authoritative." \ + >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f192f3f1e..cd8784994 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -137,6 +137,7 @@ jobs: python3 scripts/ci/test-prepare-napi-packages.py python3 scripts/ci/test-publish-npm-artifacts.py python3 scripts/ci/test-amend-npm-main-artifact.py + python3 scripts/ci/test-publish-track.py - name: Test CI storage policy run: python3 scripts/ci/test-ci-storage-policy.py diff --git a/docs/development/publication-order.md b/docs/development/publication-order.md index 31e96dd76..d64a2bddf 100644 --- a/docs/development/publication-order.md +++ b/docs/development/publication-order.md @@ -40,9 +40,38 @@ Before a maintainer authorizes publication (publish-track or human close): 5. PyPI and crates.io trusted publishing are configured for `CurateLabs/graphforge` and `publish.yaml`; the npm token has scoped public-package write access and Bypass 2FA. -6. The maintainer (or publish-track automation) creates the immutable tag and - GitHub Release identity that `publish.yaml` consumes. Implementation CI and - ordinary issue close do not run Binding RC or the human-close cascade. +6. The maintainer explicitly enables the immutable tag and GitHub Release + identity. Implementation CI and ordinary issue close do not run Binding RC + or the human-close cascade. + +## Publish-track orchestration + +`.github/workflows/publish-track.yml` runs on a six-hour schedule and by +maintainer dispatch. It resolves the current `main` SHA (or rejects a supplied +SHA that is not current `main`), then looks for one successful Binding RC run +with exactly one unexpired retained artifact for each required group. It +downloads and validates the whole candidate before treating it as reusable. +Therefore an unexpired candidate for the same SHA skips another RC; a missing, +expired, duplicate, or invalid candidate cannot reach tag or registry-write +steps. + +For ordinary scheduled runs, the workflow only dispatches the existing +`binding-release-candidate.yml` for an exact current `main` SHA when no valid +candidate is retained. It neither creates a tag nor writes a registry. + +To publish a validated candidate, a maintainer dispatches `Publish Track` with: + +1. `create_release: true`; +2. `confirm_registry_publish: true`; and +3. `release_tag` exactly equal to `v`. + +Both booleans are intentional release controls: creating the published GitHub +Release emits the existing release event that starts `publish.yaml`. The +orchestrator refuses an existing Release identity and directs recovery to the +explicit `publish.yaml` recovery dispatch instead. It verifies an existing tag +resolves to the candidate SHA and never moves it. `publish.yaml` remains the +only registry writer and independently revalidates the retained manifest, +partitions, rehearsal, live registry truth, and conflict evidence. ## Planner-driven execution @@ -149,11 +178,12 @@ yank/deprecate mechanism and prepare one coordinated later GraphForge version. ## Remaining human decisions -Automation stops before these actions unless separately authorized: +Publish-track automates candidate dispatch and makes the release/publish +boundary explicit. Maintainers still make these decisions: - choose the final v0.5.1 commit after normal exact-head PR CI; -- create the immutable v0.5.1 tag and GitHub Release; -- allow the release-triggered publication workflow to make registry writes; +- explicitly enable both publish-track release controls for the immutable + v0.5.1 tag and GitHub Release; - decide any registry-specific yank/deprecation if reconciliation finds a conflict; and - close the human release tracker only after the 24-node summary is complete. diff --git a/docs/engineering/PUBLISHING.md b/docs/engineering/PUBLISHING.md index 6a8e91f54..c7b201a1e 100644 --- a/docs/engineering/PUBLISHING.md +++ b/docs/engineering/PUBLISHING.md @@ -20,6 +20,16 @@ Wall-clock targets: Binding RC ≤20m p50 warm / ≤35m cold; publish-track (M1 load, checkpoint, m20/m21 surface aggregates) on top of publish-track honesty. Those gates must not block every publish-track run. +`Publish Track` schedules exact-`main` Binding RC dispatch every six hours and +reuses a complete, unexpired 30-day candidate for the same SHA. It does not +create a tag or publish by schedule. A maintainer must dispatch it with both +`create_release` and `confirm_registry_publish`, plus the exact +`v` tag; that published GitHub Release triggers the existing +`publish.yaml`. Any missing/expired partition, manifest validation failure, +mixed SHA, existing release identity, or registry conflict stops the path. +See [`publication-order.md`](../development/publication-order.md) for the +operational sequence and recovery boundary. + ## Artifacts and destinations | Artifact | Destination | Versioned by | Owner | diff --git a/scripts/ci/test-ci-storage-policy.py b/scripts/ci/test-ci-storage-policy.py index cc6847154..655bddfc2 100644 --- a/scripts/ci/test-ci-storage-policy.py +++ b/scripts/ci/test-ci-storage-policy.py @@ -68,6 +68,11 @@ "M1-Release-Candidate-npm-${{ steps.source.outputs.release_sha }}": 1, "M1-Release-Candidate-crates-${{ steps.source.outputs.release_sha }}": 1, "M1-Release-Candidate-evidence-${{ steps.source.outputs.release_sha }}": 1, + "M1-Release-Candidate-manifest-${{ needs.resolve_source.outputs.release_sha }}": 1, + "M1-Release-Candidate-python-${{ needs.resolve_source.outputs.release_sha }}": 1, + "M1-Release-Candidate-npm-${{ needs.resolve_source.outputs.release_sha }}": 1, + "M1-Release-Candidate-crates-${{ needs.resolve_source.outputs.release_sha }}": 1, + "M1-Release-Candidate-evidence-${{ needs.resolve_source.outputs.release_sha }}": 1, } ) EXPECTED_DEPENDENCY_KEYS = Counter( @@ -258,16 +263,31 @@ def artifact_contracts(text: str) -> tuple[list[str], list[str]]: assert field(step, "repository") == "${{ github.repository }}", ( f"cross-run artifact repository drift: {name}" ) - expected_run_id = { - "non-cypher-evidence": "${{ inputs.rust_run_id }}", - "binding-rc-aggregate": "${{ inputs.binding_rc_run_id }}", - "candidate": "${{ steps.candidate.outputs.run_id }}", - "candidate/release-artifacts/python": ("${{ steps.candidate.outputs.run_id }}"), - "candidate/release-artifacts/npm": "${{ steps.candidate.outputs.run_id }}", - "candidate/release-artifacts/crates": ("${{ steps.candidate.outputs.run_id }}"), - "candidate/release-artifacts": "${{ steps.candidate.outputs.run_id }}", + expected_run_ids = { + "non-cypher-evidence": {"${{ inputs.rust_run_id }}"}, + "binding-rc-aggregate": {"${{ inputs.binding_rc_run_id }}"}, + "candidate": { + "${{ steps.candidate.outputs.run_id }}", + "${{ needs.locate_candidate.outputs.candidate_run_id }}", + }, + "candidate/release-artifacts/python": { + "${{ steps.candidate.outputs.run_id }}", + "${{ needs.locate_candidate.outputs.candidate_run_id }}", + }, + "candidate/release-artifacts/npm": { + "${{ steps.candidate.outputs.run_id }}", + "${{ needs.locate_candidate.outputs.candidate_run_id }}", + }, + "candidate/release-artifacts/crates": { + "${{ steps.candidate.outputs.run_id }}", + "${{ needs.locate_candidate.outputs.candidate_run_id }}", + }, + "candidate/release-artifacts": { + "${{ steps.candidate.outputs.run_id }}", + "${{ needs.locate_candidate.outputs.candidate_run_id }}", + }, }[path] - assert field(step, "run-id") == expected_run_id, ( + assert field(step, "run-id") in expected_run_ids, ( f"cross-run artifact run ID drift: {name}" ) else: diff --git a/scripts/ci/test-publish-track.py b/scripts/ci/test-publish-track.py new file mode 100644 index 000000000..993406aab --- /dev/null +++ b/scripts/ci/test-publish-track.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Deterministic contract for publish-track orchestration and safety gates.""" + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +WORKFLOW = ROOT / ".github" / "workflows" / "publish-track.yml" +PUBLISH = ROOT / ".github" / "workflows" / "publish.yaml" + + +def section(text: str, start: str, end: str) -> str: + _, found, remainder = text.partition(start) + assert found, f"missing workflow marker: {start}" + result, found, _ = remainder.partition(end) + assert found, f"missing workflow marker: {end}" + return result + + +workflow = WORKFLOW.read_text(encoding="utf-8") +publish = PUBLISH.read_text(encoding="utf-8") + +assert "name: Publish Track" in workflow +assert 'cron: "17 */6 * * *"' in workflow +assert "workflow_dispatch:" in workflow +assert "create_release:" in workflow +assert "confirm_registry_publish:" in workflow +assert "release_tag:" in workflow +assert "binding-release-candidate.yml" in workflow +assert '-f "commit_sha=$RELEASE_SHA"' in workflow +assert "actions: write" in workflow + +resolve = section(workflow, " resolve_source:\n", " locate_candidate:\n") +assert "refs/heads/main:refs/remotes/origin/main" in resolve +assert 'test "$REQUESTED_SHA" = "$main_sha"' in resolve +assert "scripts/set_release_version.py --check" in resolve +assert "publish-track requires a non-development version" in resolve + +locate = section(workflow, " locate_candidate:\n", " validate_candidate:\n") +assert "status=success&head_sha=$RELEASE_SHA" in locate +assert 'artifact_name="M1-Release-Candidate-$group-$RELEASE_SHA"' in locate +assert ".expired == false" in locate +assert 'if test "$count" != 1; then' in locate +assert "candidate_state=incomplete" in locate + +validate = section(workflow, " validate_candidate:\n", " dispatch_binding_rc:\n") +assert "actions/download-artifact@v8" in validate +for group in ("manifest", "python", "npm", "crates", "evidence"): + assert ( + f"M1-Release-Candidate-{group}-${{{{ needs.resolve_source.outputs.release_sha }}}}" + in validate + ) +assert "scripts/ci/release-candidate.py validate" in validate +assert '--expected-sha "$RELEASE_SHA"' in validate +assert '--version "$RELEASE_VERSION"' in validate + +dispatch = section(workflow, " dispatch_binding_rc:\n", " create_release:\n") +assert "always()" in dispatch +assert "needs.locate_candidate.result == 'success'" in dispatch +assert "needs.validate_candidate.result != 'success'" in dispatch +assert "inputs.create_release" in dispatch +assert "gh workflow run binding-release-candidate.yml" in dispatch + +release = workflow.split(" create_release:\n", 1)[1] +assert "inputs.create_release" in release +assert "inputs.confirm_registry_publish" in release +assert "needs.validate_candidate.result == 'success'" in release +assert 'test "$RELEASE_TAG" = "v$RELEASE_VERSION"' in release +assert 'test "$(git rev-parse "$RELEASE_TAG^{}")" = "$RELEASE_SHA"' in release +assert 'gh release view "$RELEASE_TAG"' in release +assert 'gh release create "$RELEASE_TAG"' in release +assert '--target "$RELEASE_SHA"' in release +assert "publish.yaml" in release + +for forbidden in ( + "m1-release-certification", + "checkpoint-recovery", + "m20-contract", + "m21-contract", + "uv publish", + "npm publish", + "cargo publish", + "sleep", + "retry", + "continue-on-error", + "|| true", +): + assert forbidden not in workflow.lower(), forbidden + +# publish.yaml remains the only registry writer and revalidates retained bytes. +assert "uv publish candidate/release-artifacts/python/*" in publish +assert "scripts/ci/release-candidate.py validate" in publish +assert "PyO3/maturin-action" not in publish +assert "napi build" not in publish + +print("publish-track workflow tests passed")