feat(gastown): implement town reconciler with event-driven state management

Replace the imperative patrol/scheduling/review-queue alarm phases with a
declarative reconciler that drains events, computes desired state, and
applies corrective actions. This is the complete implementation of the
reconciliation spec (Phases 1-5).

Core changes:
- Event table (town_events) with 8 event types, dual-write in all RPC handlers
- 6 reconciler functions: agents, beads, review queue, convoys, GUPP, GC
- applyEvent/applyAction for all event and action types
- Event-only agentDone/agentCompleted (no direct mutations)
- Lazy assignment for slingConvoy/startConvoy (#1249)
- Container status observation pre-phase (replaces witnessPatrol)
- Agent activity watermark via enriched heartbeat
- Invariant checker + reconciler metrics in getAlarmStatus
- Rework flow: gt_request_changes tool for refinery change requests
- Refinery system prompt wired in (buildRefinerySystemPrompt)

Bug fixes discovered during rollout:
- Convoy landing MR cycling (reconciler created duplicates)
- Multi-agent hook mutual exclusion (hookBead now unhooks stale agents)
- agentCompleted no longer closes beads when gt_done wasn't called
- GUPP NaN bug for agents with null last_activity_at
- Container status event flood (upsert instead of insert per tick)
- Rule 4 scoped to in_progress MR beads only

Dead code removed: ~1,578 lines from patrol.ts, scheduling.ts,
review-queue.ts, and Town.do.ts (witnessPatrol, deaconPatrol,
processReviewQueue, processConvoyLandings, pollPendingPRs, etc.)

Author: jrf0110

cloudflare-gastown/container/plugin/client.ts

@@ -116,6 +116,16 @@ export class GastownClient {
     });
   }
 
+  async requestChanges(input: {
+    feedback: string;
+    files?: string[];
+  }): Promise<{ rework_bead_id: string }> {
+    return this.request<{ rework_bead_id: string }>(this.agentPath('/request-changes'), {
+      method: 'POST',
+      body: JSON.stringify(input),
+    });
+  }
+
   async checkMail(): Promise<Mail[]> {
     return this.request<Mail[]>(this.agentPath('/mail'));
   }

cloudflare-gastown/container/plugin/tools.ts

@@ -78,6 +78,38 @@ export function createTools(client: GastownClient) {
       },
     }),
 
+    gt_request_changes: tool({
+      description:
+        'Request changes on the code you are reviewing. This creates a rework task ' +
+        'for a polecat to address your feedback. After calling this, call gt_done to ' +
+        'release your session. The polecat will push fixes to the same branch, and ' +
+        'you will be re-dispatched to re-review once the rework is complete. ' +
+        'Only available to refinery agents.',
+      args: {
+        feedback: tool.schema
+          .string()
+          .describe(
+            'Detailed description of what needs to change. Be specific: ' +
+              'reference file names, function names, and the exact issues found.'
+          ),
+        files: tool.schema
+          .array(tool.schema.string())
+          .describe('Optional list of specific file paths that need changes')
+          .optional(),
+      },
+      async execute(args) {
+        const result = await client.requestChanges({
+          feedback: args.feedback,
+          files: args.files,
+        });
+        return (
+          `Rework request created (bead ${result.rework_bead_id}). ` +
+          'A polecat will be assigned to address your feedback. ' +
+          'Call gt_done now to release your session. You will be re-dispatched to re-review once the rework is complete.'
+        );
+      },
+    }),
+
     gt_mail_send: tool({
       description:
         'Send a typed message to another agent in the rig. ' +

cloudflare-gastown/container/src/heartbeat.ts

@@ -53,6 +53,10 @@ async function sendHeartbeats(): Promise<void> {
       townId: agent.townId,
       status: agent.status,
       timestamp: new Date().toISOString(),
+      lastEventType: agent.lastEventType ?? null,
+      lastEventAt: agent.lastEventAt ?? null,
+      activeTools: agent.activeTools ?? [],
+      messageCount: agent.messageCount ?? 0,
     };
 
     try {

cloudflare-gastown/container/src/process-manager.ts

@@ -449,6 +449,8 @@ async function subscribeToEvents(
       if (sessionID && sessionID !== agent.sessionId) continue;
 
       agent.lastActivityAt = new Date().toISOString();
+      agent.lastEventType = event.type ?? 'unknown';
+      agent.lastEventAt = new Date().toISOString();
 
       // Track active tool calls
       if (event.properties && 'activeTools' in event.properties) {
@@ -534,6 +536,8 @@ export async function startAgent(
     workdir,
     startedAt: now,
     lastActivityAt: now,
+    lastEventType: null,
+    lastEventAt: null,
     activeTools: [],
     messageCount: 0,
     exitReason: null,

cloudflare-gastown/container/src/types.ts

@@ -103,6 +103,10 @@ export type ManagedAgent = {
   workdir: string;
   startedAt: string;
   lastActivityAt: string;
+  /** Event type of the most recent SDK event (e.g. 'message_part.updated') */
+  lastEventType: string | null;
+  /** ISO 8601 timestamp of the most recent SDK event */
+  lastEventAt: string | null;
   /** Last known active tool calls (populated from SSE events) */
   activeTools: string[];
   /** Total messages sent to this agent */
@@ -299,6 +303,11 @@ export type HeartbeatPayload = {
   townId: string;
   status: AgentStatus;
   timestamp: string;
+  // SDK activity watermark
+  lastEventType: string | null;
+  lastEventAt: string | null;
+  activeTools: string[];
+  messageCount: number;
 };
 
 // ── Stream ticket (for WebSocket streaming) ─────────────────────────────

cloudflare-gastown/docs/post-deploy-monitoring.md

@@ -4,22 +4,46 @@ Guide for an AI agent to verify town health after a production deploy.
 
 ## Prerequisites
 
-- The debug endpoint is deployed: `GET /debug/towns/:townId/status` (unauthenticated)
+- The debug endpoint is deployed: `GET /debug/towns/:townId/status`
+- The debug endpoint is protected by Cloudflare Access. Requests must include service token headers.
 - Base URL: `https://gastown.kiloapps.io`
 - Town ID: obtain from `GET /trpc/gastown.listOrgTowns` (requires auth) or from the user
 
+### Authentication
+
+The debug endpoint requires Cloudflare Access service token headers. These are the same credentials the Next.js app uses to communicate with gastown:
+
+```bash
+# Set these from your Cloudflare Access service token
+export CF_ACCESS_CLIENT_ID="<service-token-client-id>"
+export CF_ACCESS_CLIENT_SECRET="<service-token-client-secret>"
+```
+
+All `curl` commands in this document use a helper function that includes these headers:
+
+```bash
+debug_curl() {
+  curl -s \
+    -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \
+    -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \
+    "$@"
+}
+```
+
 ## 1. Monitor Script
 
 The monitoring script at `scripts/monitor-town.sh` polls the debug endpoint:
 
 ```bash
+export CF_ACCESS_CLIENT_ID="<client-id>"
+export CF_ACCESS_CLIENT_SECRET="<client-secret>"
 ./scripts/monitor-town.sh <townId> [interval_seconds]
 ```
 
 Or poll manually:
 
 ```bash
-curl -s "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status" | python3 -c "
+debug_curl "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status" | python3 -c "
 import sys, json
 d = json.load(sys.stdin)
 a = d['alarmStatus']
@@ -29,6 +53,9 @@ ref = [x for x in d['agentMeta'] if x.get('role') == 'refinery']
 if ref:
     r = ref[0]
     print(f\"Refinery: status={r['status']} hook={r.get('current_hook_bead_id') or 'NULL'}\")
+recon = a.get('reconciler')
+if recon:
+    print(f\"Reconciler: events={recon['eventsDrained']} actions={recon['actionsEmitted']} violations={recon['invariantViolations']} wallMs={recon['wallClockMs']}\")
 for e in a.get('recentEvents', [])[:5]:
     print(f\"  {e['time'][:19]}  {e['message'][:80]}\")
 "
@@ -44,7 +71,7 @@ The Durable Object reinitializes. Check the alarm is running:
 
 ```bash
 # Alarm should show 'active (5s)' within 10s of deploy
-curl -s ".../debug/towns/$TOWN_ID/status" | python3 -c "
+debug_curl "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status" | python3 -c "
 import sys, json; d = json.load(sys.stdin)
 print(d['alarmStatus']['alarm']['intervalLabel'])
 "
@@ -67,23 +94,24 @@ Poll until agents start working:
 
 ```bash
 # Should see Working > 0 within 2-3 min
-curl -s ".../debug/towns/$TOWN_ID/status" | python3 -c "
+debug_curl "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status" | python3 -c "
 import sys, json; d = json.load(sys.stdin)
 print(f\"Working: {d['alarmStatus']['agents']['working']}\")
 "
 ```
 
 ### Phase 3: Agent Recovery (1-5min)
 
-Verify agents recover from the container restart:
+Verify agents recover from the container restart. The reconciler handles all recovery:
 
-- **Polecats**: idle+hooked agents should be dispatched by `schedulePendingWork`
-- **Refinery**: if it was mid-review, `recoverStuckReviews` handles it after the timeout
-- **Orphaned beads**: `rehookOrphanedBeads` picks up beads with stale assignees after 2 min
+- **Polecats**: idle+hooked agents are re-dispatched by `reconcileBeads` Rule 2
+- **Refinery**: if it was mid-review, the container status observation detects the dead container and sets the refinery to idle. `reconcileReviewQueue` Rule 6 re-dispatches it.
+- **Orphaned beads**: `reconcileBeads` Rule 3 resets in-progress beads with no working agent to open after 5 min, then Rule 1 assigns a new agent.
 
 **Red flags**:
+
 - `Working: 0` for more than 5 min after container is active
-- `orphanedHooks` growing without dispatches
+- `invariantViolations > 0` in reconciler metrics
 - `failed` count increasing rapidly (dispatch attempts burning out)
 
 ### Phase 4: Review Pipeline (5-15min)
@@ -95,10 +123,34 @@ in_progress → in_review → review_completed → closed
 ```
 
 Check that:
+
 - The refinery picks up MR beads (status transitions to `working`)
 - Reviews complete as `merged` (not `Refinery container failed to start`)
 - Source beads reach `closed` and stay closed
 
+### Phase 5: Reconciler Health
+
+Verify the reconciler is running correctly:
+
+```bash
+debug_curl "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status" | python3 -c "
+import sys, json; d = json.load(sys.stdin)
+r = d['alarmStatus'].get('reconciler')
+if r:
+    print(f\"Events drained: {r['eventsDrained']}\")
+    print(f\"Actions emitted: {r['actionsEmitted']}\")
+    print(f\"Invariant violations: {r['invariantViolations']}\")
+    print(f\"Wall clock: {r['wallClockMs']}ms\")
+    print(f\"Pending events: {r['pendingEventCount']}\")
+    if r.get('actionsByType'):
+        print(f\"Action types: {r['actionsByType']}\")
+else:
+    print('No reconciler metrics yet')
+"
+```
+
+**Expected**: `invariantViolations: 0`, `wallClockMs < 100`, `pendingEventCount: 0`
+
 ## 3. Test Convoy
 
 Create a simple test convoy to verify the full pipeline. Use the tRPC `slingConvoy` endpoint:
@@ -127,7 +179,7 @@ Then monitor until both beads reach `closed`:
 ```bash
 # Poll every 30s until no non-terminal issue beads remain
 while true; do
-  RESP=$(curl -s ".../debug/towns/$TOWN_ID/status")
+  RESP=$(debug_curl "https://gastown.kiloapps.io/debug/towns/$TOWN_ID/status")
   ISSUES=$(echo "$RESP" | python3 -c "
 import sys, json
 beads = json.load(sys.stdin).get('beadSummary', [])
@@ -144,22 +196,24 @@ done
 ```
 
 **Expected timeline**:
-- 0-2 min: beads created, polecats dispatched
+
+- 0-2 min: beads created, polecats dispatched by reconciler (lazy assignment)
 - 2-10 min: polecats work, submit reviews
 - 10-15 min: refinery reviews and merges
 - 15-25 min: second bead goes through the same cycle
 - 25-30 min: convoy lands (all beads closed)
 
 **Failure indicators**:
-- Beads stuck in `open` for >5 min → check `schedulePendingWork` / agent hooks
+
+- Beads stuck in `open` for >5 min → check reconciler actions (should emit `dispatch_agent`)
 - Beads stuck in `in_review` for >15 min → check refinery status and MR beads
 - MR beads stuck in `in_progress` for >5 min → check refinery dispatch retry
-- Beads cycling `in_progress → open` → check `dispatchAgent` failures
+- Beads cycling `in_progress → open` → check `agentCompleted` events and STALE_IN_PROGRESS_TIMEOUT_MS
 - Reviews completing as `failed` → check container start errors on refinery agent status message
 
 ## 4. Cleanup
 
-After monitoring is complete, clean up test beads and the debug endpoint.
+After monitoring is complete, clean up test beads.
 
 ### Remove test convoy beads
 
@@ -186,23 +240,15 @@ for b in data:
 done
 ```
 
-### Remove debug endpoint
-
-The `/debug/towns/:townId/status` endpoint is unauthenticated and must be removed before merging to main. Remove these from the codebase:
-
-1. `src/gastown.worker.ts` — the `app.get('/debug/towns/:townId/status', ...)` route
-2. `src/dos/Town.do.ts` — the `debugAgentMetadata()` and `debugBeadSummary()` methods
-3. `src/trpc/router.ts` — the `debugAgentMetadata` tRPC procedure
-
-Search for `// DEBUG` comments to find all debug code to remove.
-
 ## 5. Key Metrics to Watch
 
-| Metric | Healthy | Unhealthy |
-|--------|---------|-----------|
-| Working agents | >0 when beads exist | 0 for >5 min with open beads |
-| Failed bead count | Stable | Increasing rapidly |
-| Orphaned hooks | 0 | >0 for >5 min |
-| Refinery status | `working` during review, `idle` between | `idle` with in_progress MR for >5 min |
-| Review outcomes | `merged` | `Refinery container failed to start` |
-| Alarm interval | `active (5s)` with work | Stuck at same `nextFireAt` |
+| Metric                | Healthy                                 | Unhealthy                             |
+| --------------------- | --------------------------------------- | ------------------------------------- |
+| Working agents        | >0 when beads exist                     | 0 for >5 min with open beads          |
+| Failed bead count     | Stable                                  | Increasing rapidly                    |
+| Invariant violations  | 0                                       | >0 (check reconciler logs)            |
+| Refinery status       | `working` during review, `idle` between | `idle` with in_progress MR for >5 min |
+| Review outcomes       | `merged`                                | `Refinery container failed to start`  |
+| Alarm interval        | `active (5s)` with work                 | Stuck at same `nextFireAt`            |
+| Reconciler wall clock | <100ms                                  | >500ms consistently                   |
+| Pending event count   | 0 between ticks                         | Growing (events not draining)         |

cloudflare-gastown/scripts/monitor-town.sh

@@ -1,19 +1,32 @@
 #!/bin/bash
 # Continuously monitor a town's state via the debug endpoint.
 # Usage: ./scripts/monitor-town.sh [townId] [interval_seconds]
+#
+# Requires Cloudflare Access service token credentials:
+#   export CF_ACCESS_CLIENT_ID="<service-token-client-id>"
+#   export CF_ACCESS_CLIENT_SECRET="<service-token-client-secret>"
 
 TOWN_ID="${1:-8a6f9375-b806-4ee0-ad6e-1697ea2dbfff}"
 INTERVAL="${2:-15}"
 BASE_URL="${GASTOWN_URL:-https://gastown.kiloapps.io}"
 URL="${BASE_URL}/debug/towns/${TOWN_ID}/status"
 
+if [ -z "$CF_ACCESS_CLIENT_ID" ] || [ -z "$CF_ACCESS_CLIENT_SECRET" ]; then
+  echo "Error: CF_ACCESS_CLIENT_ID and CF_ACCESS_CLIENT_SECRET must be set"
+  echo "These are the Cloudflare Access service token credentials."
+  exit 1
+fi
+
 echo "Monitoring town ${TOWN_ID} every ${INTERVAL}s"
 echo "Endpoint: ${URL}"
 echo "Press Ctrl+C to stop"
 echo "=========================================="
 
 while true; do
-  RESP=$(curl -s --max-time 10 "${URL}" 2>/dev/null)
+  RESP=$(curl -s --max-time 10 \
+    -H "CF-Access-Client-Id: $CF_ACCESS_CLIENT_ID" \
+    -H "CF-Access-Client-Secret: $CF_ACCESS_CLIENT_SECRET" \
+    "${URL}" 2>/dev/null)
   if [ -z "$RESP" ]; then
     echo "$(date -u +%H:%M:%S)  [ERROR] No response from ${URL}"
     sleep "$INTERVAL"
@@ -34,6 +47,7 @@ alarm = d.get('alarmStatus', {})
 agents_info = alarm.get('agents', {})
 beads_info = alarm.get('beads', {})
 patrol_info = alarm.get('patrol', {})
+recon = alarm.get('reconciler') or {}
 events = alarm.get('recentEvents', [])
 
 working = agents_info.get('working', 0)
@@ -42,7 +56,9 @@ op = beads_info.get('open', 0)
 ip = beads_info.get('inProgress', 0)
 ir = beads_info.get('inReview', 0)
 failed = beads_info.get('failed', 0)
-orphaned = patrol_info.get('orphanedHooks', 0)
+violations = recon.get('invariantViolations', '-')
+actions = recon.get('actionsEmitted', '-')
+wall_ms = recon.get('wallClockMs', '-')
 
 # Agent details
 agents = d.get('agentMeta', [])
@@ -52,7 +68,7 @@ refinery = [a for a in agents if a.get('role') == 'refinery']
 # Non-terminal beads
 beads = d.get('beadSummary', [])
 
-print(f'{ts}  W={working} I={idle} | open={op} prog={ip} review={ir} fail={failed} | hooks={orphaned} hooked={len(hooked_agents)}')
+print(f'{ts}  W={working} I={idle} | open={op} prog={ip} review={ir} fail={failed} | v={violations} act={actions} ms={wall_ms}')
 
 # Show refinery state
 for r in refinery: