feat: implement issue #1019 — [dev-lead timeout C/3] Fleet-monitor observability: surface timeout count/rate

scripts/fleet_monitor.sh

@@ -257,6 +257,34 @@ if ! gh api "search/issues?q=org:${ORG}+label:fleet-tracker+is:open+is:issue&per
   : > "$issues_lookup_file"  # ensure file exists but is empty on failure
 fi
 
+# ---------------------------------------------------------------------------
+# 2c. Dev-Lead timeout observability (#1019, story C of #901)
+# Count dev-lead status markers by failure reason over the window, per repo, so a
+# rising timeout ("wall") rate is visible distinctly from generic engine-error.
+# Markers are HTML comments (`<!-- dev-lead-issue ... reason=timeout ... -->`,
+# Story B / #1018) that GitHub search cannot index, so — like auto_rebase_health.sh
+# — we pull each repo's issue comments server-side filtered by `since=`. Only
+# repos with >=1 marker in the window get a row; the section is omitted entirely
+# when the fleet had no dev-lead activity. Best-effort: a comment-read failure on
+# one repo is a per-repo warning, not a fatal error for the monitor.
+# ---------------------------------------------------------------------------
+dev_lead_reason_file=$(mktemp)
+for repo in "${repos[@]}"; do
+  if ! comments_raw=$(gh api \
+    "repos/${repo}/issues/comments?since=${CUTOFF}&per_page=100" --paginate \
+    --jq '.[] | {body}' 2>/dev/null); then
+    echo "::warning::Cannot read issue comments for ${repo} — dev-lead timeout counts may undercount"
+    continue
+  fi
+  comments_json=$(printf '%s' "$comments_raw" | jq -s '.' 2>/dev/null || echo '[]')
+  IFS=$'\t' read -r dl_timeout dl_engine_error dl_total \
+    < <(summarize_dev_lead_timeouts "$comments_json")
+  if [ "${dl_total:-0}" -gt 0 ]; then
+    printf '%s\t%s\t%s\t%s\n' "$repo" "$dl_timeout" "$dl_engine_error" "$dl_total" \
+      >> "$dev_lead_reason_file"
+  fi
+done
+
 # ---------------------------------------------------------------------------
 # 3. Generate reports
 # ---------------------------------------------------------------------------
@@ -278,16 +306,24 @@ stub_drift_section() {
   done
 }
 
+# dev_lead_timeout_section — appends the Dev-Lead timeout ("wall") observability
+# block (#1019) when any repo had dev-lead status markers in the window.
+dev_lead_timeout_section() {
+  [ -s "$dev_lead_reason_file" ] || return 0
+  printf '\n'
+  generate_dev_lead_timeout_report "$dev_lead_reason_file"
+}
+
 # Step Summary — Tier 1 visualizations only (Mermaid not rendered there)
 # GitHub Step Summary has a 1 MB hard limit per job. At ~200 bytes per row
 # this supports ~5 000 workflows before truncation.
 if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
-  { report_header; generate_report "$metrics_file" "$failed_file" "false" "$issues_lookup_file"; stub_drift_section; } \
+  { report_header; generate_report "$metrics_file" "$failed_file" "false" "$issues_lookup_file"; dev_lead_timeout_section; stub_drift_section; } \
     >> "$GITHUB_STEP_SUMMARY"
 fi
 
 # Report file — full report with Mermaid charts (used as Issue body)
-{ report_header; generate_report "$metrics_file" "$failed_file" "true" "$issues_lookup_file"; stub_drift_section; } \
+{ report_header; generate_report "$metrics_file" "$failed_file" "true" "$issues_lookup_file"; dev_lead_timeout_section; stub_drift_section; } \
   > "$REPORT_FILE"
 
 # ---------------------------------------------------------------------------
@@ -336,7 +372,7 @@ if [ -n "${GITHUB_ENV:-}" ]; then
   [ "$stub_drift_count" -gt 0 ] && echo "HAS_STUB_DRIFT=true" >> "$GITHUB_ENV"
 fi
 
-rm -f "$metrics_file" "$failed_file" "$issues_lookup_file"
+rm -f "$metrics_file" "$failed_file" "$issues_lookup_file" "$dev_lead_reason_file"
 [ ${#stub_drift_files[@]} -gt 0 ] && rm -f "${stub_drift_files[@]}"
 
 # ---------------------------------------------------------------------------

scripts/fleet_report.sh

@@ -15,6 +15,16 @@
 # gate list for testing; add permanent gates to the default below.
 FLEET_GATE_WORKFLOWS="${FLEET_GATE_WORKFLOWS:-test-deletion-guard.yml holdout-guard.yml}"
 
+# Dev-Lead status-marker prefix (#1019, story C of #901). Mirrors
+# ISSUE_MARKER_PREFIX in dev-lead-fix-issue.sh / dev-lead-retry.sh: every
+# dev-lead failure/retry/escalation posts an HTML-comment marker of the form
+#   <!-- dev-lead-issue <N> status=<s> attempt=<k> reason=<r> run=<id> [reset=...] -->
+# GitHub search cannot index HTML comments, so callers pass the repo's windowed
+# issue comments (issues/comments?since=) and summarize_dev_lead_timeouts tallies
+# the markers by their reason= field. Kept in one place so a rename in the
+# dev-lead scripts is a one-line fix here.
+DEV_LEAD_ISSUE_MARKER="${DEV_LEAD_ISSUE_MARKER:-<!-- dev-lead-issue }"
+
 # label_to_icon <label>
 # Returns the health icon matching the scorecard legend.
 label_to_icon() {
@@ -115,6 +125,65 @@ filter_high_failure() {
   ' < "$f"
 }
 
+# summarize_dev_lead_timeouts <comments_json>
+# Reads a JSON array of issue-comment objects ({body,...}) and tallies dev-lead
+# status markers by failure reason. A stage timeout ("wall") is reason=timeout
+# (Story B, #1018); generic engine failures are reason=engine-error. Emits TSV:
+#   timeout <TAB> engine_error <TAB> total
+# `total` is every dev-lead-issue marker in the window (all reasons; there is no
+# success marker, so total == dev-lead failure/retry/escalation markers).
+# Comments that merely mention `reason=` without the marker prefix are ignored.
+# Absent/empty JSON → "0\t0\t0".
+summarize_dev_lead_timeouts() {
+  local json="${1:-}"
+  [ -n "$json" ] || json='[]'
+  printf '%s' "$json" | jq -r --arg p "$DEV_LEAD_ISSUE_MARKER" '
+    [ .[]? | (.body // "") | select(contains($p))
+      | (try (capture("reason=(?<r>[a-z]+(?:-[a-z]+)*)").r) catch "unknown") ] as $reasons
+    | [ ([ $reasons[] | select(. == "timeout") ]      | length),
+        ([ $reasons[] | select(. == "engine-error") ] | length),
+        ($reasons | length) ]
+    | @tsv'
+}
+
+# generate_dev_lead_timeout_report <reason_tsv_file>
+# Renders the Dev-Lead timeout ("wall") observability section (#1019, story C of
+# #901). Input is a TSV file with one row per repo that had >=1 dev-lead marker
+# in the window: `repo <TAB> timeout <TAB> engine_error <TAB> total`. Prints a
+# per-repo table, fleet totals, and the timeout rate (timeouts / all dev-lead
+# failure markers) — the "wall rate" Story A/B aim to drive down. An empty or
+# missing file prints nothing, so the section is omitted when there was no
+# dev-lead activity in the window.
+generate_dev_lead_timeout_report() {
+  local f="${1:-}"
+  [ -n "$f" ] && [ -s "$f" ] || return 0
+
+  local repo t e tot
+  local sum_t=0 sum_e=0 sum_tot=0
+
+  printf '## Dev-Lead Timeouts (walls)\n\n'
+  printf 'Stage timeouts (`reason=timeout`) that escalate to a human, broken out from generic `engine-error`, over the window.\n\n'
+  printf '| Repo | Timeouts | Engine-errors | Total dev-lead failures |\n'
+  printf '|---|---|---|---|\n'
+  while IFS=$'\t' read -r repo t e tot; do
+    [ -n "$repo" ] || continue
+    printf '| `%s` | %s | %s | %s |\n' "$repo" "$t" "$e" "$tot"
+    sum_t=$(( sum_t + ${t:-0} ))
+    sum_e=$(( sum_e + ${e:-0} ))
+    sum_tot=$(( sum_tot + ${tot:-0} ))
+  done < "$f"
+  printf '| **Fleet total** | **%s** | **%s** | **%s** |\n\n' "$sum_t" "$sum_e" "$sum_tot"
+
+  local rate
+  rate=$(awk -v t="$sum_t" -v n="$sum_tot" 'BEGIN {
+    if (n <= 0) { print "n/a"; exit }
+    pct = t * 100 / n
+    printf (pct == int(pct)) ? "%d%%" : "%.1f%%", pct
+  }')
+  printf '**Timeout rate** (`reason=timeout` / all dev-lead failure markers): %s (%s / %s)\n' \
+    "$rate" "$sum_t" "$sum_tot"
+}
+
 # detect_systemic_failures <metrics_file>
 # Prints the name of each workflow file that has failures in 3 or more repos.
 detect_systemic_failures() {

tests/fleet_report.bats

@@ -289,7 +289,7 @@ setup() {
 # Writes a metrics TSV to a temp file and echoes its path.
 _mk_metrics() {
   local f
-  f="$(mktemp)" || { echo "Failed to create temp file" >&2; exit 1; }
+  f="$(mktemp "$BATS_TEST_TMPDIR/fr.XXXXXX")" || { echo "Failed to create temp file" >&2; exit 1; }
   printf '%s\n' "$@" > "$f"
   echo "$f"
 }
@@ -358,3 +358,92 @@ _mk_metrics() {
   [ "$(echo "$output" | jq -r '.[0].workflow')" = "ci.yml" ]
   rm -f "$m"
 }
+
+# ---------------------------------------------------------------------------
+# summarize_dev_lead_timeouts — dev-lead timeout observability (#1019, story C)
+# Tallies dev-lead status markers by failure reason. Emits TSV:
+#   timeout <TAB> engine_error <TAB> total
+# ---------------------------------------------------------------------------
+
+@test "dev-lead timeouts: counts timeout, engine-error, and total across mixed markers" {
+  local json='[
+    {"body":"<!-- dev-lead-issue 5 status=needs-human attempt=1 reason=timeout run=1 -->\nx"},
+    {"body":"<!-- dev-lead-issue 6 status=failed attempt=1 reason=engine-error run=2 -->\ny"},
+    {"body":"<!-- dev-lead-issue 9 status=needs-human attempt=3 reason=timeout run=3 -->"}
+  ]'
+  run summarize_dev_lead_timeouts "$json"
+  [ "$status" -eq 0 ]
+  [ "$output" = $'2\t1\t3' ]
+}
+
+@test "dev-lead timeouts: ignores comments that merely mention reason= but lack the marker" {
+  local json='[
+    {"body":"<!-- dev-lead-issue 5 status=needs-human attempt=1 reason=timeout run=1 -->"},
+    {"body":"a normal comment mentioning reason=timeout but not a marker"}
+  ]'
+  run summarize_dev_lead_timeouts "$json"
+  [ "$output" = $'1\t0\t1' ]
+}
+
+@test "dev-lead timeouts: empty array yields zero counts" {
+  run summarize_dev_lead_timeouts '[]'
+  [ "$output" = $'0\t0\t0' ]
+}
+
+@test "dev-lead timeouts: absent argument yields zero counts" {
+  run summarize_dev_lead_timeouts
+  [ "$output" = $'0\t0\t0' ]
+}
+
+@test "dev-lead timeouts: rate-limited counts toward total but not timeout/engine-error" {
+  local json='[
+    {"body":"<!-- dev-lead-issue 7 status=rate-limited attempt=1 reason=rate-limited run=1 reset=2026-01-01T00:00:00Z -->"},
+    {"body":"<!-- dev-lead-issue 8 status=needs-human attempt=1 reason=timeout run=2 -->"}
+  ]'
+  run summarize_dev_lead_timeouts "$json"
+  [ "$output" = $'1\t0\t2' ]
+}
+
+# ---------------------------------------------------------------------------
+# generate_dev_lead_timeout_report — renders the per-repo section (#1019)
+# Input: TSV file rows `repo <TAB> timeout <TAB> engine_error <TAB> total`.
+# ---------------------------------------------------------------------------
+
+@test "dev-lead timeout report: renders per-repo rows, fleet total, and timeout rate" {
+  local f
+  f="$(mktemp "$BATS_TEST_TMPDIR/fr.XXXXXX")"
+  printf '%s\n' \
+    $'petry-projects/a\t2\t1\t4' \
+    $'petry-projects/b\t1\t1\t3' \
+    > "$f"
+  run generate_dev_lead_timeout_report "$f"
+  [ "$status" -eq 0 ]
+  [[ "$output" =~ "petry-projects/a" ]]
+  [[ "$output" =~ "petry-projects/b" ]]
+  # Fleet totals: timeout 3, engine-error 2, total 7
+  [[ "$output" =~ "Fleet total" ]]
+  [[ "$output" =~ "reason=timeout" ]]
+  # Rate = 3 / 7 = 42.9%
+  [[ "$output" =~ "42.9%" ]]
+  rm -f "$f"
+}
+
+@test "dev-lead timeout report: whole-number rate renders without a decimal" {
+  local f
+  f="$(mktemp "$BATS_TEST_TMPDIR/fr.XXXXXX")"
+  printf '%s\n' $'petry-projects/a\t1\t1\t2' > "$f"
+  run generate_dev_lead_timeout_report "$f"
+  [[ "$output" =~ "50%" ]]
+  [[ ! "$output" =~ "50.0%" ]]
+  rm -f "$f"
+}
+
+@test "dev-lead timeout report: empty file produces no output" {
+  local f
+  f="$(mktemp "$BATS_TEST_TMPDIR/fr.XXXXXX")"
+  : > "$f"
+  run generate_dev_lead_timeout_report "$f"
+  [ "$status" -eq 0 ]
+  [ -z "$output" ]
+  rm -f "$f"
+}