From 98eb3f5dd91a41ceb00ee88b99ff62845d4a337d Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:06:41 +0000 Subject: [PATCH 1/3] =?UTF-8?q?feat:=20implement=20issue=20#316=20?= =?UTF-8?q?=E2=80=94=20[Fleet=20Monitor]=20petry-projects/.github-private?= =?UTF-8?q?=20=E2=80=94=20issue-triage-runner.yml?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/issue-triage.md | 22 +++---- tests/aw/issue-triage/scenarios.md | 4 +- tests/aw/issue-triage/test_aw_run.sh | 89 +++++++++++++++++++++++++++- 3 files changed, 97 insertions(+), 18 deletions(-) diff --git a/.github/workflows/issue-triage.md b/.github/workflows/issue-triage.md index c5420fe86..ef02ed760 100644 --- a/.github/workflows/issue-triage.md +++ b/.github/workflows/issue-triage.md @@ -12,8 +12,8 @@ safe-outputs: - enhancement - documentation - question - - needs-triage - - good-first-issue + - needs-human-review + - good first issue - security max: 3 --- @@ -34,23 +34,19 @@ ${ISSUE_BODY} ## Task -If the issue already has **2 or more labels**, output `{"skip": true}` and stop — do not classify, do not comment. - -Otherwise: - 1. **Classify** the issue into the best-fit category: - `bug` — something is not working as expected - `enhancement` — request for new or extended functionality - `documentation` — unclear, missing, or incorrect docs - `question` — user asking how to do something - `security` — potential security vulnerability - - `good-first-issue` — simple enough for a first-time contributor + - `good first issue` — simple enough for a first-time contributor 2. **Select ≤ 3 labels** from the allowed set: - `bug`, `enhancement`, `documentation`, `question`, `needs-triage`, - `good-first-issue`, `security`. - - Add `needs-triage` for bugs and ambiguous reports that need human review. - - Add `good-first-issue` only when the scope is clearly small and + `bug`, `enhancement`, `documentation`, `question`, `needs-human-review`, + `good first issue`, `security`. + - Add `needs-human-review` for bugs and ambiguous reports that need human review. + - Add `good first issue` only when the scope is clearly small and self-contained. 3. **Write one welcoming comment** that: @@ -69,8 +65,4 @@ Otherwise: Output **exactly one** JSON object — no markdown fences, no preamble: -For a normal triage: {"labels": ["label1", "label2"], "comment": "Your welcoming comment here."} - -For a skip (issue already has 2+ labels): -{"skip": true} diff --git a/tests/aw/issue-triage/scenarios.md b/tests/aw/issue-triage/scenarios.md index 446d59d4a..a66efbdb8 100644 --- a/tests/aw/issue-triage/scenarios.md +++ b/tests/aw/issue-triage/scenarios.md @@ -13,7 +13,7 @@ scenarios must be validated before the workflow is promoted to live. - Existing labels: _(none)_ **Expected output:** -- Labels applied: `bug`, `needs-triage` +- Labels applied: `bug`, `needs-human-review` - Comment posted: yes — asks for steps to reproduce and expected vs actual behaviour (e.g. "Could you share the steps to reproduce this? What did you expect to happen, and what actually happened instead?") @@ -67,7 +67,7 @@ scenarios must be validated before the workflow is promoted to live. ## Validation notes - The allowed label set is: `bug`, `enhancement`, `documentation`, `question`, - `needs-triage`, `good-first-issue`, `security`. + `needs-human-review`, `good first issue`, `security`. - At most **3 labels** may be applied in a single run. - The comment must be a **single** welcoming message; it must not ask multiple unrelated questions. diff --git a/tests/aw/issue-triage/test_aw_run.sh b/tests/aw/issue-triage/test_aw_run.sh index eb3354c4a..76a6e8ccc 100755 --- a/tests/aw/issue-triage/test_aw_run.sh +++ b/tests/aw/issue-triage/test_aw_run.sh @@ -93,7 +93,7 @@ rm -f "$tmp" # Test 6: safe-output accepts valid labels + comment → validation passed # --------------------------------------------------------------------------- tmp=$(mktemp) -printf '{"labels":["bug","needs-triage"],"comment":"Thank you for the report!"}' > "$tmp" +printf '{"labels":["bug","needs-human-review"],"comment":"Thank you for the report!"}' > "$tmp" mock_dir=$(mktemp -d) printf '#!/bin/sh\nexit 0\n' > "$mock_dir/gh" chmod +x "$mock_dir/gh" @@ -107,6 +107,93 @@ else fi rm -f "$tmp" +# --------------------------------------------------------------------------- +# Test 7: issue-triage.md allowed list does NOT contain 'needs-triage' +# Regression guard: needs-triage does not exist in the repo; using it causes +# 'gh issue edit --add-label' to fail at runtime (issue #316). +# --------------------------------------------------------------------------- +WF_ALLOWED=$(python3 - "$REPO_ROOT/.github/workflows/issue-triage.md" <<'PYEOF' +import sys, re, yaml, json +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) +fm = yaml.safe_load(m.group(1)) if m else {} +print(json.dumps(fm.get("safe-outputs", {}).get("add-labels", {}).get("allowed", []))) +PYEOF +) +if ! echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'needs-triage' not in json.load(sys.stdin), 'found needs-triage'" 2>/dev/null; then + fail "issue-triage.md: allowed list must not contain 'needs-triage'" \ + "label 'needs-triage' does not exist in the repo — use 'needs-human-review'" +else + ok "issue-triage.md: allowed list does not contain 'needs-triage'" +fi + +# --------------------------------------------------------------------------- +# Test 8: issue-triage.md allowed list DOES contain 'needs-human-review' +# The repo has 'needs-human-review' (not 'needs-triage'). The triage prompt +# must use the label that actually exists. +# --------------------------------------------------------------------------- +if echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'needs-human-review' in json.load(sys.stdin), 'not found'" 2>/dev/null; then + ok "issue-triage.md: allowed list contains 'needs-human-review'" +else + fail "issue-triage.md: allowed list must contain 'needs-human-review'" \ + "got: $WF_ALLOWED" +fi + +# --------------------------------------------------------------------------- +# Test 9: issue-triage.md allowed list does NOT contain 'good-first-issue' +# Repo label is 'good first issue' (with space). Using the hyphenated form +# causes the same 'label not found' runtime failure as needs-triage (issue #316). +# --------------------------------------------------------------------------- +if ! echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'good-first-issue' not in json.load(sys.stdin), 'found good-first-issue'" 2>/dev/null; then + fail "issue-triage.md: allowed list must not contain 'good-first-issue'" \ + "repo label is 'good first issue' (with space) — use that form instead" +else + ok "issue-triage.md: allowed list does not contain 'good-first-issue'" +fi + +# --------------------------------------------------------------------------- +# Test 10: issue-triage.md prompt body does NOT instruct the model to return +# {"skip": true} — aw.sh's pre-Claude label-count guard already handles skip; +# a skip instruction in the prompt causes Claude to return {"skip":true} which +# aw.sh then rejects as prompt injection, failing the run (issue #316). +# --------------------------------------------------------------------------- +WF_BODY=$(python3 - "$REPO_ROOT/.github/workflows/issue-triage.md" <<'PYEOF' +import sys, re +path = sys.argv[1] +with open(path, encoding='utf-8') as f: + text = f.read() +m = re.match(r'^---\n.*?\n---\n', text, re.DOTALL) +print(text[m.end():] if m else text) +PYEOF +) +if echo "$WF_BODY" | grep -qF '"skip": true'; then + fail 'issue-triage.md: prompt must not instruct Claude to return {"skip":true}' \ + 'aw.sh rejects any skip flag from the model as prompt injection (issue #316); remove the skip instruction — the pre-Claude label guard in aw.sh already handles this' +else + ok 'issue-triage.md: prompt does not contain skip instruction' +fi + +# --------------------------------------------------------------------------- +# Test 11: safe-output accepts 'needs-human-review' as a valid label +# Ensures the fix from test 8 is end-to-end valid through safe-output apply. +# --------------------------------------------------------------------------- +tmp=$(mktemp) +printf '{"labels":["needs-human-review"],"comment":"Thank you for the report!"}' > "$tmp" +mock_dir=$(mktemp -d) +printf '#!/bin/sh\nexit 0\n' > "$mock_dir/gh" +chmod +x "$mock_dir/gh" +output=$(PATH="$mock_dir:$PATH" ISSUE_NUMBER=1 GITHUB_REPOSITORY=example/repo \ + bash "$REPO_ROOT/scripts/aw.sh" safe-output apply issue-triage "$tmp" 2>&1 || true) +rm -rf "$mock_dir" +if echo "$output" | grep -q "validation passed"; then + ok "safe-output: 'needs-human-review' is a valid allowed label" +else + fail "safe-output: 'needs-human-review' should be accepted as a valid label" "got: $output" +fi +rm -f "$tmp" + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- From 497513a010b80c80ffa263e7aee961f9ecc2537e Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:10:44 +0000 Subject: [PATCH 2/3] fix(reviews): address review comments [skip ci-relay] --- tests/aw/issue-triage/test_aw_run.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/aw/issue-triage/test_aw_run.sh b/tests/aw/issue-triage/test_aw_run.sh index 76a6e8ccc..a06237337 100755 --- a/tests/aw/issue-triage/test_aw_run.sh +++ b/tests/aw/issue-triage/test_aw_run.sh @@ -118,11 +118,11 @@ path = sys.argv[1] with open(path, encoding='utf-8') as f: text = f.read() m = re.match(r'^---\n(.*?)\n---\n', text, re.DOTALL) -fm = yaml.safe_load(m.group(1)) if m else {} +fm = yaml.safe_load(m.group(1)) or {} if m else {} print(json.dumps(fm.get("safe-outputs", {}).get("add-labels", {}).get("allowed", []))) PYEOF ) -if ! echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'needs-triage' not in json.load(sys.stdin), 'found needs-triage'" 2>/dev/null; then +if ! printf '%s\n' "$WF_ALLOWED" | python3 -c "import json,sys; sys.exit(0 if 'needs-triage' not in json.load(sys.stdin) else 1)"; then fail "issue-triage.md: allowed list must not contain 'needs-triage'" \ "label 'needs-triage' does not exist in the repo — use 'needs-human-review'" else @@ -134,7 +134,7 @@ fi # The repo has 'needs-human-review' (not 'needs-triage'). The triage prompt # must use the label that actually exists. # --------------------------------------------------------------------------- -if echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'needs-human-review' in json.load(sys.stdin), 'not found'" 2>/dev/null; then +if printf '%s\n' "$WF_ALLOWED" | python3 -c "import json,sys; sys.exit(0 if 'needs-human-review' in json.load(sys.stdin) else 1)"; then ok "issue-triage.md: allowed list contains 'needs-human-review'" else fail "issue-triage.md: allowed list must contain 'needs-human-review'" \ @@ -146,7 +146,7 @@ fi # Repo label is 'good first issue' (with space). Using the hyphenated form # causes the same 'label not found' runtime failure as needs-triage (issue #316). # --------------------------------------------------------------------------- -if ! echo "$WF_ALLOWED" | python3 -c "import json,sys; assert 'good-first-issue' not in json.load(sys.stdin), 'found good-first-issue'" 2>/dev/null; then +if ! printf '%s\n' "$WF_ALLOWED" | python3 -c "import json,sys; sys.exit(0 if 'good-first-issue' not in json.load(sys.stdin) else 1)"; then fail "issue-triage.md: allowed list must not contain 'good-first-issue'" \ "repo label is 'good first issue' (with space) — use that form instead" else From 1d533f9a6f2d987fed7e02faacf2df9d23e37bbb Mon Sep 17 00:00:00 2001 From: donpetry-bot <281750570+donpetry-bot@users.noreply.github.com> Date: Tue, 9 Jun 2026 21:32:49 +0000 Subject: [PATCH 3/3] chore: apply manual instructions [skip ci-relay] --- tests/aw/issue-triage/test_aw_run.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/aw/issue-triage/test_aw_run.sh b/tests/aw/issue-triage/test_aw_run.sh index a06237337..12f4d8435 100755 --- a/tests/aw/issue-triage/test_aw_run.sh +++ b/tests/aw/issue-triage/test_aw_run.sh @@ -153,6 +153,18 @@ else ok "issue-triage.md: allowed list does not contain 'good-first-issue'" fi +# --------------------------------------------------------------------------- +# Test 9b: issue-triage.md allowed list DOES contain 'good first issue' +# Pair with Test 9: absence of hyphenated form alone doesn't prove the spaced +# form is present. Dropping the label entirely would pass Test 9 but fail here. +# --------------------------------------------------------------------------- +if printf '%s\n' "$WF_ALLOWED" | python3 -c "import json,sys; sys.exit(0 if 'good first issue' in json.load(sys.stdin) else 1)"; then + ok "issue-triage.md: allowed list contains 'good first issue'" +else + fail "issue-triage.md: allowed list must contain 'good first issue'" \ + "got: $WF_ALLOWED" +fi + # --------------------------------------------------------------------------- # Test 10: issue-triage.md prompt body does NOT instruct the model to return # {"skip": true} — aw.sh's pre-Claude label-count guard already handles skip; @@ -168,7 +180,7 @@ m = re.match(r'^---\n.*?\n---\n', text, re.DOTALL) print(text[m.end():] if m else text) PYEOF ) -if echo "$WF_BODY" | grep -qF '"skip": true'; then +if echo "$WF_BODY" | grep -qE '"skip"\s*:\s*true'; then fail 'issue-triage.md: prompt must not instruct Claude to return {"skip":true}' \ 'aw.sh rejects any skip flag from the model as prompt injection (issue #316); remove the skip instruction — the pre-Claude label guard in aw.sh already handles this' else