From fcbd1c6af45fafb9aaa3094e962d1955fc8a459a Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:12:20 +0000 Subject: [PATCH 1/7] =?UTF-8?q?feat:=20implement=20issue=20#1995=20?= =?UTF-8?q?=E2=80=94=20info=5Fstatus=5Fpattern=20covers=20only=20SonarClou?= =?UTF-8?q?d=20=E2=80=94=20Codex/Qodo/CodeRabbit=20usage-limit=20notices?= =?UTF-8?q?=20are=20still=20undispositionable=20and=20still=20strand=20PRs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/lib/reviewer-sources.tsv | 14 ++++- .../unit/test_maintainer_comment_gate.bats | 59 +++++++++++++++++- tests/test_reviewer_sources.bats | 61 +++++++++++++++++-- 3 files changed, 123 insertions(+), 11 deletions(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index 70b16f33f..a7a31915b 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -45,15 +45,23 @@ # It must NOT match the bot's finding-bearing comments (SonarCloud's # "Quality Gate failed" must still block) — the gate fails closed on # anything this pattern does not match. Data, not hard-coded logins. +# FOR NO-FINDING SERVICE NOTICES ONLY (#1995): declare one only for a +# usage-limit / trial-ended / "review limit reached" / clean-status +# message that carries no finding. Anchor it tightly — several required +# literals in sequence, like the sonarqubecloud row — so it cannot match +# a review body that merely mentions a limit. Never reach for this column +# to wave away an inconvenient review: a finding-producing reviewer +# (codeant-ai, graphite-app) carries "-" and its findings go through +# dev-lead's verified-fix flow. # rationale — why this bot is classified as it is. # # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes yes - - Inline review comments; dev-lead addresses the findings. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. -chatgpt-codex-connector yes yes yes - - Inline review comments (pull_request_review_comment); dev-lead addresses the findings. -coderabbitai yes yes no - - Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. +chatgpt-codex-connector yes yes yes - You have reached your Codex usage limits for code reviews[\s\S]*Codex usage dashboard Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limits for code reviews … Codex usage dashboard" notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +coderabbitai yes yes no - Review limit reached[\s\S]*credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached … credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. -qodo-code-review yes yes yes - - Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). +qodo-code-review yes yes yes - Qodo reviews are paused[\s\S]*trial has ended Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its "Qodo reviews are paused … trial has ended" notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. codeant-ai yes yes yes - - Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). graphite-app yes yes yes Graphite / AI Reviews - Inline review comments AND clean passes reported ONLY as the "Graphite / AI Reviews" check run (no PR review) — the scorecard must read that check run or every clean pass is miscounted as No response (#1908). Root cause of the 2026-08-02 unclearable-thread deadlock — trusted so dev-lead can act on its findings (#1425). cubic-dev-ai yes yes yes - - Inline review comments plus a review summary; added 2026-09-22 on a 7-day trial (#1903). Flip advisory_gate to no (or remove this row) if the trial is not converted to paid (~2026-09-29). diff --git a/tests/dev-lead/unit/test_maintainer_comment_gate.bats b/tests/dev-lead/unit/test_maintainer_comment_gate.bats index 8fdda46d6..cad6720c9 100644 --- a/tests/dev-lead/unit/test_maintainer_comment_gate.bats +++ b/tests/dev-lead/unit/test_maintainer_comment_gate.bats @@ -93,9 +93,12 @@ _run_check() { [ "$status" -eq 1 ] } -# AC9(b): an undispositioned qodo-code-review comment blocks. -@test "AC9b: undispositioned qodo-code-review comment → 1 (block)" { - local json='{"comments":[{"author":{"login":"qodo-code-review"},"body":" Qodo reviews are paused because your trial has ended.","createdAt":"2026-09-13T04:23:06Z","isMinimized":false,"minimizedReason":""}]}' +# AC9(b): an undispositioned qodo-code-review FINDING blocks. (Its trial-ended +# service notice is now auto-cleared via an info_status_pattern, #1995 — covered +# by the #1995 tests below — so this case uses a genuine finding, which carries no +# info-status pattern and must still block.) +@test "AC9b: undispositioned qodo-code-review finding → 1 (block)" { + local json='{"comments":[{"author":{"login":"qodo-code-review"},"body":"PR Review: `parseConfig` does not validate the timeout bound before use.","createdAt":"2026-09-13T04:23:06Z","isMinimized":false,"minimizedReason":""}]}' _run_check "$json" [ "$status" -eq 1 ] } @@ -288,6 +291,56 @@ _sonar_json() { [ "$status" -eq 0 ] } +# ──────────────────────────────────────────────────────────────────── +# #1995 — the info-status classifier generalized beyond SonarCloud: Codex, +# CodeRabbit and Qodo post service notices carrying NO finding (usage-limit / +# review-limit / trial-ended). Each now declares an info_status_pattern, so a +# matching notice is auto-cleared (→0) with no dev-lead and no human, while a +# real review body from the SAME bot carries no pattern match and still blocks +# (→1). The notice bodies are this repo's own history (#1902/#1887/#1873). +# ──────────────────────────────────────────────────────────────────── + +_notice_json() { + # _notice_json — a one-comment, un-minimized gate snapshot. + jq -cn --arg l "$1" --arg b "$2" \ + '{reviews:[], comments:[{author:{login:$l}, body:$b, isMinimized:false, minimizedReason:""}]}' +} + +@test "AC1(#1995): Codex usage-limit notice clears → 0" { + _run_check "$(_notice_json chatgpt-codex-connector "You have reached your Codex usage limits for code reviews. You can see your limits in the Codex usage dashboard.")" + [ "$status" -eq 0 ] +} + +@test "AC1(#1995): Codex usage-limit notice with a [bot] suffix login clears → 0" { + _run_check "$(_notice_json 'chatgpt-codex-connector[bot]' "You have reached your Codex usage limits for code reviews. You can see your limits in the Codex usage dashboard.")" + [ "$status" -eq 0 ] +} + +@test "AC2(#1995): a real Codex review body still blocks → 1" { + _run_check "$(_notice_json chatgpt-codex-connector "The cubic free trial ended handling is too broad — this code path should be narrower.")" + [ "$status" -eq 1 ] +} + +@test "AC1(#1995): CodeRabbit review-limit notice clears → 0" { + _run_check "$(_notice_json coderabbitai "Review limit reached — you have used up your prepaid credits.")" + [ "$status" -eq 0 ] +} + +@test "AC2(#1995): a real CodeRabbit review body still blocks → 1" { + _run_check "$(_notice_json coderabbitai "Consider guarding against a nil pointer before dereferencing \`cfg\` here.")" + [ "$status" -eq 1 ] +} + +@test "AC1(#1995): Qodo trial-ended notice clears → 0" { + _run_check "$(_notice_json qodo-code-review "Qodo reviews are paused because your trial has ended.")" + [ "$status" -eq 0 ] +} + +@test "AC2(#1995): a real Qodo review body still blocks → 1" { + _run_check "$(_notice_json qodo-code-review "Suggestion: extract this block into a helper to reduce duplication.")" + [ "$status" -eq 1 ] +} + # ──────────────────────────────────────────────────────────────────── # INTEGRATION / WIRING TESTS (review-one-pr.sh) # ──────────────────────────────────────────────────────────────────── diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index b91d62448..b53d0650a 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -157,17 +157,68 @@ TSV @test "info-status: a source without an info-status pattern is not listed" { local patterns patterns="$(reviewer_sources_info_status_patterns)" - # These seven sources carry "-" in the info_status_pattern column, so none may + # These sources carry "-" in the info_status_pattern column, so none may # appear — enumerate every one (mirroring the check-run negative test) so giving # any unlisted source a non-"-" pattern must fail this test rather than pass - # silently and weaken the gate's fail-closed guarantee. + # silently and weaken the gate's fail-closed guarantee. codeant-ai and + # graphite-app are finding-producing reviewers and must stay out (#1995). [[ "$patterns" != *"copilot-pull-request-reviewer"* ]] [[ "$patterns" != *"gemini-code-assist"* ]] - [[ "$patterns" != *"chatgpt-codex-connector"* ]] - [[ "$patterns" != *"coderabbitai"* ]] - [[ "$patterns" != *"qodo-code-review"* ]] [[ "$patterns" != *"codeant-ai"* ]] [[ "$patterns" != *"graphite-app"* ]] + [[ "$patterns" != *"cubic-dev-ai"* ]] +} + +# ── Generalized info-status patterns for service notices (issue #1995) ──────── +# +# #1918 added info_status_pattern for sonarqubecloud only. Every other reviewer +# that posts a service notice carrying NO finding (a usage-limit / trial-ended / +# "review limit reached" message) was still undispositionable and stranded the +# approval gate. #1995 declares tightly-anchored patterns for those notices. +# +# Each pattern must (AC #3) MATCH the real notice body and NOT match a real review +# body from the same bot. _info_pattern_for reads the stored pattern; the gate +# applies it via jq test() (case-sensitive), so these tests do the same. + +_info_pattern_for() { + reviewer_sources_info_status_patterns | awk -F'\t' -v l="$1" '$1==l{print $2}' +} + +# _body_matches_pattern — 0 iff matches under +# the exact jq test() semantics the maintainer-comment gate uses. +_body_matches_pattern() { + jq -ne --arg p "$1" --arg b "$2" '$b | test($p)' >/dev/null +} + +@test "info-status: chatgpt-codex-connector matches its Codex usage-limit notice, not a review (#1995)" { + local pat + pat="$(_info_pattern_for chatgpt-codex-connector)" + [ -n "$pat" ] + run _body_matches_pattern "$pat" "You have reached your Codex usage limits for code reviews. You can see your limits in the Codex usage dashboard." + [ "$status" -eq 0 ] + # A real Codex review body that merely discusses a limit must NOT be swallowed. + run _body_matches_pattern "$pat" "The cubic free trial ended handling is too broad — this code path should be narrower." + [ "$status" -ne 0 ] +} + +@test "info-status: coderabbitai matches its review-limit notice, not a review (#1995)" { + local pat + pat="$(_info_pattern_for coderabbitai)" + [ -n "$pat" ] + run _body_matches_pattern "$pat" "Review limit reached — you have used up your prepaid credits." + [ "$status" -eq 0 ] + run _body_matches_pattern "$pat" "Consider guarding against a nil pointer before dereferencing \`cfg\` here." + [ "$status" -ne 0 ] +} + +@test "info-status: qodo-code-review matches its trial-ended notice, not a review (#1995)" { + local pat + pat="$(_info_pattern_for qodo-code-review)" + [ -n "$pat" ] + run _body_matches_pattern "$pat" "Qodo reviews are paused because your trial has ended." + [ "$status" -eq 0 ] + run _body_matches_pattern "$pat" "Suggestion: extract this block into a helper to reduce duplication." + [ "$status" -ne 0 ] } @test "info-status: helper propagates a missing-manifest failure" { From 9b79eabff600a61727c5e4f50c72ef679f190639 Mon Sep 17 00:00:00 2001 From: don-petry <36422719+don-petry@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:24:16 +0000 Subject: [PATCH 2/7] fix(reviews): address review comments [skip ci-relay] --- scripts/lib/reviewer-sources.tsv | 6 +++--- .../dev-lead/unit/test_maintainer_comment_gate.bats | 5 +++++ tests/test_reviewer_sources.bats | 12 +++++++++--- 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index a7a31915b..75b83e61f 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -58,10 +58,10 @@ # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes yes - - Inline review comments; dev-lead addresses the findings. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. -chatgpt-codex-connector yes yes yes - You have reached your Codex usage limits for code reviews[\s\S]*Codex usage dashboard Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limits for code reviews … Codex usage dashboard" notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. -coderabbitai yes yes no - Review limit reached[\s\S]*credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached … credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +chatgpt-codex-connector yes yes yes - You have reached your Codex usage limit Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +coderabbitai yes yes no - Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. -qodo-code-review yes yes yes - Qodo reviews are paused[\s\S]*trial has ended Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its "Qodo reviews are paused … trial has ended" notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +qodo-code-review yes yes yes - Qodo reviews are paused because your trial has ended|Qodo Merge has reached your monthly usage limit for pull-request reviews Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. codeant-ai yes yes yes - - Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). graphite-app yes yes yes Graphite / AI Reviews - Inline review comments AND clean passes reported ONLY as the "Graphite / AI Reviews" check run (no PR review) — the scorecard must read that check run or every clean pass is miscounted as No response (#1908). Root cause of the 2026-08-02 unclearable-thread deadlock — trusted so dev-lead can act on its findings (#1425). cubic-dev-ai yes yes yes - - Inline review comments plus a review summary; added 2026-09-22 on a 7-day trial (#1903). Flip advisory_gate to no (or remove this row) if the trial is not converted to paid (~2026-09-29). diff --git a/tests/dev-lead/unit/test_maintainer_comment_gate.bats b/tests/dev-lead/unit/test_maintainer_comment_gate.bats index cad6720c9..b80c9d009 100644 --- a/tests/dev-lead/unit/test_maintainer_comment_gate.bats +++ b/tests/dev-lead/unit/test_maintainer_comment_gate.bats @@ -336,6 +336,11 @@ _notice_json() { [ "$status" -eq 0 ] } +@test "AC1(#1995): Qodo monthly-usage-limit notice clears → 0" { + _run_check "$(_notice_json qodo-code-review "Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository. Reviews will resume when the limit resets.")" + [ "$status" -eq 0 ] +} + @test "AC2(#1995): a real Qodo review body still blocks → 1" { _run_check "$(_notice_json qodo-code-review "Suggestion: extract this block into a helper to reduce duplication.")" [ "$status" -eq 1 ] diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index b53d0650a..8c3acc070 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -196,9 +196,12 @@ _body_matches_pattern() { [ -n "$pat" ] run _body_matches_pattern "$pat" "You have reached your Codex usage limits for code reviews. You can see your limits in the Codex usage dashboard." [ "$status" -eq 0 ] + # The shorter notice variant this repo has also observed must clear too (fixtures/reviewer_pr_node.json). + run _body_matches_pattern "$pat" "You have reached your Codex usage limit." + [ "$status" -eq 0 ] # A real Codex review body that merely discusses a limit must NOT be swallowed. run _body_matches_pattern "$pat" "The cubic free trial ended handling is too broad — this code path should be narrower." - [ "$status" -ne 0 ] + [ "$status" -eq 1 ] } @test "info-status: coderabbitai matches its review-limit notice, not a review (#1995)" { @@ -208,7 +211,7 @@ _body_matches_pattern() { run _body_matches_pattern "$pat" "Review limit reached — you have used up your prepaid credits." [ "$status" -eq 0 ] run _body_matches_pattern "$pat" "Consider guarding against a nil pointer before dereferencing \`cfg\` here." - [ "$status" -ne 0 ] + [ "$status" -eq 1 ] } @test "info-status: qodo-code-review matches its trial-ended notice, not a review (#1995)" { @@ -217,8 +220,11 @@ _body_matches_pattern() { [ -n "$pat" ] run _body_matches_pattern "$pat" "Qodo reviews are paused because your trial has ended." [ "$status" -eq 0 ] + # The monthly-usage-limit notice variant this repo has observed must clear too (fixtures/events/advisory_qodo_rate_limited.json). + run _body_matches_pattern "$pat" "Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository. Reviews will resume when the limit resets." + [ "$status" -eq 0 ] run _body_matches_pattern "$pat" "Suggestion: extract this block into a helper to reduce duplication." - [ "$status" -ne 0 ] + [ "$status" -eq 1 ] } @test "info-status: helper propagates a missing-manifest failure" { From 5861a57f83d4b7132c4635c697564b631a5fc45c Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 19:59:56 +0000 Subject: [PATCH 3/7] fix(reviewer-sources): anchor the Codex usage-limit pattern to the body start Reconciles this PR with #1998 (#1993), which pins the same Codex notice. The unanchored substring would clear any Codex comment that merely quotes the notice further down, including one carrying a real finding. Anchoring on ^ keeps both observed forms of the notice (short "usage limit." and long "usage limits for code reviews") while a quoted occurrence still blocks. Adds a negative test to the registry suite and to the gate suite; both fail without the anchor. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015fdDZDb4mhT65MU1izeA4R --- scripts/lib/reviewer-sources.tsv | 2 +- tests/dev-lead/unit/test_maintainer_comment_gate.bats | 9 +++++++++ tests/test_reviewer_sources.bats | 4 ++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index 75b83e61f..7263604db 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -58,7 +58,7 @@ # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes yes - - Inline review comments; dev-lead addresses the findings. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. -chatgpt-codex-connector yes yes yes - You have reached your Codex usage limit Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +chatgpt-codex-connector yes yes yes - ^You have reached your Codex usage limit Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start, so a longer comment that merely quotes the notice cannot clear; a real review body does not match and still blocks. coderabbitai yes yes no - Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. qodo-code-review yes yes yes - Qodo reviews are paused because your trial has ended|Qodo Merge has reached your monthly usage limit for pull-request reviews Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. diff --git a/tests/dev-lead/unit/test_maintainer_comment_gate.bats b/tests/dev-lead/unit/test_maintainer_comment_gate.bats index b80c9d009..b6f573d80 100644 --- a/tests/dev-lead/unit/test_maintainer_comment_gate.bats +++ b/tests/dev-lead/unit/test_maintainer_comment_gate.bats @@ -321,6 +321,15 @@ _notice_json() { [ "$status" -eq 1 ] } +@test "AC2(#1995): a Codex comment that only quotes the notice (not at body start) still blocks → 1" { + _run_check "$(_notice_json chatgpt-codex-connector "## Codex Review — could not complete + +Earlier this run reported: You have reached your Codex usage limits for code reviews. + +**P1** Possible null dereference in scripts/foo.sh:42 — guard the lookup before use.")" + [ "$status" -eq 1 ] +} + @test "AC1(#1995): CodeRabbit review-limit notice clears → 0" { _run_check "$(_notice_json coderabbitai "Review limit reached — you have used up your prepaid credits.")" [ "$status" -eq 0 ] diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index 8c3acc070..3bbbe324f 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -202,6 +202,10 @@ _body_matches_pattern() { # A real Codex review body that merely discusses a limit must NOT be swallowed. run _body_matches_pattern "$pat" "The cubic free trial ended handling is too broad — this code path should be narrower." [ "$status" -eq 1 ] + # Anchored to the body start (#1993): a comment that only QUOTES the notice later + # on must not clear, or a finding-bearing body could be swallowed. + run _body_matches_pattern "$pat" "Earlier this run reported: You have reached your Codex usage limits for code reviews." + [ "$status" -eq 1 ] } @test "info-status: coderabbitai matches its review-limit notice, not a review (#1995)" { From 84e7fc5a26a02e011b3fefab9f9af0b763a3a4b4 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:04:38 +0000 Subject: [PATCH 4/7] fix(reviewer-sources): end-anchor the Codex notice pattern; negatives reuse each pattern's literals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit cubic (PRRT_kwDOR9SdIs6oGf77): the negative cases shared no tokens with their patterns, so loosening an anchor would still pass. The Codex pattern also matched review prose that opens with the notice's words and then carries a finding. - Codex pattern is now ^You have reached your Codex usage limits?( for code reviews)?\. — it matches both observed notice forms and nothing that continues the sentence. - Each registry test gains a negative built from its own pattern's literals (Codex, CodeRabbit, Qodo). The new Codex negative matches under the previous pattern and fails the test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015fdDZDb4mhT65MU1izeA4R --- scripts/lib/reviewer-sources.tsv | 2 +- tests/test_reviewer_sources.bats | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index 7263604db..b67119f70 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -58,7 +58,7 @@ # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes yes - - Inline review comments; dev-lead addresses the findings. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. -chatgpt-codex-connector yes yes yes - ^You have reached your Codex usage limit Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start, so a longer comment that merely quotes the notice cannot clear; a real review body does not match and still blocks. +chatgpt-codex-connector yes yes yes - ^You have reached your Codex usage limits?( for code reviews)?\. Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start and to the end of the notice sentence, so neither a comment that merely quotes the notice nor one that opens with its words and goes on to a finding can clear; a real review body does not match and still blocks. coderabbitai yes yes no - Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. qodo-code-review yes yes yes - Qodo reviews are paused because your trial has ended|Qodo Merge has reached your monthly usage limit for pull-request reviews Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index 3bbbe324f..96ec64622 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -206,6 +206,10 @@ _body_matches_pattern() { # on must not clear, or a finding-bearing body could be swallowed. run _body_matches_pattern "$pat" "Earlier this run reported: You have reached your Codex usage limits for code reviews." [ "$status" -eq 1 ] + # Anchored to the end of the notice sentence too: review prose that OPENS with the + # notice's own words and then carries a finding must not clear. + run _body_matches_pattern "$pat" "You have reached your Codex usage limits for code reviews, but the dashboard numbers are stale — the cap math is the real bug." + [ "$status" -eq 1 ] } @test "info-status: coderabbitai matches its review-limit notice, not a review (#1995)" { @@ -216,6 +220,9 @@ _body_matches_pattern() { [ "$status" -eq 0 ] run _body_matches_pattern "$pat" "Consider guarding against a nil pointer before dereferencing \`cfg\` here." [ "$status" -eq 1 ] + # Review prose reusing the notice's leading literal must not clear. + run _body_matches_pattern "$pat" "Review limit reached handling is too broad — this code path should be narrower." + [ "$status" -eq 1 ] } @test "info-status: qodo-code-review matches its trial-ended notice, not a review (#1995)" { @@ -229,6 +236,9 @@ _body_matches_pattern() { [ "$status" -eq 0 ] run _body_matches_pattern "$pat" "Suggestion: extract this block into a helper to reduce duplication." [ "$status" -eq 1 ] + # Review prose reusing the notice's leading literal must not clear. + run _body_matches_pattern "$pat" "Qodo reviews are paused because the health check failed — lean on monitoring instead." + [ "$status" -eq 1 ] } @test "info-status: helper propagates a missing-manifest failure" { From 6f0578530c9191357f2f1c90b4691bd98b18bbe4 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:11:51 +0000 Subject: [PATCH 5/7] fix(reviewer-sources): start-anchor the CodeRabbit and Qodo notice patterns graphite (both rows): the patterns were unanchored substrings, so any review body that merely mentions the notice text mid-body was auto-cleared. - Both patterns are now anchored to the body start, allowing leading HTML marker comments (^\s*(\s*)*). Qodo prefixes its real trial notice with , which a bare ^ would reject. - The Qodo trial sentence is also end-anchored (\.), like the Codex one. - Registry and gate tests: graphite's mid-body examples now block, and the marker-prefixed Qodo notice still clears. The CodeRabbit negatives fail without the anchor. - The column docs now spell out the anchoring convention. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015fdDZDb4mhT65MU1izeA4R --- scripts/lib/reviewer-sources.tsv | 14 ++++++++------ .../unit/test_maintainer_comment_gate.bats | 15 +++++++++++++++ tests/test_reviewer_sources.bats | 14 ++++++++++++++ 3 files changed, 37 insertions(+), 6 deletions(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index b67119f70..814b8ebca 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -49,19 +49,21 @@ # usage-limit / trial-ended / "review limit reached" / clean-status # message that carries no finding. Anchor it tightly — several required # literals in sequence, like the sonarqubecloud row — so it cannot match -# a review body that merely mentions a limit. Never reach for this column -# to wave away an inconvenient review: a finding-producing reviewer -# (codeant-ai, graphite-app) carries "-" and its findings go through -# dev-lead's verified-fix flow. +# a review body that merely mentions a limit. Anchor service notices to +# the body start (^); allow leading HTML marker comments with +# ^\s*(\s*)* when the bot prefixes its notice with one. +# Never reach for this column to wave away an inconvenient review: a +# finding-producing reviewer (codeant-ai, graphite-app) carries "-" and +# its findings go through dev-lead's verified-fix flow. # rationale — why this bot is classified as it is. # # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes yes - - Inline review comments; dev-lead addresses the findings. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. chatgpt-codex-connector yes yes yes - ^You have reached your Codex usage limits?( for code reviews)?\. Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start and to the end of the notice sentence, so neither a comment that merely quotes the notice nor one that opens with its words and goes on to a finding can clear; a real review body does not match and still blocks. -coderabbitai yes yes no - Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +coderabbitai yes yes no - ^\s*(\s*)*Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. -qodo-code-review yes yes yes - Qodo reviews are paused because your trial has ended|Qodo Merge has reached your monthly usage limit for pull-request reviews Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. +qodo-code-review yes yes yes - ^\s*(\s*)*(Qodo reviews are paused because your trial has ended\.|Qodo Merge has reached your monthly usage limit for pull-request reviews) Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. codeant-ai yes yes yes - - Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). graphite-app yes yes yes Graphite / AI Reviews - Inline review comments AND clean passes reported ONLY as the "Graphite / AI Reviews" check run (no PR review) — the scorecard must read that check run or every clean pass is miscounted as No response (#1908). Root cause of the 2026-08-02 unclearable-thread deadlock — trusted so dev-lead can act on its findings (#1425). cubic-dev-ai yes yes yes - - Inline review comments plus a review summary; added 2026-09-22 on a 7-day trial (#1903). Flip advisory_gate to no (or remove this row) if the trial is not converted to paid (~2026-09-29). diff --git a/tests/dev-lead/unit/test_maintainer_comment_gate.bats b/tests/dev-lead/unit/test_maintainer_comment_gate.bats index b6f573d80..fb7273693 100644 --- a/tests/dev-lead/unit/test_maintainer_comment_gate.bats +++ b/tests/dev-lead/unit/test_maintainer_comment_gate.bats @@ -345,6 +345,21 @@ Earlier this run reported: You have reached your Codex usage limits for code rev [ "$status" -eq 0 ] } +@test "AC1(#1995): Qodo trial-ended notice with its HTML marker prefix clears → 0" { + _run_check "$(_notice_json qodo-code-review " Qodo reviews are paused because your trial has ended.")" + [ "$status" -eq 0 ] +} + +@test "AC2(#1995): a Qodo review that mentions the trial notice mid-body still blocks → 1" { + _run_check "$(_notice_json qodo-code-review "The handling of the case where Qodo reviews are paused because your trial has ended is too lenient.")" + [ "$status" -eq 1 ] +} + +@test "AC2(#1995): a CodeRabbit review that mentions the limit notice mid-body still blocks → 1" { + _run_check "$(_notice_json coderabbitai "The error handling is broken. Review limit reached — you have used up your prepaid credits feature needs better UX.")" + [ "$status" -eq 1 ] +} + @test "AC1(#1995): Qodo monthly-usage-limit notice clears → 0" { _run_check "$(_notice_json qodo-code-review "Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository. Reviews will resume when the limit resets.")" [ "$status" -eq 0 ] diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index 96ec64622..e0f6e3f78 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -223,6 +223,9 @@ _body_matches_pattern() { # Review prose reusing the notice's leading literal must not clear. run _body_matches_pattern "$pat" "Review limit reached handling is too broad — this code path should be narrower." [ "$status" -eq 1 ] + # Anchored to the body start: prose that only mentions the notice text mid-body must not clear. + run _body_matches_pattern "$pat" "The error handling is broken. Review limit reached — you have used up your prepaid credits feature needs better UX." + [ "$status" -eq 1 ] } @test "info-status: qodo-code-review matches its trial-ended notice, not a review (#1995)" { @@ -239,6 +242,17 @@ _body_matches_pattern() { # Review prose reusing the notice's leading literal must not clear. run _body_matches_pattern "$pat" "Qodo reviews are paused because the health check failed — lean on monitoring instead." [ "$status" -eq 1 ] + # The real trial notice is prefixed with Qodo's HTML marker comment and must still clear. + run _body_matches_pattern "$pat" " Qodo reviews are paused because your trial has ended." + [ "$status" -eq 0 ] + # Anchored to the body start: prose that only mentions either notice mid-body must not clear. + run _body_matches_pattern "$pat" "The handling of the case where Qodo reviews are paused because your trial has ended is too lenient." + [ "$status" -eq 1 ] + run _body_matches_pattern "$pat" "When Qodo Merge has reached your monthly usage limit for pull-request reviews, the fallback logic is broken." + [ "$status" -eq 1 ] + # End-anchored trial sentence: a body that opens with it and goes on to a finding must not clear. + run _body_matches_pattern "$pat" "Qodo reviews are paused because your trial has ended is the wrong message here — fix the copy." + [ "$status" -eq 1 ] } @test "info-status: helper propagates a missing-manifest failure" { From 1b86c65a36ad3b75275c3e22b5346553dbbf05a0 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:44:38 +0000 Subject: [PATCH 6/7] fix(info-status): bot path checks the body; patterns match complete notices only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on #2000, plus a regression I introduced in 6f057853. 1. pr-review finding 1 (MEDIUM, blocking). maintainer-resolve-comment.sh authorized the registered-bot path by login only. Now that Codex, CodeRabbit and Qodo are registered, that let a maintainer minimize any of their FINDINGS as RESOLVED. The bot path now also requires the comment body to match the author's info_status_pattern (new pure helper mrc_bot_body_matches, the same case-sensitive jq test() the gate uses). The snapshot query reads `body`. A finding, or SonarCloud's "Quality Gate failed", is refused with exit 3. 2. CodeRabbit security review (moderate). A matching notice prefix cleared the whole comment, including finding text after it. Each pattern now matches the bot's COMPLETE known notice body, anchored at both ends (^ ... \s*$), with the observed optional parts (dashboard/settings links, billing link, the second sentence). 3. Regression. The start anchor in 6f057853 rejected Qodo's REAL trial notice, which reads "\n\n**ⓘ Qodo reviews are paused ...**". The pattern now allows the bold and ⓘ wrapper. Tests: the real Codex and Qodo bodies posted on these PRs clear; each notice followed by a finding blocks; the bot-path refusals. The new resolve-comment tests fail without the script change. The advisory_gate projection is still 5 rows. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015fdDZDb4mhT65MU1izeA4R --- scripts/lib/reviewer-sources.tsv | 13 ++--- scripts/maintainer-resolve-comment.sh | 51 ++++++++++++++---- .../unit/test_maintainer_comment_gate.bats | 10 ++++ .../unit/test_maintainer_resolve_comment.bats | 52 +++++++++++++++++-- tests/test_reviewer_sources.bats | 23 ++++++++ 5 files changed, 128 insertions(+), 21 deletions(-) diff --git a/scripts/lib/reviewer-sources.tsv b/scripts/lib/reviewer-sources.tsv index 8da639a1d..739f53bb2 100644 --- a/scripts/lib/reviewer-sources.tsv +++ b/scripts/lib/reviewer-sources.tsv @@ -49,9 +49,10 @@ # usage-limit / trial-ended / "review limit reached" / clean-status # message that carries no finding. Anchor it tightly — several required # literals in sequence, like the sonarqubecloud row — so it cannot match -# a review body that merely mentions a limit. Anchor service notices to -# the body start (^); allow leading HTML marker comments with -# ^\s*(\s*)* when the bot prefixes its notice with one. +# a review body that merely mentions a limit. Match the bot's COMPLETE +# known notice body, anchored at both ends (^ … \s*$), so a notice +# followed by finding text never clears; allow leading HTML marker +# comments with ^\s*(\s*)* when the bot prefixes one. # Never reach for this column to wave away an inconvenient review: a # finding-producing reviewer (codeant-ai, graphite-app) carries "-" and # its findings go through dev-lead's verified-fix flow. @@ -60,10 +61,10 @@ # login creates_threads dev_lead_trusted advisory_gate check_run_name info_status_pattern rationale copilot-pull-request-reviewer yes yes no - - Inline review comments; dev-lead addresses the findings. Dropped from the approval-wait set (advisory_gate=no, #1997) because zero reviews were observed across eight consecutive PRs (#1888, #1887, #1873, #1799, #1796, #1794, #1793, #1770), measured 2026-09-23. Stays dev-lead-trusted and on the #1220 scorecard; it leaves only the approval-wait set. gemini-code-assist yes yes yes - - Inline review comments; dev-lead addresses the findings. -chatgpt-codex-connector yes yes no - ^You have reached your Codex usage limits?( for code reviews)?\. Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start and to the end of the notice sentence, so neither a comment that merely quotes the notice nor one that opens with its words and goes on to a finding can clear; a real review body does not match and still blocks. Dropped from the approval-wait set (advisory_gate=no, #1997) because zero reviews were observed across eight consecutive PRs (#1888, #1887, #1873, #1799, #1796, #1794, #1793, #1770), measured 2026-09-23. Stays dev-lead-trusted and on the #1220 scorecard; it leaves only the approval-wait set. -coderabbitai yes yes no - ^\s*(\s*)*Review limit reached — you have used up your prepaid credits Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. +chatgpt-codex-connector yes yes no - ^You have reached your Codex usage limits?( for code reviews)?\.(\s+You can see your limits in the (\[Codex usage dashboard\]\([^)\s]*\)|Codex usage dashboard)\.)?(\s+To continue using code reviews, you can upgrade your account or add credits to your account and enable them for code reviews in your \[settings\]\([^)\s]*\)\.)?\s*$ Inline review comments (pull_request_review_comment); dev-lead addresses the findings. Its out-of-quota "You have reached your Codex usage limit[s for code reviews …]" notice — short and long forms alike — carries no finding, so the gate auto-clears it (#1995, #1993). Anchored to the body start and to the end of the notice sentence, so neither a comment that merely quotes the notice nor one that opens with its words and goes on to a finding can clear; a real review body does not match and still blocks. Dropped from the approval-wait set (advisory_gate=no, #1997) because zero reviews were observed across eight consecutive PRs (#1888, #1887, #1873, #1799, #1796, #1794, #1793, #1770), measured 2026-09-23. Stays dev-lead-trusted and on the #1220 scorecard; it leaves only the approval-wait set. +coderabbitai yes yes no - ^\s*(\s*)*Review limit reached — you have used up your prepaid credits\.?\s*$ Inline review comments; tracked by the scorecard and rate-limit detector but not part of the advisory approval-wait set. Its "Review limit reached — you have used up your prepaid credits" out-of-quota notice carries no finding, so the gate auto-clears it (#1995); a real review body does not match and still blocks. sonarqubecloud no yes yes - \*\*Quality Gate passed\*\*[\s\S]*\[0 New issues\][\s\S]*\[0 Security Hotspots\] Top-level quality-gate comment only — no inline review threads; trusted so dev-lead acts on the comment via the issue-comment path. Its "Quality Gate passed" status is re-posted on every push and carries no finding, so the gate auto-clears it (#1918); "Quality Gate failed" does not match and still blocks. -qodo-code-review yes yes no - ^\s*(\s*)*(Qodo reviews are paused because your trial has ended\.|Qodo Merge has reached your monthly usage limit for pull-request reviews) Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. Dropped from the approval-wait set (advisory_gate=no, #1997) because zero reviews were observed across eight consecutive PRs (#1888, #1887, #1873, #1799, #1796, #1794, #1793, #1770), measured 2026-09-23. Stays dev-lead-trusted and on the #1220 scorecard; it leaves only the approval-wait set. +qodo-code-review yes yes no - ^\s*(\s*)*((\*\*)?(ⓘ\s*)?Qodo reviews are paused because your trial has ended\.(\*\*)?(\s+Ask your workspace admin to add credits to resume reviews\.(\s+\[Manage billing\]\([^)\s]*\))?)?|Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository\.(\s+Reviews will resume when the limit resets\.)?)\s*$ Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). Its trial-ended and monthly-usage-limit notices carry no finding, so the gate auto-clears either (#1995); a real review body does not match and still blocks. Dropped from the approval-wait set (advisory_gate=no, #1997) because zero reviews were observed across eight consecutive PRs (#1888, #1887, #1873, #1799, #1796, #1794, #1793, #1770), measured 2026-09-23. Stays dev-lead-trusted and on the #1220 scorecard; it leaves only the approval-wait set. codeant-ai yes yes yes - - Inline review comments (#1355 recognized it for gating/scorecard; trusted for acting per #1425). graphite-app yes yes yes Graphite / AI Reviews - Inline review comments AND clean passes reported ONLY as the "Graphite / AI Reviews" check run (no PR review) — the scorecard must read that check run or every clean pass is miscounted as No response (#1908). Root cause of the 2026-08-02 unclearable-thread deadlock — trusted so dev-lead can act on its findings (#1425). Kept at advisory_gate=yes under #1997: its zero-review count is a measurement artefact (a clean pass posts only the check run, invisible to the reviews API), not an absence of signal; its 2026-08-21 silence was a lapsed 30-day trial and the org moved to the paid plan on 2026-09-22, so do NOT re-drop it. Known limitation (not a #1997 regression): the advisory gate counts only reviews/comments, so graphite's check-run signal stays uncountable by the gate — gate-side check-run counting needs its own issue. cubic-dev-ai yes yes yes - - Inline review comments plus a review summary; added 2026-09-22 on a 7-day trial (#1903). Flip advisory_gate to no (or remove this row) if the trial is not converted to paid (~2026-09-29). diff --git a/scripts/maintainer-resolve-comment.sh b/scripts/maintainer-resolve-comment.sh index 2606255d5..42e79b67f 100755 --- a/scripts/maintainer-resolve-comment.sh +++ b/scripts/maintainer-resolve-comment.sh @@ -159,6 +159,27 @@ mrc_is_registered_bot() { [[ "$search" == *$'\n'"$norm"$'\n'* ]] } +# mrc_bot_body_matches +# 0 iff matches the info_status_pattern that +# ("loginpattern" lines, as reviewer_sources_info_status_patterns emits) +# declares for ("[bot]" suffix stripped). The match is the same +# case-sensitive jq test() the maintainer-comment gate applies (#1918). This is +# what keeps the bot path a NOTICE-only escape hatch: a registered bot that also +# produces findings (Codex, CodeRabbit, Qodo, #1995) is authorized only for the +# comment body its pattern recognizes, never for a finding it posts. An empty +# author or body, a login with no pattern, or a jq error fails closed. Pure +# apart from jq. +mrc_bot_body_matches() { + local author="${1:-}" body="${2:-}" patterns="${3:-}" norm pattern result + [[ -n "$author" && -n "$body" && -n "$patterns" ]] || return 1 + norm="$(mrc_normalize_login "$author")" + [[ -n "$norm" ]] || return 1 + pattern="$(printf '%s\n' "$patterns" | awk -F'\t' -v l="$norm" '$1==l {print $2; exit}')" + [[ -n "$pattern" ]] || return 1 + result="$(jq -nr --arg b "$body" --arg p "$pattern" '($b | test($p))' 2>/dev/null)" || return 1 + [[ "$result" == "true" ]] +} + # mrc_reason_ok # 0 iff is non-empty after trimming whitespace. The bot-comment path # REQUIRES a reason (posted as a reply before minimizing, #1918 AC #3) so the @@ -348,7 +369,7 @@ if [[ "${BASH_SOURCE[0]}" = "${0}" ]]; then # viewer, in one GraphQL round-trip. Fail closed if either login is unreadable. # `url` is the authoritative source for the repo + PR number when posting the # bot-path reply (it is a /pull/…#issuecomment-… URL for a PR comment). - _q='query($id:ID!){viewer{login} node(id:$id){... on IssueComment{author{__typename login} isMinimized minimizedReason url}}}' + _q='query($id:ID!){viewer{login} node(id:$id){... on IssueComment{author{__typename login} body isMinimized minimizedReason url}}}' set +e _snap=$(gh api graphql -f query="$_q" -f id="$_node" 2>/dev/null) _status=$? @@ -378,6 +399,7 @@ if [[ "${BASH_SOURCE[0]}" = "${0}" ]]; then _is_min=$(printf '%s' "$_snap" | jq -r '.data.node.isMinimized // false | tostring' 2>/dev/null || echo "false") _min_reason=$(printf '%s' "$_snap" | jq -r '.data.node.minimizedReason // ""' 2>/dev/null || echo "") _url=$(printf '%s' "$_snap" | jq -r '.data.node.url // ""' 2>/dev/null || echo "") + _body=$(printf '%s' "$_snap" | jq -r '.data.node.body // ""' 2>/dev/null || echo "") # Authorize FIRST — before the idempotent early return — so a caller cannot pass # an already-RESOLVED comment they are not entitled to and receive success while @@ -386,11 +408,14 @@ if [[ "${BASH_SOURCE[0]}" = "${0}" ]]; then # a bot-authored comment must always take the bot path (its mandatory --reason # + audit reply), never the self path, even if gh is authenticated as that # same bot (#1918 review: keep bot authors out of self-authorization). - # • REGISTERED INFO-STATUS BOT (#1918): the author is a bot that declares an - # info-status pattern in the reviewer-source registry — i.e. a clean-status - # re-poster like SonarCloud, NOT a finding-producing reviewer (codeant-ai, - # graphite-app). The maintainer (a confirmed viewer) is the human-in-the-loop - # and MUST supply --reason, posted as a reply before the minimize. + # • REGISTERED INFO-STATUS NOTICE (#1918, #1995): the author is a bot that + # declares an info-status pattern in the reviewer-source registry AND this + # comment's body matches that pattern — i.e. a no-finding service notice + # (SonarCloud's clean status, Codex/CodeRabbit/Qodo usage-limit notices). + # The body check matters because some registered bots also post findings: + # the login alone must never authorize minimizing a finding. The maintainer + # (a confirmed viewer) is the human-in-the-loop and MUST supply --reason, + # posted as a reply before the minimize. # Another human's comment, or a finding-producing bot's, matches neither and is # refused — a finding still requires dev-lead's verified-fix flow or the author. _authz_kind="" @@ -404,17 +429,20 @@ if [[ "${BASH_SOURCE[0]}" = "${0}" ]]; then # Fail closed if the registry cannot be read — never widen authorization on a # registry we could not consult. _lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib" - _registered="" + _patterns="" if [[ -f "$_lib_dir/reviewer-sources.sh" ]]; then - _registered="$( + _patterns="$( REVIEWER_SOURCES_MANIFEST="$_lib_dir/reviewer-sources.tsv" export REVIEWER_SOURCES_MANIFEST # shellcheck source=lib/reviewer-sources.sh source "$_lib_dir/reviewer-sources.sh" 2>/dev/null \ - && reviewer_sources_info_status_patterns 2>/dev/null | cut -f1 - )" || _registered="" + && reviewer_sources_info_status_patterns 2>/dev/null + )" || _patterns="" fi - if [[ -n "$_viewer" && "$_author_type" == "Bot" ]] && mrc_is_registered_bot "$_author" "$_registered"; then + _registered="$(printf '%s\n' "$_patterns" | cut -f1)" + if [[ -n "$_viewer" && "$_author_type" == "Bot" ]] \ + && mrc_is_registered_bot "$_author" "$_registered" \ + && mrc_bot_body_matches "$_author" "$_body" "$_patterns"; then _authz_kind="bot" fi fi @@ -423,6 +451,7 @@ if [[ "${BASH_SOURCE[0]}" = "${0}" ]]; then echo "[maintainer-resolve-comment] ERROR: refusing to resolve — this path minimizes only YOUR OWN comment or a REGISTERED REVIEWER BOT's comment." >&2 echo " authenticated as: '${_viewer:-}'; comment author: '${_author:-}'" >&2 echo " a finding authored by another person must be dispositioned by dev-lead (verified fix) or by its author." >&2 + echo " a registered bot's comment qualifies only when its body matches that bot's info_status_pattern (a no-finding notice)." >&2 exit 3 fi diff --git a/tests/dev-lead/unit/test_maintainer_comment_gate.bats b/tests/dev-lead/unit/test_maintainer_comment_gate.bats index fb7273693..ae560cad7 100644 --- a/tests/dev-lead/unit/test_maintainer_comment_gate.bats +++ b/tests/dev-lead/unit/test_maintainer_comment_gate.bats @@ -360,6 +360,16 @@ Earlier this run reported: You have reached your Codex usage limits for code rev [ "$status" -eq 1 ] } +@test "AC1(#1995): Qodo's real trial notice (marker + bold ⓘ + billing link) clears → 0" { + _run_check "$(_notice_json qodo-code-review $'\n\n**ⓘ Qodo reviews are paused because your trial has ended.** Ask your workspace admin to add credits to resume reviews. [Manage billing](https://app.qodo.ai/account/billing/manage-subscription?traffic_source=pr_comment)')" + [ "$status" -eq 0 ] +} + +@test "AC2(#1995): a Codex notice followed by a finding still blocks → 1" { + _run_check "$(_notice_json chatgpt-codex-connector $'You have reached your Codex usage limits for code reviews.\n\n**P1** Possible null dereference in scripts/foo.sh:42.')" + [ "$status" -eq 1 ] +} + @test "AC1(#1995): Qodo monthly-usage-limit notice clears → 0" { _run_check "$(_notice_json qodo-code-review "Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository. Reviews will resume when the limit resets.")" [ "$status" -eq 0 ] diff --git a/tests/dev-lead/unit/test_maintainer_resolve_comment.bats b/tests/dev-lead/unit/test_maintainer_resolve_comment.bats index 4b9930cf0..ae83675d6 100644 --- a/tests/dev-lead/unit/test_maintainer_resolve_comment.bats +++ b/tests/dev-lead/unit/test_maintainer_resolve_comment.bats @@ -277,14 +277,20 @@ _mrc_minimized_reason() { # #1918 review: the registered-bot path must require a GitHub App actor. A PERSON # whose login happens to equal a registered bot login (e.g. a user account named # `sonarqubecloud`) must be refused like any other human — never minimized. +# A SonarCloud clean-status body that matches its registered info_status_pattern. +SONAR_PASS_JSON_BODY='**Quality Gate passed**\\n[0 New issues]\\n[0 Security Hotspots]' + _mrc_fake_gh() { - # $1 = author __typename. Writes a `gh` shim on PATH that answers the one GraphQL - # read and records any other call (a minimize or reply would be a failure). + # $1 = author __typename, $2 = author login (default sonarqubecloud), $3 = the + # comment body as a JSON string fragment (default a matching SonarCloud clean + # status). Writes a `gh` shim on PATH that answers the one GraphQL read and + # records any other call (a minimize or reply would be a failure). + local login="${2:-sonarqubecloud}" body="${3:-$SONAR_PASS_JSON_BODY}" local bin="$BATS_TEST_TMPDIR/bin"; mkdir -p "$bin" cat > "$bin/gh" <> "$BATS_TEST_TMPDIR/gh-writes.log" @@ -310,6 +316,44 @@ SHIM [ -s "$BATS_TEST_TMPDIR/gh-writes.log" ] } +# #1995 review: registering a bot that ALSO posts findings (Codex, CodeRabbit, +# Qodo) must not let the bot path minimize those findings. The body must match +# the author's info_status_pattern, not just the login. +@test "#1995: a registered bot's FINDING (body does not match its pattern) is refused (exit 3, no write)" { + _mrc_fake_gh Bot chatgpt-codex-connector '**P1** Possible null dereference in scripts/foo.sh:42 — guard the lookup.' + run bash "$MRC" "IC_kwDOfake1" --reason "status only" + [ "$status" -eq 3 ] + echo "$output" | grep -qi "info_status_pattern" + [ ! -s "$BATS_TEST_TMPDIR/gh-writes.log" ] +} + +@test "#1995: a registered bot's matching NOTICE is still authorized for the bot path" { + _mrc_fake_gh Bot chatgpt-codex-connector 'You have reached your Codex usage limits for code reviews. You can see your limits in the Codex usage dashboard.' + run bash "$MRC" "IC_kwDOfake1" --reason "usage-limit notice" + [ "$status" -ne 3 ] + [ -s "$BATS_TEST_TMPDIR/gh-writes.log" ] +} + +@test "#1995: a SonarCloud 'Quality Gate failed' comment is refused on the bot path" { + _mrc_fake_gh Bot sonarqubecloud '**Quality Gate failed**\\n[3 New issues]' + run bash "$MRC" "IC_kwDOfake1" --reason "status only" + [ "$status" -eq 3 ] + [ ! -s "$BATS_TEST_TMPDIR/gh-writes.log" ] +} + +@test "#1995: mrc_bot_body_matches is login-pinned and fails closed" { + local pats + pats=$'chatgpt-codex-connector\t^You have reached your Codex usage limits?( for code reviews)?\\.' + run bash -c "source '$MRC'; mrc_bot_body_matches 'chatgpt-codex-connector[bot]' 'You have reached your Codex usage limit.' \"\$1\"" _ "$pats" + [ "$status" -eq 0 ] + run bash -c "source '$MRC'; mrc_bot_body_matches 'someone-else' 'You have reached your Codex usage limit.' \"\$1\"" _ "$pats" + [ "$status" -ne 0 ] + run bash -c "source '$MRC'; mrc_bot_body_matches 'chatgpt-codex-connector' '' \"\$1\"" _ "$pats" + [ "$status" -ne 0 ] + run bash -c "source '$MRC'; mrc_bot_body_matches 'chatgpt-codex-connector' 'You have reached your Codex usage limit.' ''" + [ "$status" -ne 0 ] +} + # #1920 review: the reply-idempotency lookup must (a) only accept an existing # reply the invoking maintainer authored — a forged marker from any other login # must NOT suppress the audit reply (CWE-345) — and (b) fail closed if the @@ -323,7 +367,7 @@ _mrc_gh_reply_shim() { #!/usr/bin/env bash args="\$*" if [[ "\$args" == *"viewer{login}"* ]]; then - printf '%s' '{"data":{"viewer":{"login":"don-petry"},"node":{"author":{"__typename":"Bot","login":"sonarqubecloud"},"isMinimized":false,"minimizedReason":null,"url":"https://github.com/o/r/pull/1#issuecomment-1"}}}' + printf '%s' '{"data":{"viewer":{"login":"don-petry"},"node":{"author":{"__typename":"Bot","login":"sonarqubecloud"},"body":"$SONAR_PASS_JSON_BODY","isMinimized":false,"minimizedReason":null,"url":"https://github.com/o/r/pull/1#issuecomment-1"}}}' exit 0 fi if [[ "\$args" == *"minimizeComment"* ]]; then diff --git a/tests/test_reviewer_sources.bats b/tests/test_reviewer_sources.bats index fe7187e90..556f4f5f5 100644 --- a/tests/test_reviewer_sources.bats +++ b/tests/test_reviewer_sources.bats @@ -255,6 +255,29 @@ _body_matches_pattern() { [ "$status" -eq 1 ] } +# Each pattern matches the bot's COMPLETE known notice body and is anchored at +# both ends, so the real notices posted on this repo clear while a notice followed +# by finding text still blocks (CodeRabbit security review on #2000). +@test "info-status: the real Codex, Qodo and CodeRabbit notice bodies clear (#1995)" { + run _body_matches_pattern "$(_info_pattern_for chatgpt-codex-connector)" $'You have reached your Codex usage limits for code reviews. You can see your limits in the [Codex usage dashboard](https://chatgpt.com/codex/cloud/settings/usage).\nTo continue using code reviews, you can upgrade your account or add credits to your account and enable them for code reviews in your [settings](https://chatgpt.com/codex/cloud/settings/code-review).' + [ "$status" -eq 0 ] + run _body_matches_pattern "$(_info_pattern_for qodo-code-review)" $'\n\n**ⓘ Qodo reviews are paused because your trial has ended.** Ask your workspace admin to add credits to resume reviews. [Manage billing](https://app.qodo.ai/account/billing/manage-subscription?traffic_source=pr_comment)' + [ "$status" -eq 0 ] + run _body_matches_pattern "$(_info_pattern_for coderabbitai)" "Review limit reached — you have used up your prepaid credits." + [ "$status" -eq 0 ] +} + +@test "info-status: a complete notice FOLLOWED by finding text does not clear (#1995)" { + run _body_matches_pattern "$(_info_pattern_for chatgpt-codex-connector)" $'You have reached your Codex usage limits for code reviews.\n\n**P1** Possible null dereference in scripts/foo.sh:42.' + [ "$status" -eq 1 ] + run _body_matches_pattern "$(_info_pattern_for qodo-code-review)" $'**ⓘ Qodo reviews are paused because your trial has ended.** Also: parseConfig skips the bound check.' + [ "$status" -eq 1 ] + run _body_matches_pattern "$(_info_pattern_for qodo-code-review)" "Qodo Merge has reached your monthly usage limit for pull-request reviews on this repository. Also: the retry loop never terminates." + [ "$status" -eq 1 ] + run _body_matches_pattern "$(_info_pattern_for coderabbitai)" $'Review limit reached — you have used up your prepaid credits.\n\nActionable: guard cfg before use.' + [ "$status" -eq 1 ] +} + @test "info-status: helper propagates a missing-manifest failure" { REVIEWER_SOURCES_MANIFEST="/nonexistent/reviewer-sources.tsv" \ run reviewer_sources_info_status_patterns From 71380052baaa0c6b8e4789fb9a70c4d7fc96cfb2 Mon Sep 17 00:00:00 2001 From: donpetry-bot <{}+donpetry-bot@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:55:32 +0000 Subject: [PATCH 7/7] test(resolve-comment): fake gh bodies carry real newlines cubic P3: the shim's JSON used \\n, so the body decoded to a literal backslash-n rather than a newline, unlike a real GitHub response. The SonarCloud pass and fail fixtures now use \n. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015fdDZDb4mhT65MU1izeA4R --- tests/dev-lead/unit/test_maintainer_resolve_comment.bats | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/dev-lead/unit/test_maintainer_resolve_comment.bats b/tests/dev-lead/unit/test_maintainer_resolve_comment.bats index ae83675d6..c14e98773 100644 --- a/tests/dev-lead/unit/test_maintainer_resolve_comment.bats +++ b/tests/dev-lead/unit/test_maintainer_resolve_comment.bats @@ -278,7 +278,7 @@ _mrc_minimized_reason() { # whose login happens to equal a registered bot login (e.g. a user account named # `sonarqubecloud`) must be refused like any other human — never minimized. # A SonarCloud clean-status body that matches its registered info_status_pattern. -SONAR_PASS_JSON_BODY='**Quality Gate passed**\\n[0 New issues]\\n[0 Security Hotspots]' +SONAR_PASS_JSON_BODY='**Quality Gate passed**\n[0 New issues]\n[0 Security Hotspots]' _mrc_fake_gh() { # $1 = author __typename, $2 = author login (default sonarqubecloud), $3 = the @@ -335,7 +335,7 @@ SHIM } @test "#1995: a SonarCloud 'Quality Gate failed' comment is refused on the bot path" { - _mrc_fake_gh Bot sonarqubecloud '**Quality Gate failed**\\n[3 New issues]' + _mrc_fake_gh Bot sonarqubecloud '**Quality Gate failed**\n[3 New issues]' run bash "$MRC" "IC_kwDOfake1" --reason "status only" [ "$status" -eq 3 ] [ ! -s "$BATS_TEST_TMPDIR/gh-writes.log" ]