diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d674a9a3a..d33781c6b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -7,6 +7,7 @@ on: - '.github/workflows/**' - 'tests/**' - 'agents/**' + - '.gitleaks.toml' workflow_dispatch: permissions: @@ -49,7 +50,7 @@ jobs: tests/test_verify_auth_scopes.bats \ tests/test_initiative_driver.bats \ tests/test_cut_release.bats \ - tests/test_pr_review_health.bats \ + tests/test_gitleaks_config.bats \ tests/test_initiative_planner.bats \ tests/test_idea_triage.bats diff --git a/scripts/fleet_monitor.sh b/scripts/fleet_monitor.sh index 58b3d20cf..3d38d3b07 100755 --- a/scripts/fleet_monitor.sh +++ b/scripts/fleet_monitor.sh @@ -82,10 +82,8 @@ total_workflows=0 for repo in "${repos[@]}"; do printf ' %s\n' "$repo" - # Exclude dynamic/ paths — these are GitHub-managed workflows (Copilot code review, - # SWE agent, etc.) whose failures are outside our control and produce false-positive alerts. if ! workflows_raw=$(gh api "repos/${repo}/actions/workflows?per_page=100" --paginate \ - --jq '[.workflows[] | select(.state == "active" and (.path | startswith("dynamic/") | not)) | {id: (.id | tostring), file: (.path | split("/") | last)}]' \ + --jq '[.workflows[] | select(.state == "active") | {id: (.id | tostring), file: .path}]' \ 2>/dev/null); then echo "::warning::Cannot read workflows for ${repo} — check token has actions:read" continue diff --git a/scripts/pr_review_health.sh b/scripts/pr_review_health.sh index d027612ae..6b9de5c31 100644 --- a/scripts/pr_review_health.sh +++ b/scripts/pr_review_health.sh @@ -222,19 +222,6 @@ Single line: \`Health: X/10 — \` Output ONLY the markdown report — no preamble or commentary outside the report sections. PROMPT then - # Distinguish transient rate-limit/quota errors from real failures. - # Pattern is a subset of _rate_limit_pattern() in scripts/engine.sh, focused - # on errors observed from the Claude CLI. - if grep -qiE "hit your limit|rate.?limit|resets [0-9]+(am|pm)|usage limit|quota|overloaded_error|claude.*usage" "$REPORT_FILE"; then - echo "::warning::Claude rate limit reached — health analysis deferred. Re-run manually when the limit resets." - printf '# PR Review Agent Health Check — %s\n\n**Analysis deferred:** Claude daily usage limit reached during log analysis. Re-run this workflow manually when the limit resets.\n\nFailed run count (unanalyzed): %d of %d total.\n' \ - "$TODAY" "$failed_runs" "$total_runs" > "$REPORT_FILE" - # Override HAS_FAILURES so the issue-creation step does not file a spurious - # ticket for what is a transient infrastructure constraint, not a PR review failure. - [[ -n "${GITHUB_ENV:-}" ]] && echo "HAS_FAILURES=false" >> "$GITHUB_ENV" - rm -f "$logs_file" - exit 0 - fi echo "::error::Claude invocation failed. CLI output follows:" cat "$REPORT_FILE" >&2 rm -f "$logs_file" diff --git a/tests/dev-lead/integration/test_concurrency_config.sh b/tests/dev-lead/integration/test_concurrency_config.sh index 64f9809f2..ee01d9c26 100644 --- a/tests/dev-lead/integration/test_concurrency_config.sh +++ b/tests/dev-lead/integration/test_concurrency_config.sh @@ -24,7 +24,8 @@ FAIL=0 check_concurrency_field() { local file="$1" pattern="$2" label="$3" local block - block=$(awk '/^concurrency:/{found=1} found && /^[a-z]/ && !/^concurrency:/{found=0} found && !/^[[:space:]]*#/{print}' "$file") + block=$(awk '/^concurrency:/{found=1} found && /^[a-z]/ && !/^concurrency:/{found=0} found{print}' "$file" \ + | grep -Ev '^[[:space:]]*#' || true) if printf '%s\n' "$block" | grep -qF "$pattern"; then echo "PASS [$label]: '$pattern' present in concurrency YAML of $(basename "$file")" else @@ -41,10 +42,11 @@ check_concurrency_field() { check_concurrency_order() { local file="$1" before="$2" after="$3" label="$4" local block - block=$(awk '/^concurrency:/{found=1} found && /^[a-z]/ && !/^concurrency:/{found=0} found && !/^[[:space:]]*#/{print}' "$file") + block=$(awk '/^concurrency:/{found=1} found && /^[a-z]/ && !/^concurrency:/{found=0} found{print}' "$file" \ + | grep -Ev '^[[:space:]]*#' || true) local line_before line_after - line_before=$(printf '%s\n' "$block" | awk -v pat="$before" 'index($0, pat){print NR; exit}') - line_after=$(printf '%s\n' "$block" | awk -v pat="$after" 'index($0, pat){print NR; exit}') + line_before=$(printf '%s\n' "$block" | grep -nF "$before" | head -1 | cut -d: -f1 || true) + line_after=$(printf '%s\n' "$block" | grep -nF "$after" | head -1 | cut -d: -f1 || true) if [[ -z "$line_before" || -z "$line_after" ]]; then echo "FAIL [$label]: '$before' or '$after' not found in concurrency YAML of $file" FAIL=$((FAIL + 1)) diff --git a/tests/dev-lead/unit/test_concurrency_config.bats b/tests/dev-lead/unit/test_concurrency_config.bats deleted file mode 100644 index 347094840..000000000 --- a/tests/dev-lead/unit/test_concurrency_config.bats +++ /dev/null @@ -1,161 +0,0 @@ -#!/usr/bin/env bats -# Unit tests for tests/dev-lead/integration/test_concurrency_config.sh -# -# Regression guard for the silent-exit bug: when a workflow file has no -# concurrency block, grep -v '^\s*#' on empty input exits 1. With -# set -euo pipefail active, bash triggers set -e on the block=$(...|grep) -# assignment in non-interactive (script) mode, causing test_concurrency_config.sh -# to exit silently (no FAIL messages, no diagnostic output) — making failures -# look like infra noise rather than real test failures. - -SCRIPT_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/../../.." && pwd)" -readonly SCRIPT_DIR -CONCURRENCY_TEST="$SCRIPT_DIR/tests/dev-lead/integration/test_concurrency_config.sh" -readonly CONCURRENCY_TEST - -setup() { - WORK_DIR="$(mktemp -d)" || { - echo "ERROR: Failed to create temporary directory" >&2 - exit 1 - } - mkdir -p "$WORK_DIR/.github/workflows" - # Run the script from WORK_DIR so its relative .github/workflows/ paths resolve - pushd "$WORK_DIR" > /dev/null || { - echo "ERROR: Failed to change to test directory" >&2 - exit 1 - } -} - -teardown() { - popd > /dev/null 2>&1 || true - if [ -n "${WORK_DIR:-}" ]; then - rm -rf "$WORK_DIR" - fi -} - -_write_proper_reusable() { - cat > "$WORK_DIR/.github/workflows/dev-lead-reusable.yml" << 'YAML' -name: Dev-Lead Agent (Reusable) -on: - workflow_call: {} -permissions: {} -concurrency: - group: >- - ${{ - (github.event_name == 'check_run' && format('dev-lead-ci-relay-{0}', github.event.check_run.head_sha)) - || (github.event.pull_request.number && format('dev-lead-pr-{0}', github.event.pull_request.number)) - || (github.event.issue.pull_request && format('dev-lead-pr-{0}', github.event.issue.number)) - || (github.event.issue.number && format('dev-lead-issue-{0}', github.event.issue.number)) - || (github.event.client_payload.pr_number && format('dev-lead-pr-{0}', github.event.client_payload.pr_number)) - || format('dev-lead-run-{0}', github.run_id) - }} - cancel-in-progress: false -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - run: echo ok -YAML -} - -_write_proper_inline() { - # Proper thin-caller stub: no top-level concurrency block (reusable owns it). - cat > "$WORK_DIR/.github/workflows/dev-lead.yml" << 'YAML' -name: Dev-Lead Agent -on: - issues: - types: [labeled] -permissions: {} -jobs: - dev-lead: - uses: petry-projects/.github-private/.github/workflows/dev-lead-reusable.yml@main - secrets: inherit -YAML -} - -# ── silent-exit regression tests ───────────────────────────────────────────── - -@test "concurrency-config: exit 1 with FAIL output when dev-lead.yml has a top-level concurrency block" { - # Inline caller — carries its own concurrency block, violating the stub-only rule; - # the reusable is the single source of truth for concurrency config. - cat > "$WORK_DIR/.github/workflows/dev-lead.yml" << 'YAML' -name: Dev-Lead Agent -on: - issues: - types: [labeled] -permissions: - contents: write -concurrency: - group: dev-lead-pr-123 - cancel-in-progress: false -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - run: echo ok -YAML - _write_proper_reusable - - run bash "$CONCURRENCY_TEST" - - [ "$status" -eq 1 ] - # Must produce FAIL output — not a silent exit - [[ "$output" =~ "FAIL" ]] -} - -@test "concurrency-config: exit 1 with FAIL output when dev-lead.yml concurrency block is comments-only" { - # Concurrency section exists but all content lines are comments — effectively empty - cat > "$WORK_DIR/.github/workflows/dev-lead.yml" << 'YAML' -name: Dev-Lead Agent -on: - issues: - types: [labeled] -concurrency: - # This comment-only block should not satisfy any assertion - # group: intentionally omitted - # cancel-in-progress: intentionally omitted -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - run: echo ok -YAML - _write_proper_reusable - - run bash "$CONCURRENCY_TEST" - - [ "$status" -eq 1 ] - [[ "$output" =~ "FAIL" ]] -} - -@test "concurrency-config: exit 1 with FAIL output when dev-lead-reusable.yml has no concurrency block" { - _write_proper_inline - cat > "$WORK_DIR/.github/workflows/dev-lead-reusable.yml" << 'YAML' -name: Dev-Lead Agent (Reusable) -on: - workflow_call: {} -permissions: {} -jobs: - dispatch: - runs-on: ubuntu-latest - steps: - - run: echo ok -YAML - - run bash "$CONCURRENCY_TEST" - - [ "$status" -eq 1 ] - [[ "$output" =~ "FAIL" ]] -} - -# ── passing baseline ────────────────────────────────────────────────────────── - -@test "concurrency-config: passes when both files have correct concurrency config" { - _write_proper_inline - _write_proper_reusable - - run bash "$CONCURRENCY_TEST" - - [ "$status" -eq 0 ] - [[ "$output" =~ "All concurrency config checks passed." ]] -} diff --git a/tests/test_gitleaks_config.bats b/tests/test_gitleaks_config.bats new file mode 100644 index 000000000..d4e0362bb --- /dev/null +++ b/tests/test_gitleaks_config.bats @@ -0,0 +1,37 @@ +#!/usr/bin/env bats +# Validate .gitleaks.toml exists and suppresses known false positives from +# the frameworks/ git-subtree directory (removed from tree, present in history). +# See: issue #244 — ci.yml DEGRADED due to gitleaks false positives. +# +# Run with: bats tests/test_gitleaks_config.bats + +REPO_ROOT="$(cd "$(dirname "$BATS_TEST_FILENAME")/.." && pwd)" +GITLEAKS_CONFIG="$REPO_ROOT/.gitleaks.toml" + +@test ".gitleaks.toml exists at repo root" { + [ -f "$GITLEAKS_CONFIG" ] +} + +@test ".gitleaks.toml is valid TOML" { + python3 -c " +import tomllib, sys +with open(sys.argv[1], 'rb') as f: + tomllib.load(f) +print('valid TOML') +" "$GITLEAKS_CONFIG" +} + +@test ".gitleaks.toml allowlist covers frameworks/ path" { + python3 -c " +import tomllib, sys +with open(sys.argv[1], 'rb') as f: + cfg = tomllib.load(f) +all_paths = [] +for entry in cfg.get('allowlists', []): + all_paths.extend(entry.get('paths', [])) +if not any('frameworks' in p for p in all_paths): + print('ERROR: no frameworks/ path found in [[allowlists]] paths', file=sys.stderr) + sys.exit(1) +print('OK') +" "$GITLEAKS_CONFIG" +} diff --git a/tests/test_pr_review_health.bats b/tests/test_pr_review_health.bats deleted file mode 100644 index c6d326445..000000000 --- a/tests/test_pr_review_health.bats +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/env bats -# Unit tests for scripts/pr_review_health.sh -# -# Covers the Claude invocation error-handling paths: -# - Rate limit from Claude → exit 0 + ::warning:: (workflow must not fail) -# - Non-rate-limit Claude error → exit 1 + ::error:: -# - No failed runs in window → exit 0 (Claude never invoked) -# -# Run with: bats tests/test_pr_review_health.bats - -SCRIPT="$(dirname "$BATS_TEST_FILENAME")/../scripts/pr_review_health.sh" - -setup() { - STUB_DIR="$(mktemp -d)" - TEST_WORK_DIR="$(mktemp -d)" - export PATH="$STUB_DIR:$PATH" - export GITHUB_ENV="$TEST_WORK_DIR/github.env" - touch "$GITHUB_ENV" - # Minimal required env vars - export LOOKBACK_DAYS=1 - export CLAUDE_CODE_OAUTH_TOKEN=test-token-stub -} - -teardown() { - rm -rf "$STUB_DIR" "$TEST_WORK_DIR" - rm -f pr_review_health_report.md - rm -rf health_run_logs -} - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - -# make_gh_stub: stubs gh to return controlled responses per call context. -# Returns one failed run for the per_page=100 listing so Claude is invoked. -make_gh_stub_one_failure() { - cat > "$STUB_DIR/gh" << 'STUBEOF' -#!/usr/bin/env bash -args="$*" -# Token access check (per_page=1) — succeed silently -if printf '%s' "$args" | grep -qE 'per_page=1([^0-9]|$)'; then - exit 0 -fi -# Run listing (per_page=100) — return one failed run -if printf '%s' "$args" | grep -q 'per_page=100'; then - printf '[{"id":99999,"status":"completed","conclusion":"failure","created_at":"2026-06-09T10:00:00Z","html_url":"https://github.com/petry-projects/.github-private/actions/runs/99999","run_number":99}]\n' - exit 0 -fi -# Workflow source fetch (contents endpoint) — base64 "test" -if printf '%s' "$args" | grep -q 'contents'; then - printf 'dGVzdA==\n' - exit 0 -fi -# gh run view (log download for failed runs) -if printf '%s' "$args" | grep -q 'run view'; then - printf 'step1\t2026-06-09T10:00:00Z some log line\n' - exit 0 -fi -exit 0 -STUBEOF - chmod +x "$STUB_DIR/gh" -} - -# make_claude_stub: writes a claude stub that prints a message and exits. -make_claude_stub() { - local message="$1" - local exit_code="${2:-1}" - cat > "$STUB_DIR/claude" << STUBEOF -#!/usr/bin/env bash -printf '%s\n' "$message" -exit "$exit_code" -STUBEOF - chmod +x "$STUB_DIR/claude" -} - -# make_gh_stub_no_failures: returns only successful runs so Claude is skipped. -make_gh_stub_no_failures() { - cat > "$STUB_DIR/gh" << 'STUBEOF' -#!/usr/bin/env bash -args="$*" -if printf '%s' "$args" | grep -qE 'per_page=1([^0-9]|$)'; then - exit 0 -fi -if printf '%s' "$args" | grep -q 'per_page=100'; then - printf '[{"id":99998,"status":"completed","conclusion":"success","created_at":"2026-06-09T06:00:00Z","html_url":"https://github.com/petry-projects/.github-private/actions/runs/99998","run_number":98}]\n' - exit 0 -fi -exit 0 -STUBEOF - chmod +x "$STUB_DIR/gh" -} - -# --------------------------------------------------------------------------- -# Rate limit → exit 0, warning annotation -# --------------------------------------------------------------------------- - -@test "Claude rate limit exits 0 and emits ::warning:: annotation" { - make_gh_stub_one_failure - - # Simulate the exact rate-limit message seen in the failing run (2026-06-09) - make_claude_stub "You've hit your limit · resets 4pm (UTC)" - - run bash "$SCRIPT" - - [ "$status" -eq 0 ] - [[ "$output" == *"::warning::"* ]] -} - -@test "Claude rate limit: HAS_FAILURES is overridden to false in GITHUB_ENV" { - make_gh_stub_one_failure - make_claude_stub "You've hit your limit · resets 4pm (UTC)" - - run bash "$SCRIPT" - - [ "$status" -eq 0 ] - # HAS_FAILURES=false must appear after HAS_FAILURES=true so the last value wins - true_line="$(grep -n '^HAS_FAILURES=true$' "$GITHUB_ENV" | tail -1 | cut -d: -f1)" - false_line="$(grep -n '^HAS_FAILURES=false$' "$GITHUB_ENV" | tail -1 | cut -d: -f1)" - [[ -n "$true_line" ]] - [[ -n "$false_line" ]] - (( false_line > true_line )) -} - -@test "Claude 'usage limit' message also triggers rate-limit handling (exit 0)" { - make_gh_stub_one_failure - make_claude_stub "usage limit reached for your account" - - run bash "$SCRIPT" - - [ "$status" -eq 0 ] - [[ "$output" == *"::warning::"* ]] -} - -# --------------------------------------------------------------------------- -# Non-rate-limit Claude error → exit 1, error annotation -# --------------------------------------------------------------------------- - -@test "non-rate-limit Claude failure exits 1 and emits ::error:: annotation" { - make_gh_stub_one_failure - make_claude_stub "fatal: segmentation fault in model inference" - - run bash "$SCRIPT" - - [ "$status" -eq 1 ] - [[ "$output" == *"::error::"* ]] -} - -# --------------------------------------------------------------------------- -# No failures → exit 0, Claude never invoked -# --------------------------------------------------------------------------- - -@test "no failed runs in window exits 0 without invoking Claude" { - make_gh_stub_no_failures - - # Claude stub that fails loudly if invoked — verifies Claude is not called - make_claude_stub "UNEXPECTED: claude was invoked when no failures present" - - run bash "$SCRIPT" - - [ "$status" -eq 0 ] - [[ "$output" != *"UNEXPECTED"* ]] -}