feat: implement issue #809 — upsert_gitignore_baseline must preserve baseline negations + migrate unmarkered baselines (blocks #800 apply)

scripts/lib/gitignore-baseline.sh

@@ -55,6 +55,80 @@ _gib_has_markers() {
     && printf '%s\n' "$content" | grep -qxF "$GIB_END_MARKER"
 }
 
+# _gib_neutralize_l2 <block>
+# Read candidate L2 content on stdin; print it with every line that would fight
+# the L1 block removed. A line is dropped when — after CRLF and trailing-whitespace
+# trimming — it exactly matches a *pattern* line of <block> (comment and blank
+# lines of the block are ignored, so they never strip a repo's own comments). This
+# does double duty:
+#
+#   • Negations win (identical patterns): a bare re-hiding pattern the target kept
+#     in L2 (e.g. `*.pem` with no `!public.pem`) is identical to the block's own
+#     broad pattern, so it is stripped.
+#   • Negations win (non-identical globs): a glob-variant pattern (e.g. `**/*.pem`
+#     or `.env*`) that is NOT an exact match for any block line but would still
+#     re-hide a baseline-negated path (e.g. `!public.pem` or `!.env.example`) is
+#     handled by the *re-allow tail*: all baseline negation lines are re-emitted at
+#     the very end of the L2 output so they always evaluate after any L2 pattern —
+#     identical or not — and the negated paths remain re-allowed.
+#   • Migrate, don't duplicate: an old *unmarkered* baseline pasted into L2 has its
+#     pattern lines folded away (they all live in the block now), so upsert stops
+#     shipping a second full copy. The genuine ecosystem/OS entries the repo added
+#     stay put.
+#
+# Only pattern lines are matched for dropping — never comments — because block
+# comments include bare `#` separators that would otherwise clobber a repo's own
+# comment lines. Runs of blank lines left behind by the removed patterns are
+# squeezed (leading and consecutive blanks dropped) so migration output stays tidy.
+# Idempotent: exact-match lines are gone after one pass; the re-allow tail lines
+# match exactly on re-run and are dropped before being re-appended. The tail is
+# only appended when at least one real (non-comment, non-blank) pattern survives
+# in L2 — a comment-only L2 cannot re-hide anything, so no tail is emitted and
+# an already-current file stays byte-for-byte identical on re-run.
+_gib_neutralize_l2() {
+  local block="$1"
+  block="$(tr -d '\r' <<< "$block")"
+  tr -d '\r' | awk -v block="$block" '
+    BEGIN {
+      n = split(block, barr, "\n")
+      for (i = 1; i <= n; i++) {
+        bl = barr[i]
+        sub(/[ \t]+$/, "", bl)
+        if (bl ~ /^[ \t]*$/) continue    # skip blank block lines
+        if (bl ~ /^[ \t]*#/) continue    # skip comment block lines (incl. bare #)
+        drop[bl] = 1
+        if (bl ~ /^!/) neg[++neg_count] = bl   # collect negation anchors for tail
+      }
+      prev_blank = 1   # squeeze any blank lines that would lead the output
+    }
+    {
+      line = $0
+      key = line
+      sub(/[ \t]+$/, "", key)
+      if (key in drop) next
+      is_blank = (key ~ /^[ \t]*$/)
+      if (is_blank && prev_blank) next
+      print line
+      prev_blank = is_blank
+      if (!is_blank && !(key ~ /^[ \t]*#/)) had_pattern = 1
+    }
+    END {
+      # Re-emit baseline negation anchors last so they win over any L2 pattern
+      # (identical or non-identical glob) that would otherwise re-hide a
+      # baseline-negated path (e.g. `.env*` re-hides `!.env.example`, or
+      # `**/*.pem` re-hides `!public.pem`). Idempotent: the appended negations
+      # are in `drop`, so a second pass strips them before re-appending here.
+      # Guard: only emit when a real pattern survived — comment-only L2 cannot
+      # re-hide anything, and emitting the tail would make an already-current
+      # file appear changed (triggering spurious REFRESH PRs).
+      if (had_pattern && neg_count > 0) {
+        if (!prev_blank) print ""
+        for (i = 1; i <= neg_count; i++) print neg[i]
+      }
+    }
+  '
+}
+
 # upsert_gitignore_baseline <block_file> [existing_file]
 # Print the upserted .gitignore content to stdout:
 #   • <block_file>   — a file containing the marker-wrapped L1 block (e.g. the
@@ -64,10 +138,17 @@ _gib_has_markers() {
 #
 # Behaviour (idempotent):
 #   • existing has the markers  → REPLACE the block in place; content above BEGIN
-#                                 and the entire L2 below END are preserved verbatim.
-#   • existing lacks the markers → INSERT the block on top; the whole existing file
-#                                 becomes L2 below the END marker.
+#                                 is preserved verbatim and the L2 below END is
+#                                 neutralized (see below).
+#   • existing lacks the markers → INSERT the block on top; the existing file
+#                                 becomes L2 below the END marker, neutralized.
 #   • existing empty/absent      → output is exactly the block.
+#
+# L2 neutralization (#809): every L2 line that duplicates a block *pattern* line is
+# dropped, so a re-hiding pattern the target kept in L2 (e.g. a bare `*.pem` with no
+# `!public.pem`) can't override the block's negations, and an old unmarkered
+# baseline pasted into L2 folds into the single block instead of duplicating.
+# Genuine per-repo L2 (ecosystem/OS entries) is preserved. See _gib_neutralize_l2.
 # Re-running on its own output is a no-op.
 upsert_gitignore_baseline() {
   local block_file="${1:-}" existing_file="${2:-}"
@@ -101,10 +182,13 @@ upsert_gitignore_baseline() {
     return 2
   fi
 
-  # Marker-less target: block on top, existing content becomes L2.
+  # Marker-less target: block on top, existing content becomes L2 — but neutralize
+  # any L2 line that duplicates/fights the block (see _gib_neutralize_l2).
   if [ "$has_begin" -eq 0 ]; then
+    local l2
+    l2="$(_gib_neutralize_l2 "$block" <<< "$existing")"
     printf '%s\n' "$block"
-    printf '%s\n' "$existing"
+    [ -n "$l2" ] && printf '%s\n' "$l2"
     return 0
   fi
 
@@ -114,6 +198,7 @@ upsert_gitignore_baseline() {
   local pre post
   pre="$(tr -d '\r' <<< "$existing" | awk -v b="$GIB_BEGIN_MARKER" '$0 == b { exit } { print }')"
   post="$(tr -d '\r' <<< "$existing" | awk -v e="$GIB_END_MARKER" 'seen { print } $0 == e { seen = 1 }')"
+  post="$(_gib_neutralize_l2 "$block" <<< "$post")"
 
   [ -n "$pre" ] && printf '%s\n' "$pre"
   printf '%s\n' "$block"

test/scripts/compliance-remediate/gitignore-baseline.bats

@@ -182,7 +182,11 @@ run_remediate() {
 # no mutating call, and it is recorded as a skip (not a failure).
 # ---------------------------------------------------------------------------
 @test "already-current baseline is an idempotent no-op (no PR opened)" {
-  GH_GITIGNORE_B64="$(b64 < "$CANONICAL")"; export GH_GITIGNORE_B64
+  # "Current" = the upsert steady state (block + neutralized L2 + negation tail
+  # per #809), i.e. a repo already synced — not the raw canonical.
+  source "${TT_REPO_ROOT}/scripts/lib/gitignore-baseline.sh"
+  _blk="$(mktemp "$TT_TMP/block.XXXXXX")"; gib_extract_baseline_block "$CANONICAL" > "$_blk"
+  GH_GITIGNORE_B64="$(upsert_gitignore_baseline "$_blk" "$CANONICAL" | b64)"; export GH_GITIGNORE_B64
 
   findings="$(tt_write_finding "markets" "push-protection" "gitignore_baseline")"
   run_remediate "$findings"

test/scripts/lib/gitignore-baseline.bats

@@ -182,3 +182,148 @@ EOF
   [ "$status" -eq 2 ]
   [[ "$output" == *"half-open"* ]]
 }
+
+# ── #809: baseline negations must win over a re-hiding L2 pattern ──────────────
+# Reproduces the fleet dry-run failure on `markets` / `google-app-scripts`: an
+# ad-hoc L2 with a broad `*.pem` and no `!public.pem` re-hid the file the L1 block
+# re-allows. Assert against real git semantics with `git check-ignore`.
+
+# _gib_check_ignored <gitignore_content> <path>
+# Return 0 if <path> is ignored, 1 if not — mirrors the issue's repro probe.
+_gib_check_ignored() {
+  local content="$1" path="$2" d
+  d="$(mktemp -d "$BATS_TEST_TMPDIR/stub.XXXXXX")"
+  ( cd "$d" && git init -q )
+  printf '%s\n' "$content" > "$d/.gitignore"
+  ( cd "$d" && git -c core.excludesfile=/dev/null check-ignore -q "$path" )
+  local rc=$?
+  rm -rf "$d"
+  return "$rc"
+}
+
+@test "#809 markets-like: broad *.pem in L2 does not re-hide baseline-negated public.pem" {
+  # Use the REAL canonical block (it carries !public.pem after *.pem).
+  local realblock="${TT_TMP}/realblock"
+  gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock"
+
+  local existing="${TT_TMP}/markets"
+  printf '# markets ad-hoc\n*.pem\nnode_modules/\n' > "$existing"
+
+  # Capture the upsert output in its own var — bats clobbers $output on every `run`.
+  local gi
+  gi="$(upsert_gitignore_baseline "$realblock" "$existing")"
+
+  run _gib_check_ignored "$gi" public.pem
+  [ "$status" -eq 1 ]   # NOT ignored
+  # A genuine secret pem is still ignored by the baseline.
+  run _gib_check_ignored "$gi" secret.pem
+  [ "$status" -eq 0 ]
+  # Genuine L2 survives.
+  grep -qxF 'node_modules/' <<< "$gi"
+}
+
+@test "#809 google-app-scripts-like: broad *.pem in L2 does not re-hide public.pem" {
+  local realblock="${TT_TMP}/realblock"
+  gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock"
+
+  local existing="${TT_TMP}/gas"
+  printf '.clasp.json\n*.pem\n' > "$existing"
+
+  local gi
+  gi="$(upsert_gitignore_baseline "$realblock" "$existing")"
+
+  run _gib_check_ignored "$gi" public.pem
+  [ "$status" -eq 1 ]   # NOT ignored
+  grep -qxF '.clasp.json' <<< "$gi"
+}
+
+@test "#809 non-identical: .env* glob in L2 does not re-hide baseline-negated .env.example" {
+  # `.env*` is NOT an exact match for `.env` or `.env.*` in the baseline, so
+  # the current exact-drop logic leaves it in L2 — but it would re-hide
+  # `!.env.example`. The re-allow tail must win.
+  local realblock="${TT_TMP}/realblock"
+  gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock"
+
+  local existing="${TT_TMP}/envglob"
+  printf '# project extras\n.env*\nnode_modules/\n' > "$existing"
+
+  local gi
+  gi="$(upsert_gitignore_baseline "$realblock" "$existing")"
+
+  run _gib_check_ignored "$gi" .env.example
+  [ "$status" -eq 1 ]   # NOT ignored — baseline negation must win
+  run _gib_check_ignored "$gi" .env.production
+  [ "$status" -eq 0 ]   # ignored — broad coverage still applies
+  grep -qxF 'node_modules/' <<< "$gi"
+}
+
+@test "#809 non-identical: **/*.pem glob in L2 does not re-hide baseline-negated public.pem" {
+  # `**/*.pem` is NOT an exact match for `*.pem` in the baseline, so the
+  # exact-drop logic leaves it in L2 — but it would re-hide `!public.pem`.
+  # The re-allow tail must win.
+  local realblock="${TT_TMP}/realblock"
+  gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock"
+
+  local existing="${TT_TMP}/doublepem"
+  printf '# project extras\n**/*.pem\nnode_modules/\n' > "$existing"
+
+  local gi
+  gi="$(upsert_gitignore_baseline "$realblock" "$existing")"
+
+  run _gib_check_ignored "$gi" public.pem
+  [ "$status" -eq 1 ]   # NOT ignored — baseline negation must win
+  run _gib_check_ignored "$gi" secret.pem
+  [ "$status" -eq 0 ]   # ignored — broad coverage still applies
+  grep -qxF 'node_modules/' <<< "$gi"
+}
+
+# ── #809: migrate an existing unmarkered baseline, don't duplicate it ──────────
+@test "#809 TalkTerm-like: unmarkered baseline in L2 is folded into one block, not duplicated" {
+  # L2 IS a copy of the old unmarkered baseline (same secret lines, no markers),
+  # plus genuine ecosystem/OS entries the repo legitimately added.
+  local existing="${TT_TMP}/talkterm"
+  cat > "$existing" <<EOF
+.env
+!.env.example
+*.pem
+!*.pub
+*.key
+node_modules/
+dist/
+*.log
+.DS_Store
+EOF
+
+  run upsert_gitignore_baseline "$BLOCK" "$existing"
+  [ "$status" -eq 0 ]
+
+  # Exactly one marker-wrapped block.
+  [ "$(printf '%s\n' "$output" | grep -cxF "$GIB_BEGIN_MARKER")" -eq 1 ]
+  [ "$(printf '%s\n' "$output" | grep -cxF "$GIB_END_MARKER")" -eq 1 ]
+  # Broad patterns appear exactly once (the L2 copies are folded away; they are
+  # not negations so they are not re-emitted in the re-allow tail).
+  [ "$(printf '%s\n' "$output" | grep -cxF '.env')" -eq 1 ]
+  [ "$(printf '%s\n' "$output" | grep -cxF '*.pem')" -eq 1 ]
+  # Negation anchors appear twice: once inside the block and once in the
+  # re-allow tail that ensures they win over any surviving L2 pattern.
+  [ "$(printf '%s\n' "$output" | grep -cxF '!.env.example')" -eq 2 ]
+  # Genuine ecosystem/OS L2 entries are preserved.
+  grep -qxF 'node_modules/' <<< "$output"
+  grep -qxF 'dist/' <<< "$output"
+  grep -qxF '*.log' <<< "$output"
+  grep -qxF '.DS_Store' <<< "$output"
+}
+
+@test "#809 negation-win + migration is idempotent" {
+  local realblock="${TT_TMP}/realblock"
+  gib_extract_baseline_block "${TT_REPO_ROOT}/.gitignore" > "$realblock"
+
+  local existing="${TT_TMP}/markets2"
+  printf '# markets ad-hoc\n*.pem\nnode_modules/\n' > "$existing"
+
+  local first second
+  first="$(upsert_gitignore_baseline "$realblock" "$existing")"
+  printf '%s\n' "$first" > "${TT_TMP}/after1"
+  second="$(upsert_gitignore_baseline "$realblock" "${TT_TMP}/after1")"
+  [ "$first" = "$second" ]
+}

test/scripts/sync-gitignore-baseline/dry-run.bats

@@ -61,7 +61,11 @@ b64() { base64 -w 0 2>/dev/null || base64 -b 0; }
 }
 
 @test "dry-run skips a repo whose .gitignore already carries the current baseline" {
-  GH_CONTENT_B64="$(b64 < "$CANONICAL")"; export GH_CONTENT_B64
+  # "Current" = the upsert steady state (block + neutralized L2 + negation tail
+  # per #809), i.e. a repo already synced by this tool — not the raw canonical.
+  source "${REPO_ROOT}/scripts/lib/gitignore-baseline.sh"
+  _blk="$(mktemp "$TT_TMP/block.XXXXXX")"; gib_extract_baseline_block "$CANONICAL" > "$_blk"
+  GH_CONTENT_B64="$(upsert_gitignore_baseline "$_blk" "$CANONICAL" | b64)"; export GH_CONTENT_B64
   install_gh_stub
   run env GH_TOKEN=x bash "$SCRIPT" --dry-run --repo markets
   [ "$status" -eq 0 ]