mw-plugin-test: --allow known-debt ratchet for the plugins gate

test/MeshWeaver.PluginTester.Test/GateAllowlistTest.cs

@@ -0,0 +1,204 @@
+#pragma warning disable CS1591
+
+using System;
+using System.IO;
+using System.Linq;
+using MeshWeaver.PluginTester;
+using Xunit;
+
+namespace MeshWeaver.PluginTester.Test;
+
+/// <summary>
+/// The known-debt ratchet, pinned check by check: parsing (including the malformed-line throw —
+/// a typo that silently allowed nothing would defeat the ratchet), the classification of every
+/// failure into new-vs-known, the STALE rule (a passing check fails the run until its entry is
+/// removed — the list only shrinks), and the unverifiable escape (a skipped check or absent
+/// scope proves nothing, so it warns and never fails).
+/// </summary>
+public class GateAllowlistTest
+{
+    // ── parsing ──────────────────────────────────────────────────────────────────────────────
+
+    [Fact]
+    public void Parse_ReadsEntries_SkipsCommentsAndBlanks()
+    {
+        var allow = GateAllowlist.Parse(
+        [
+            "# the debt list",
+            "",
+            "Claims idempotence   # tracked in #235",
+            "Edu/Exercise tests",
+        ]);
+        Assert.Equal(2, allow.Entries.Count);
+        Assert.True(allow.Allows("Claims", "idempotence"));
+        Assert.True(allow.Allows("Edu/Exercise", "tests"));
+        Assert.False(allow.Allows("Claims", "install"));
+    }
+
+    [Fact]
+    public void Parse_MatchesCaseInsensitively()
+    {
+        var allow = GateAllowlist.Parse(["claims IDEMPOTENCE"]);
+        Assert.True(allow.Allows("Claims", "idempotence"));
+    }
+
+    [Theory]
+    [InlineData("Claims")]                      // missing check
+    [InlineData("Claims idempotence extra")]    // too many tokens
+    [InlineData("Claims flakiness")]            // unknown check name
+    public void Parse_ThrowsOnMalformedLine(string line)
+    {
+        var ex = Assert.Throws<FormatException>(() => GateAllowlist.Parse([line]));
+        Assert.Contains("line 1", ex.Message);
+    }
+
+    // ── classification ───────────────────────────────────────────────────────────────────────
+
+    private static GateReport Report(params PackageResult[] packages) => new([.. packages]);
+
+    private static PackageResult IdempotenceBroken(string id) =>
+        new(id) { IdempotenceError = "re-install wrote 1 node(s)" };
+
+    private static PackageResult WithType(string id, NodeTypeResult type) =>
+        new(id) { NodeTypes = [type] };
+
+    private static NodeTypeResult TestsRed(string path) =>
+        new(path, path.Split('/')[0]) { Tests = CheckOutcome.Failed, TestsDetail = "1 red row" };
+
+    [Fact]
+    public void Evaluate_SplitsFailures_IntoKnownDebtAndNew()
+    {
+        var report = Report(IdempotenceBroken("Claims"), IdempotenceBroken("Video"));
+        var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"]));
+
+        Assert.Single(verdict.KnownDebt);
+        Assert.True(verdict.IsKnownDebt("Claims", "idempotence"));
+        Assert.Single(verdict.NewFailures);
+        Assert.Equal("Video", verdict.NewFailures[0].Scope);
+        Assert.False(verdict.Success);
+    }
+
+    [Fact]
+    public void Evaluate_AllKnown_IsGreen()
+    {
+        var report = Report(
+            IdempotenceBroken("Claims"),
+            WithType("Edu", TestsRed("Edu/Exercise")));
+        var verdict = GateVerdict.Evaluate(
+            report, GateAllowlist.Parse(["Claims idempotence", "Edu/Exercise tests"]));
+
+        Assert.Empty(verdict.NewFailures);
+        Assert.Empty(verdict.Stale);
+        Assert.Equal(2, verdict.KnownDebt.Count);
+        Assert.True(verdict.Success);
+    }
+
+    [Fact]
+    public void Evaluate_TypeChecks_MatchOnTheTypePath()
+    {
+        var report = Report(WithType("Edu", TestsRed("Edu/Exercise")));
+        var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Edu tests"]));
+
+        // The package-scoped entry does NOT cover a type-level failure: exact scope only,
+        // or the ratchet stops ratcheting.
+        Assert.Single(verdict.NewFailures);
+        Assert.False(verdict.Success);
+    }
+
+    // ── the stale rule ───────────────────────────────────────────────────────────────────────
+
+    [Fact]
+    public void Evaluate_EntryWhoseCheckPasses_IsStale_AndFailsTheRun()
+    {
+        var healthy = new PackageResult("Claims");
+        var verdict = GateVerdict.Evaluate(
+            Report(healthy), GateAllowlist.Parse(["Claims idempotence"]));
+
+        Assert.Single(verdict.Stale);
+        Assert.Empty(verdict.NewFailures);
+        Assert.False(verdict.Success);
+    }
+
+    [Fact]
+    public void Evaluate_TypeEntryWhoseTestsPass_IsStale()
+    {
+        var green = new NodeTypeResult("Edu/Exercise", "Edu") { Tests = CheckOutcome.Passed };
+        var verdict = GateVerdict.Evaluate(
+            Report(WithType("Edu", green)), GateAllowlist.Parse(["Edu/Exercise tests"]));
+
+        Assert.Single(verdict.Stale);
+        Assert.False(verdict.Success);
+    }
+
+    // ── the unverifiable escape ──────────────────────────────────────────────────────────────
+
+    [Fact]
+    public void Evaluate_AbsentScope_IsUnverifiable_NotStale()
+    {
+        var verdict = GateVerdict.Evaluate(
+            Report(new PackageResult("Claims")), GateAllowlist.Parse(["Gone/Type tests"]));
+
+        Assert.Single(verdict.Unverifiable);
+        Assert.Empty(verdict.Stale);
+        Assert.True(verdict.Success);
+    }
+
+    [Fact]
+    public void Evaluate_SkippedTests_AreUnverifiable_NotStale()
+    {
+        var skipped = new NodeTypeResult("Edu/Exercise", "Edu") { Tests = CheckOutcome.Skipped };
+        var verdict = GateVerdict.Evaluate(
+            Report(WithType("Edu", skipped)), GateAllowlist.Parse(["Edu/Exercise tests"]));
+
+        Assert.Single(verdict.Unverifiable);
+        Assert.Empty(verdict.Stale);
+        Assert.True(verdict.Success);
+    }
+
+    [Fact]
+    public void Evaluate_IdempotenceEntry_BehindAFailedInstall_IsUnverifiable()
+    {
+        // The idempotence pin never ran if the install itself failed — its entry must not be
+        // stale (that would demand removing an entry for debt that may well still exist).
+        var broken = new PackageResult("Claims") { InstallError = "boom" };
+        var verdict = GateVerdict.Evaluate(
+            Report(broken), GateAllowlist.Parse(["Claims idempotence"]));
+
+        Assert.Single(verdict.Unverifiable);
+        Assert.Empty(verdict.Stale);
+        // The install failure itself is unlisted → a new failure → the run still fails.
+        Assert.Single(verdict.NewFailures);
+        Assert.False(verdict.Success);
+    }
+
+    // ── the summary states ONE verdict ───────────────────────────────────────────────────────
+
+    [Fact]
+    public void Summary_LabelsKnownDebt_AndStatesTheVerdictLine()
+    {
+        var report = Report(IdempotenceBroken("Claims"));
+        var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"]));
+        var output = new StringWriter();
+        report.WriteSummary(output, verdict);
+        var text = output.ToString();
+
+        Assert.Contains("[DEBT] Claims", text);
+        Assert.Contains("[known-debt]", text);
+        Assert.Contains("GREEN — 1 known-debt failure(s) allowed", text);
+        Assert.DoesNotContain("GATE FAILED", text);
+    }
+
+    [Fact]
+    public void Summary_NamesStaleEntries_AndFails()
+    {
+        var report = Report(new PackageResult("Claims"));
+        var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"]));
+        var output = new StringWriter();
+        report.WriteSummary(output, verdict);
+        var text = output.ToString();
+
+        Assert.Contains("STALE allow entry", text);
+        Assert.Contains("Claims idempotence", text);
+        Assert.Contains("GATE FAILED", text);
+    }
+}

tools/MeshWeaver.PluginTester/GateAllowlist.cs

@@ -0,0 +1,164 @@
+namespace MeshWeaver.PluginTester;
+
+/// <summary>
+/// The gate's known-debt ratchet — the same one-way contract as the plugins repo's
+/// <c>compile-check.allow</c>: a failure listed here is reported as debt but does not fail the
+/// run; a NEW failure (not listed) fails; and an entry whose check now PASSES is STALE and fails
+/// the run, so the list can only shrink. Without this, arming the gate on a repo with
+/// pre-existing debt means either merging red or silencing the whole job — both worse than a
+/// ratchet.
+///
+/// <para>File format: one entry per line, <c>#</c> comments, blank lines ignored:</para>
+/// <code>
+/// &lt;scope&gt; &lt;check&gt;
+/// Claims idempotence          # package-level checks: install, idempotence
+/// Edu/Exercise tests          # type-level checks: compile, render, tests
+/// </code>
+/// </summary>
+public sealed record GateAllowlist(IReadOnlyList<AllowEntry> Entries)
+{
+    /// <summary>The empty list — every failure is a new failure.</summary>
+    public static readonly GateAllowlist Empty = new([]);
+
+    /// <summary>The valid check names, package-level and type-level.</summary>
+    public static readonly IReadOnlySet<string> Checks =
+        new HashSet<string>(StringComparer.OrdinalIgnoreCase)
+            { "install", "idempotence", "compile", "render", "tests" };
+
+    /// <summary>Parses the allow file, throwing on a malformed line — a typo that silently
+    /// allowed nothing (or everything) would defeat the ratchet.</summary>
+    public static GateAllowlist Parse(IEnumerable<string> lines)
+    {
+        var entries = new List<AllowEntry>();
+        var number = 0;
+        foreach (var raw in lines)
+        {
+            number++;
+            var line = StripComment(raw).Trim();
+            if (line.Length == 0)
+                continue;
+            var parts = line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries);
+            if (parts.Length != 2 || !Checks.Contains(parts[1]))
+                throw new FormatException(
+                    $"allow file line {number}: expected '<scope> <check>' with check one of " +
+                    $"[{string.Join(", ", Checks)}], got '{line}'");
+            entries.Add(new AllowEntry(parts[0], parts[1].ToLowerInvariant()));
+        }
+        return new GateAllowlist(entries);
+    }
+
+    /// <summary>Loads and parses the allow file at <paramref name="path"/>.</summary>
+    public static GateAllowlist Load(string path) => Parse(File.ReadLines(path));
+
+    private static string StripComment(string line) =>
+        line.IndexOf('#') is var i && i >= 0 ? line[..i] : line;
+
+    /// <summary>Whether a failure at <paramref name="scope"/>/<paramref name="check"/> is known debt.</summary>
+    public bool Allows(string scope, string check) => Entries.Any(e => e.Matches(scope, check));
+}
+
+/// <summary>One known-debt entry: a scope (package id or NodeType path) and a check name.</summary>
+public sealed record AllowEntry(string Scope, string Check)
+{
+    /// <summary>Case-insensitive match, exact scope — no globs: an entry must name exactly the
+    /// debt it tolerates, or the ratchet stops ratcheting.</summary>
+    public bool Matches(string scope, string check) =>
+        string.Equals(Scope, scope, StringComparison.OrdinalIgnoreCase)
+        && string.Equals(Check, check, StringComparison.OrdinalIgnoreCase);
+
+    /// <inheritdoc />
+    public override string ToString() => $"{Scope} {Check}";
+}
+
+/// <summary>One observed failure: where, which check, and the detail text.</summary>
+public sealed record GateFailure(string Scope, string Check, string Detail);
+
+/// <summary>
+/// The run's verdict once the allowlist is applied: which failures are NEW (fail the run), which
+/// are known debt (reported, tolerated), which allow entries are STALE (their check now passes —
+/// fail the run so the list shrinks), and which are unverifiable (their scope/check was skipped
+/// or absent this run — warned, never failed: a skipped Tests area proves nothing either way).
+/// </summary>
+public sealed record GateVerdict(
+    IReadOnlyList<GateFailure> NewFailures,
+    IReadOnlyList<GateFailure> KnownDebt,
+    IReadOnlyList<AllowEntry> Stale,
+    IReadOnlyList<AllowEntry> Unverifiable)
+{
+    /// <summary>Applies <paramref name="allow"/> to <paramref name="report"/>.</summary>
+    public static GateVerdict Evaluate(GateReport report, GateAllowlist allow)
+    {
+        var failures = Failures(report).ToList();
+        var newFailures = new List<GateFailure>();
+        var knownDebt = new List<GateFailure>();
+        foreach (var failure in failures)
+            (allow.Allows(failure.Scope, failure.Check) ? knownDebt : newFailures).Add(failure);
+
+        var stale = new List<AllowEntry>();
+        var unverifiable = new List<AllowEntry>();
+        foreach (var entry in allow.Entries)
+        {
+            if (failures.Any(f => entry.Matches(f.Scope, f.Check)))
+                continue;
+            (Passed(report, entry) ? stale : unverifiable).Add(entry);
+        }
+        return new GateVerdict(newFailures, knownDebt, stale, unverifiable);
+    }
+
+    /// <summary>Whether the failure at <paramref name="scope"/>/<paramref name="check"/> is
+    /// tolerated debt in this verdict. Pure.</summary>
+    public bool IsKnownDebt(string scope, string check) =>
+        KnownDebt.Any(f =>
+            string.Equals(f.Scope, scope, StringComparison.OrdinalIgnoreCase)
+            && string.Equals(f.Check, check, StringComparison.OrdinalIgnoreCase));
+
+    /// <summary>Green iff no new failure and no stale entry (and the run itself was not fatal —
+    /// the report carries that separately).</summary>
+    public bool Success => NewFailures.Count == 0 && Stale.Count == 0;
+
+    private static IEnumerable<GateFailure> Failures(GateReport report)
+    {
+        foreach (var package in report.Packages)
+        {
+            if (package.InstallError is not null)
+                yield return new GateFailure(package.Id, "install", package.InstallError);
+            if (package.IdempotenceError is not null)
+                yield return new GateFailure(package.Id, "idempotence", package.IdempotenceError);
+            foreach (var type in package.NodeTypes)
+            {
+                if (type.Compile == CheckOutcome.Failed)
+                    yield return new GateFailure(type.Path, "compile", type.CompileDetail ?? "");
+                if (type.Render == CheckOutcome.Failed)
+                    yield return new GateFailure(type.Path, "render", type.RenderDetail ?? "");
+                if (type.Tests == CheckOutcome.Failed)
+                    yield return new GateFailure(type.Path, "tests", type.TestsDetail ?? "");
+            }
+        }
+    }
+
+    // An entry is stale only when its check AFFIRMATIVELY passed this run; a skipped check or an
+    // absent scope proves nothing, so those entries are kept (and listed as unverifiable).
+    private static bool Passed(GateReport report, AllowEntry entry)
+    {
+        foreach (var package in report.Packages)
+        {
+            if (string.Equals(package.Id, entry.Scope, StringComparison.OrdinalIgnoreCase))
+                return entry.Check.ToLowerInvariant() switch
+                {
+                    "install" => package.InstallError is null,
+                    "idempotence" => package.InstallError is null && package.IdempotenceError is null,
+                    _ => false,
+                };
+            foreach (var type in package.NodeTypes)
+                if (string.Equals(type.Path, entry.Scope, StringComparison.OrdinalIgnoreCase))
+                    return entry.Check.ToLowerInvariant() switch
+                    {
+                        "compile" => type.Compile == CheckOutcome.Passed,
+                        "render" => type.Render == CheckOutcome.Passed,
+                        "tests" => type.Tests == CheckOutcome.Passed,
+                        _ => false,
+                    };
+        }
+        return false;
+    }
+}