diff --git a/test/MeshWeaver.PluginTester.Test/GateAllowlistTest.cs b/test/MeshWeaver.PluginTester.Test/GateAllowlistTest.cs new file mode 100644 index 0000000000..db42f206d2 --- /dev/null +++ b/test/MeshWeaver.PluginTester.Test/GateAllowlistTest.cs @@ -0,0 +1,204 @@ +#pragma warning disable CS1591 + +using System; +using System.IO; +using System.Linq; +using MeshWeaver.PluginTester; +using Xunit; + +namespace MeshWeaver.PluginTester.Test; + +/// +/// The known-debt ratchet, pinned check by check: parsing (including the malformed-line throw — +/// a typo that silently allowed nothing would defeat the ratchet), the classification of every +/// failure into new-vs-known, the STALE rule (a passing check fails the run until its entry is +/// removed — the list only shrinks), and the unverifiable escape (a skipped check or absent +/// scope proves nothing, so it warns and never fails). +/// +public class GateAllowlistTest +{ + // ── parsing ────────────────────────────────────────────────────────────────────────────── + + [Fact] + public void Parse_ReadsEntries_SkipsCommentsAndBlanks() + { + var allow = GateAllowlist.Parse( + [ + "# the debt list", + "", + "Claims idempotence # tracked in #235", + "Edu/Exercise tests", + ]); + Assert.Equal(2, allow.Entries.Count); + Assert.True(allow.Allows("Claims", "idempotence")); + Assert.True(allow.Allows("Edu/Exercise", "tests")); + Assert.False(allow.Allows("Claims", "install")); + } + + [Fact] + public void Parse_MatchesCaseInsensitively() + { + var allow = GateAllowlist.Parse(["claims IDEMPOTENCE"]); + Assert.True(allow.Allows("Claims", "idempotence")); + } + + [Theory] + [InlineData("Claims")] // missing check + [InlineData("Claims idempotence extra")] // too many tokens + [InlineData("Claims flakiness")] // unknown check name + public void Parse_ThrowsOnMalformedLine(string line) + { + var ex = Assert.Throws(() => GateAllowlist.Parse([line])); + Assert.Contains("line 1", ex.Message); + } + + // ── classification ─────────────────────────────────────────────────────────────────────── + + private static GateReport Report(params PackageResult[] packages) => new([.. packages]); + + private static PackageResult IdempotenceBroken(string id) => + new(id) { IdempotenceError = "re-install wrote 1 node(s)" }; + + private static PackageResult WithType(string id, NodeTypeResult type) => + new(id) { NodeTypes = [type] }; + + private static NodeTypeResult TestsRed(string path) => + new(path, path.Split('/')[0]) { Tests = CheckOutcome.Failed, TestsDetail = "1 red row" }; + + [Fact] + public void Evaluate_SplitsFailures_IntoKnownDebtAndNew() + { + var report = Report(IdempotenceBroken("Claims"), IdempotenceBroken("Video")); + var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"])); + + Assert.Single(verdict.KnownDebt); + Assert.True(verdict.IsKnownDebt("Claims", "idempotence")); + Assert.Single(verdict.NewFailures); + Assert.Equal("Video", verdict.NewFailures[0].Scope); + Assert.False(verdict.Success); + } + + [Fact] + public void Evaluate_AllKnown_IsGreen() + { + var report = Report( + IdempotenceBroken("Claims"), + WithType("Edu", TestsRed("Edu/Exercise"))); + var verdict = GateVerdict.Evaluate( + report, GateAllowlist.Parse(["Claims idempotence", "Edu/Exercise tests"])); + + Assert.Empty(verdict.NewFailures); + Assert.Empty(verdict.Stale); + Assert.Equal(2, verdict.KnownDebt.Count); + Assert.True(verdict.Success); + } + + [Fact] + public void Evaluate_TypeChecks_MatchOnTheTypePath() + { + var report = Report(WithType("Edu", TestsRed("Edu/Exercise"))); + var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Edu tests"])); + + // The package-scoped entry does NOT cover a type-level failure: exact scope only, + // or the ratchet stops ratcheting. + Assert.Single(verdict.NewFailures); + Assert.False(verdict.Success); + } + + // ── the stale rule ─────────────────────────────────────────────────────────────────────── + + [Fact] + public void Evaluate_EntryWhoseCheckPasses_IsStale_AndFailsTheRun() + { + var healthy = new PackageResult("Claims"); + var verdict = GateVerdict.Evaluate( + Report(healthy), GateAllowlist.Parse(["Claims idempotence"])); + + Assert.Single(verdict.Stale); + Assert.Empty(verdict.NewFailures); + Assert.False(verdict.Success); + } + + [Fact] + public void Evaluate_TypeEntryWhoseTestsPass_IsStale() + { + var green = new NodeTypeResult("Edu/Exercise", "Edu") { Tests = CheckOutcome.Passed }; + var verdict = GateVerdict.Evaluate( + Report(WithType("Edu", green)), GateAllowlist.Parse(["Edu/Exercise tests"])); + + Assert.Single(verdict.Stale); + Assert.False(verdict.Success); + } + + // ── the unverifiable escape ────────────────────────────────────────────────────────────── + + [Fact] + public void Evaluate_AbsentScope_IsUnverifiable_NotStale() + { + var verdict = GateVerdict.Evaluate( + Report(new PackageResult("Claims")), GateAllowlist.Parse(["Gone/Type tests"])); + + Assert.Single(verdict.Unverifiable); + Assert.Empty(verdict.Stale); + Assert.True(verdict.Success); + } + + [Fact] + public void Evaluate_SkippedTests_AreUnverifiable_NotStale() + { + var skipped = new NodeTypeResult("Edu/Exercise", "Edu") { Tests = CheckOutcome.Skipped }; + var verdict = GateVerdict.Evaluate( + Report(WithType("Edu", skipped)), GateAllowlist.Parse(["Edu/Exercise tests"])); + + Assert.Single(verdict.Unverifiable); + Assert.Empty(verdict.Stale); + Assert.True(verdict.Success); + } + + [Fact] + public void Evaluate_IdempotenceEntry_BehindAFailedInstall_IsUnverifiable() + { + // The idempotence pin never ran if the install itself failed — its entry must not be + // stale (that would demand removing an entry for debt that may well still exist). + var broken = new PackageResult("Claims") { InstallError = "boom" }; + var verdict = GateVerdict.Evaluate( + Report(broken), GateAllowlist.Parse(["Claims idempotence"])); + + Assert.Single(verdict.Unverifiable); + Assert.Empty(verdict.Stale); + // The install failure itself is unlisted → a new failure → the run still fails. + Assert.Single(verdict.NewFailures); + Assert.False(verdict.Success); + } + + // ── the summary states ONE verdict ─────────────────────────────────────────────────────── + + [Fact] + public void Summary_LabelsKnownDebt_AndStatesTheVerdictLine() + { + var report = Report(IdempotenceBroken("Claims")); + var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"])); + var output = new StringWriter(); + report.WriteSummary(output, verdict); + var text = output.ToString(); + + Assert.Contains("[DEBT] Claims", text); + Assert.Contains("[known-debt]", text); + Assert.Contains("GREEN — 1 known-debt failure(s) allowed", text); + Assert.DoesNotContain("GATE FAILED", text); + } + + [Fact] + public void Summary_NamesStaleEntries_AndFails() + { + var report = Report(new PackageResult("Claims")); + var verdict = GateVerdict.Evaluate(report, GateAllowlist.Parse(["Claims idempotence"])); + var output = new StringWriter(); + report.WriteSummary(output, verdict); + var text = output.ToString(); + + Assert.Contains("STALE allow entry", text); + Assert.Contains("Claims idempotence", text); + Assert.Contains("GATE FAILED", text); + } +} diff --git a/tools/MeshWeaver.PluginTester/GateAllowlist.cs b/tools/MeshWeaver.PluginTester/GateAllowlist.cs new file mode 100644 index 0000000000..6626190ced --- /dev/null +++ b/tools/MeshWeaver.PluginTester/GateAllowlist.cs @@ -0,0 +1,164 @@ +namespace MeshWeaver.PluginTester; + +/// +/// The gate's known-debt ratchet — the same one-way contract as the plugins repo's +/// compile-check.allow: a failure listed here is reported as debt but does not fail the +/// run; a NEW failure (not listed) fails; and an entry whose check now PASSES is STALE and fails +/// the run, so the list can only shrink. Without this, arming the gate on a repo with +/// pre-existing debt means either merging red or silencing the whole job — both worse than a +/// ratchet. +/// +/// File format: one entry per line, # comments, blank lines ignored: +/// +/// <scope> <check> +/// Claims idempotence # package-level checks: install, idempotence +/// Edu/Exercise tests # type-level checks: compile, render, tests +/// +/// +public sealed record GateAllowlist(IReadOnlyList Entries) +{ + /// The empty list — every failure is a new failure. + public static readonly GateAllowlist Empty = new([]); + + /// The valid check names, package-level and type-level. + public static readonly IReadOnlySet Checks = + new HashSet(StringComparer.OrdinalIgnoreCase) + { "install", "idempotence", "compile", "render", "tests" }; + + /// Parses the allow file, throwing on a malformed line — a typo that silently + /// allowed nothing (or everything) would defeat the ratchet. + public static GateAllowlist Parse(IEnumerable lines) + { + var entries = new List(); + var number = 0; + foreach (var raw in lines) + { + number++; + var line = StripComment(raw).Trim(); + if (line.Length == 0) + continue; + var parts = line.Split((char[]?)null, StringSplitOptions.RemoveEmptyEntries); + if (parts.Length != 2 || !Checks.Contains(parts[1])) + throw new FormatException( + $"allow file line {number}: expected ' ' with check one of " + + $"[{string.Join(", ", Checks)}], got '{line}'"); + entries.Add(new AllowEntry(parts[0], parts[1].ToLowerInvariant())); + } + return new GateAllowlist(entries); + } + + /// Loads and parses the allow file at . + public static GateAllowlist Load(string path) => Parse(File.ReadLines(path)); + + private static string StripComment(string line) => + line.IndexOf('#') is var i && i >= 0 ? line[..i] : line; + + /// Whether a failure at / is known debt. + public bool Allows(string scope, string check) => Entries.Any(e => e.Matches(scope, check)); +} + +/// One known-debt entry: a scope (package id or NodeType path) and a check name. +public sealed record AllowEntry(string Scope, string Check) +{ + /// Case-insensitive match, exact scope — no globs: an entry must name exactly the + /// debt it tolerates, or the ratchet stops ratcheting. + public bool Matches(string scope, string check) => + string.Equals(Scope, scope, StringComparison.OrdinalIgnoreCase) + && string.Equals(Check, check, StringComparison.OrdinalIgnoreCase); + + /// + public override string ToString() => $"{Scope} {Check}"; +} + +/// One observed failure: where, which check, and the detail text. +public sealed record GateFailure(string Scope, string Check, string Detail); + +/// +/// The run's verdict once the allowlist is applied: which failures are NEW (fail the run), which +/// are known debt (reported, tolerated), which allow entries are STALE (their check now passes — +/// fail the run so the list shrinks), and which are unverifiable (their scope/check was skipped +/// or absent this run — warned, never failed: a skipped Tests area proves nothing either way). +/// +public sealed record GateVerdict( + IReadOnlyList NewFailures, + IReadOnlyList KnownDebt, + IReadOnlyList Stale, + IReadOnlyList Unverifiable) +{ + /// Applies to . + public static GateVerdict Evaluate(GateReport report, GateAllowlist allow) + { + var failures = Failures(report).ToList(); + var newFailures = new List(); + var knownDebt = new List(); + foreach (var failure in failures) + (allow.Allows(failure.Scope, failure.Check) ? knownDebt : newFailures).Add(failure); + + var stale = new List(); + var unverifiable = new List(); + foreach (var entry in allow.Entries) + { + if (failures.Any(f => entry.Matches(f.Scope, f.Check))) + continue; + (Passed(report, entry) ? stale : unverifiable).Add(entry); + } + return new GateVerdict(newFailures, knownDebt, stale, unverifiable); + } + + /// Whether the failure at / is + /// tolerated debt in this verdict. Pure. + public bool IsKnownDebt(string scope, string check) => + KnownDebt.Any(f => + string.Equals(f.Scope, scope, StringComparison.OrdinalIgnoreCase) + && string.Equals(f.Check, check, StringComparison.OrdinalIgnoreCase)); + + /// Green iff no new failure and no stale entry (and the run itself was not fatal — + /// the report carries that separately). + public bool Success => NewFailures.Count == 0 && Stale.Count == 0; + + private static IEnumerable Failures(GateReport report) + { + foreach (var package in report.Packages) + { + if (package.InstallError is not null) + yield return new GateFailure(package.Id, "install", package.InstallError); + if (package.IdempotenceError is not null) + yield return new GateFailure(package.Id, "idempotence", package.IdempotenceError); + foreach (var type in package.NodeTypes) + { + if (type.Compile == CheckOutcome.Failed) + yield return new GateFailure(type.Path, "compile", type.CompileDetail ?? ""); + if (type.Render == CheckOutcome.Failed) + yield return new GateFailure(type.Path, "render", type.RenderDetail ?? ""); + if (type.Tests == CheckOutcome.Failed) + yield return new GateFailure(type.Path, "tests", type.TestsDetail ?? ""); + } + } + } + + // An entry is stale only when its check AFFIRMATIVELY passed this run; a skipped check or an + // absent scope proves nothing, so those entries are kept (and listed as unverifiable). + private static bool Passed(GateReport report, AllowEntry entry) + { + foreach (var package in report.Packages) + { + if (string.Equals(package.Id, entry.Scope, StringComparison.OrdinalIgnoreCase)) + return entry.Check.ToLowerInvariant() switch + { + "install" => package.InstallError is null, + "idempotence" => package.InstallError is null && package.IdempotenceError is null, + _ => false, + }; + foreach (var type in package.NodeTypes) + if (string.Equals(type.Path, entry.Scope, StringComparison.OrdinalIgnoreCase)) + return entry.Check.ToLowerInvariant() switch + { + "compile" => type.Compile == CheckOutcome.Passed, + "render" => type.Render == CheckOutcome.Passed, + "tests" => type.Tests == CheckOutcome.Passed, + _ => false, + }; + } + return false; + } +} diff --git a/tools/MeshWeaver.PluginTester/GateReport.cs b/tools/MeshWeaver.PluginTester/GateReport.cs index c4945bd091..aab1766fb4 100644 --- a/tools/MeshWeaver.PluginTester/GateReport.cs +++ b/tools/MeshWeaver.PluginTester/GateReport.cs @@ -85,8 +85,14 @@ public sealed record GateReport(IReadOnlyList Packages) /// Process exit code: 0 = all green. public int ExitCode => Success ? 0 : 1; - /// Writes the human-readable per-package summary table. - public void WriteSummary(TextWriter output) + /// + /// Writes the human-readable per-package summary table. With a + /// (an allowlist was applied), a package or check whose only failures are known debt is + /// labeled DEBT rather than FAIL, stale allow entries are listed by name (they fail the + /// run — the list must shrink), and the final line states the verdict, never the raw + /// pass/fail — two bottom lines disagreeing on "green" is how a gate gets ignored. + /// + public void WriteSummary(TextWriter output, GateVerdict? verdict = null) { output.WriteLine(); output.WriteLine("=== mw-plugin-test summary ==="); @@ -94,18 +100,19 @@ public void WriteSummary(TextWriter output) output.WriteLine($"FATAL: {FatalError}"); foreach (var package in Packages) { - output.WriteLine($"[{(package.Success ? "PASS" : "FAIL")}] {package.Id} " + + output.WriteLine($"[{Label(package, verdict)}] {package.Id} " + $"({package.NodeCount} node(s), {package.NodeTypes.Count} type(s))"); if (package.InstallError is not null) - output.WriteLine($" install: {package.InstallError}"); + output.WriteLine($" install{Debt(verdict, package.Id, "install")}: {package.InstallError}"); if (package.IdempotenceError is not null) - output.WriteLine($" idempotence: {package.IdempotenceError}"); + output.WriteLine($" idempotence{Debt(verdict, package.Id, "idempotence")}: {package.IdempotenceError}"); foreach (var type in package.NodeTypes) { output.WriteLine( $" {(type.Success ? "ok " : "RED")} {type.Path}: " + - $"compile={Describe(type.Compile, type.CompilationStatus?.ToString())} " + - $"render={Describe(type.Render)} tests={Describe(type.Tests)}"); + $"compile={Describe(type.Compile, type.CompilationStatus?.ToString())}{Debt(verdict, type.Path, "compile")} " + + $"render={Describe(type.Render)}{Debt(verdict, type.Path, "render")} " + + $"tests={Describe(type.Tests)}{Debt(verdict, type.Path, "tests")}"); if (type.CompileDetail is not null) output.WriteLine(Indent(type.CompileDetail)); if (type.RenderDetail is not null) @@ -114,9 +121,42 @@ public void WriteSummary(TextWriter output) output.WriteLine(Indent(type.TestsDetail)); } } - output.WriteLine(Success ? "ALL GREEN." : "GATE FAILED."); + if (verdict is null) + { + output.WriteLine(Success ? "ALL GREEN." : "GATE FAILED."); + return; + } + foreach (var entry in verdict.Stale) + output.WriteLine($"STALE allow entry (now passing — remove it): {entry}"); + foreach (var entry in verdict.Unverifiable) + output.WriteLine($"unverifiable allow entry (check skipped or scope absent this run): {entry}"); + var green = FatalError is null && verdict.Success; + output.WriteLine(green + ? verdict.KnownDebt.Count == 0 + ? "ALL GREEN." + : $"GREEN — {verdict.KnownDebt.Count} known-debt failure(s) allowed (shrinking list)." + : $"GATE FAILED — {verdict.NewFailures.Count} new failure(s), {verdict.Stale.Count} stale allow entr(ies)."); } + private static string Label(PackageResult package, GateVerdict? verdict) + { + if (package.Success) + return "PASS"; + if (verdict is null) + return "FAIL"; + var allKnown = + (package.InstallError is null || verdict.IsKnownDebt(package.Id, "install")) + && (package.IdempotenceError is null || verdict.IsKnownDebt(package.Id, "idempotence")) + && package.NodeTypes.All(t => + (t.Compile != CheckOutcome.Failed || verdict.IsKnownDebt(t.Path, "compile")) + && (t.Render != CheckOutcome.Failed || verdict.IsKnownDebt(t.Path, "render")) + && (t.Tests != CheckOutcome.Failed || verdict.IsKnownDebt(t.Path, "tests"))); + return allKnown ? "DEBT" : "FAIL"; + } + + private static string Debt(GateVerdict? verdict, string scope, string check) => + verdict is not null && verdict.IsKnownDebt(scope, check) ? " [known-debt]" : ""; + private static string Describe(CheckOutcome outcome, string? detail = null) => outcome switch { diff --git a/tools/MeshWeaver.PluginTester/Program.cs b/tools/MeshWeaver.PluginTester/Program.cs index 8693c5d079..121fb3cb14 100644 --- a/tools/MeshWeaver.PluginTester/Program.cs +++ b/tools/MeshWeaver.PluginTester/Program.cs @@ -4,16 +4,21 @@ using MeshWeaver.PluginTester; // mw-plugin-test [--compile-timeout ] [--render-timeout ] +// [--allow ] // // The MeshWeaver.Plugins PR gate: imports each node-repo package of the checkout into a fresh // in-process mesh, waits for every NodeType to compile (Roslyn diagnostics on error), renders // each type's default area, and EXECUTES each type's `Tests` layout area. Exit 0 = all green. +// --allow names a known-debt file (the compile-check.allow ratchet): listed failures are +// tolerated, new failures fail, and an entry whose check now passes is stale and fails. // // The one Task bridge lives HERE, at the console boundary — everything below Run() is reactive. string? root = null; var compileTimeout = TimeSpan.FromMinutes(5); var renderTimeout = TimeSpan.FromMinutes(2); +var allowlist = GateAllowlist.Empty; +var allowApplied = false; for (var i = 0; i < args.Length; i++) { @@ -27,9 +32,19 @@ renderTimeout = TimeSpan.FromSeconds( double.Parse(args[++i], CultureInfo.InvariantCulture)); break; + case "--allow" when i + 1 < args.Length: + allowlist = GateAllowlist.Load(args[++i]); + allowApplied = true; + break; + // A value-taking option as the LAST argument would otherwise fall through to the default + // case as "Unknown argument" — a misleading message for a missing value. + case "--compile-timeout" or "--render-timeout" or "--allow": + Console.Error.WriteLine($"Option '{args[i]}' requires a value. Try --help."); + return 2; case "--help" or "-h": Console.WriteLine( - "usage: mw-plugin-test [--compile-timeout ] [--render-timeout ]"); + "usage: mw-plugin-test [--compile-timeout ] [--render-timeout ] " + + "[--allow ]"); return 0; default: if (args[i].StartsWith('-') || root is not null) @@ -50,6 +65,14 @@ }; Console.WriteLine($"mw-plugin-test: gating node repos under '{Path.GetFullPath(options.RepoRoot)}'"); +if (allowApplied) + Console.WriteLine($"known-debt allowlist: {allowlist.Entries.Count} entr(ies)"); var report = await PluginGateRunner.Run(options).FirstAsync().ToTask(); -report.WriteSummary(Console.Out); -return report.ExitCode; +if (!allowApplied) +{ + report.WriteSummary(Console.Out); + return report.ExitCode; +} +var verdict = GateVerdict.Evaluate(report, allowlist); +report.WriteSummary(Console.Out, verdict); +return report.FatalError is null && verdict.Success ? 0 : 1;