Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
49 changes: 46 additions & 3 deletions src/MeshWeaver.Hosting/DynamicTypePreWarmer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -801,9 +801,27 @@ or CompilationStatus.Unavailable
{
if (node?.Content is not NodeTypeDefinition def)
return node!;
// Don't clobber an in-flight compile someone else already started.
if (def.CompilationStatus is CompilationStatus.Pending
or CompilationStatus.Compiling)
// Don't clobber an in-flight compile someone else already started…
// …but ONLY while it can still be in flight (#1462).
//
// 🚨 `Compiling` is a non-terminal state and nothing else reconciles
// it. The flip to `Compiling` is DURABLE; the terminal write is not
// guaranteed — a process death mid-compile leaves the row there for
// good. This guard then declines to touch it on every subsequent
// sweep, so the type never reaches a terminal state: it is neither
// `Ok` (usable) nor `Error` (classifiable), it holds portal
// readiness, and it parks every instance hub for the full activation
// budget. One row on `public.mesh_nodes` sat at `Compiling` for TEN
// WEEKS this way; it was harmless only because it was an orphan the
// prewarmer never enumerates.
//
// A claim older than the per-type budget cannot still be in flight —
// whoever made it would have written a terminal state or been given
// up on long ago — so it is re-driven rather than deferred to.
// Note what this is NOT: no timer, no poller, no background sweep for
// stale rows. The recovery rides the enumeration that already runs,
// and only ever reinterprets a claim that has provably expired.
if (IsLiveCompileClaim(def, budget))
return node;
return node with
{
Expand Down Expand Up @@ -1038,6 +1056,31 @@ public static IDisposable WatchForRecovery(
typePath));
}

/// <summary>
/// Whether <paramref name="def"/>'s compile claim can still be in flight, i.e. whether deferring
/// to it is honouring a live compile rather than a stranded one (#1462).
///
/// <para><c>Pending</c> is always honoured: it is the state this prewarmer itself writes to ASK
/// for a compile, and a driver picks it up promptly.</para>
///
/// <para><c>Compiling</c> is honoured only while <see cref="NodeTypeDefinition.LastCompileStartedAt"/>
/// is within <paramref name="budget"/> — the same bound the sweep gives a type to settle. Past it,
/// no driver is still working on it: either it finished (and would have written a terminal state)
/// or it died. A row with NO start timestamp at all is treated as stranded too, since a live
/// compile always stamps one (<c>NodeTypeCompilationHelpers</c>); an unstamped <c>Compiling</c> is
/// exactly the shape a row left over from an older write carries, and honouring it forever is how
/// this became permanent.</para>
/// </summary>
internal static bool IsLiveCompileClaim(NodeTypeDefinition def, TimeSpan budget) =>
def.CompilationStatus switch
{
CompilationStatus.Pending => true,
CompilationStatus.Compiling =>
def.LastCompileStartedAt is { } startedAt
&& DateTimeOffset.UtcNow - startedAt <= budget,
_ => false,
};

/// <summary>
/// Activate one dynamic NodeType's hub by subscribing to its own MeshNode stream —
/// which routes a SubscribeRequest to the owning hub, activating it and firing the
Expand Down
81 changes: 81 additions & 0 deletions test/MeshWeaver.Hosting.Test/StrandedCompileClaimTest.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
using System;
using MeshWeaver.Graph.Configuration;
using MeshWeaver.Mesh.Services;
using Xunit;
Comment thread
rbuergi marked this conversation as resolved.

namespace MeshWeaver.Hosting.Test;

/// <summary>
/// Issue #1462 — <c>Compiling</c> is a NON-TERMINAL state that nothing reconciles.
///
/// <para>The flip to <c>Compiling</c> is durable; the terminal write is not guaranteed. A process
/// death mid-compile therefore leaves the row there permanently, and the prewarmer's
/// "don't clobber an in-flight compile" guard then declines to touch it on every subsequent sweep.
/// The type never reaches a terminal state: neither <c>Ok</c> (usable) nor <c>Error</c>
/// (classifiable), it holds portal readiness, and it parks every instance hub for the full
/// activation budget. One row on <c>public.mesh_nodes</c> sat at <c>Compiling</c> for TEN WEEKS —
/// harmless only because it was an orphan the prewarmer never enumerates.</para>
///
/// <para>🚨 What the fix is NOT: a timer, a poller, or a background sweep for stale rows — the issue
/// rules those out explicitly, and so does AGENTS.md. The recovery rides the enumeration that
/// ALREADY runs, and only ever reinterprets a claim that has provably expired. These cases pin that
/// distinction, which is the whole substance of the change.</para>
/// </summary>
public class StrandedCompileClaimTest
{
private static readonly TimeSpan Budget = TimeSpan.FromMinutes(5);

private static NodeTypeDefinition Def(CompilationStatus status, DateTimeOffset? startedAt = null) =>
new() { Configuration = "config => config", CompilationStatus = status, LastCompileStartedAt = startedAt };

[Fact]
public void APendingClaimIsAlwaysHonoured()
{
// Pending is what the prewarmer itself writes to ASK for a compile, and a driver picks it up
// promptly — re-driving it would fight our own request.
DynamicTypePreWarmer.IsLiveCompileClaim(Def(CompilationStatus.Pending), Budget)
.Should().BeTrue("Pending is this prewarmer's own request, not a stranded claim");
}

[Fact]
public void ACompilingClaimInsideTheBudgetIsHonoured()
{
var justStarted = Def(CompilationStatus.Compiling, DateTimeOffset.UtcNow.AddSeconds(-5));

DynamicTypePreWarmer.IsLiveCompileClaim(justStarted, Budget)
.Should().BeTrue("a compile that started 5 s ago can still be in flight — clobbering it "
+ "would restart work that is about to finish");
}

[Fact]
public void ACompilingClaimOlderThanTheBudgetIsStranded()
{
var stale = Def(CompilationStatus.Compiling, DateTimeOffset.UtcNow - Budget - TimeSpan.FromMinutes(1));

DynamicTypePreWarmer.IsLiveCompileClaim(stale, Budget)
.Should().BeFalse("past the budget no driver is still working on it — it either finished "
+ "(and would have written a terminal state) or it died, and deferring "
+ "to it forever is what made this permanent");
}

[Fact]
public void ACompilingClaimWithNoStartTimestampIsStranded()
{
// A live compile always stamps LastCompileStartedAt. An unstamped Compiling is exactly the
// shape a row left over from an older write carries — the ten-week row's shape.
DynamicTypePreWarmer.IsLiveCompileClaim(Def(CompilationStatus.Compiling), Budget)
.Should().BeFalse("an unstamped Compiling cannot be shown to be in flight, and honouring "
+ "it forever is precisely the defect");
}

[Theory]
[InlineData(CompilationStatus.Ok)]
[InlineData(CompilationStatus.Error)]
[InlineData(CompilationStatus.Unavailable)]
[InlineData(CompilationStatus.Unknown)]
public void ATerminalOrUnclaimedStatusIsNotAClaim(CompilationStatus status)
{
DynamicTypePreWarmer.IsLiveCompileClaim(Def(status), Budget)
.Should().BeFalse("only Pending and a fresh Compiling are claims to defer to");
}
}
Loading