Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 3 additions & 10 deletions eng/run-windows-chaos-with-dump.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ for ($attempt = 1; $attempt -le $Attempts; $attempt++) {
$info.UseShellExecute = $false
$info.RedirectStandardOutput = $true
$info.RedirectStandardError = $true
$info.Environment['SHARPLINK_CHAOS_DUMP_TOOL'] = $collector
foreach ($argument in @($dll, '--duration-seconds', '120', '--transport', 'sharedmemory', '--concurrency', '32', '--restart-interval-seconds', '10', '--json-output', $report)) {
$info.ArgumentList.Add($argument)
}
Expand All @@ -28,16 +29,8 @@ for ($attempt = 1; $attempt -le $Attempts; $attempt++) {
$failure = $null
try {
while (-not $process.WaitForExit(1000)) {
if (Test-Path $report) {
try {
$checkpoint = Get-Content $report -Raw | ConvertFrom-Json
if ($checkpoint.UnexpectedFailures -gt 0) {
$failure = "Unexpected chaos failure: $($checkpoint.TerminalFailure.Message)"
}
} catch {
# The application atomically rewrites the checkpoint; re-read on the next tick.
}
}
# The application captures unexpected failures before shutdown. This
# collector is the fallback for a process that cannot reach that path.
if ($clock.Elapsed.TotalSeconds -gt 180) {
$failure = 'Chaos exceeded its 120-second workload plus 60-second shutdown bound.'
}
Expand Down
14 changes: 6 additions & 8 deletions src/SharpLink.Runtime/RpcSession.SendPump.cs
Original file line number Diff line number Diff line change
Expand Up @@ -428,10 +428,9 @@ private async ValueTask<bool> WaitForMoreUntilFlushBoundaryAsync(
long batchStartTimestamp,
int bytesAccumulated)
{
// Queue publication and policy publication share one wake authority. A policy
// generation change is the only wake that is consumed internally: it restarts the
// decision from the original batch start. An ordinary data wake keeps the static
// pump's established behavior and returns to the outer control loop immediately.
// Queue publication and policy publication share one wake authority. Every wake
// rechecks the queue, policy, stop state, and original batch deadline: a producer
// may signal only after the pump has already consumed its published frame.
while (true)
{
if (HasProgressFrames() || HasNormalFrames())
Expand Down Expand Up @@ -470,11 +469,10 @@ private async ValueTask<bool> WaitForMoreUntilFlushBoundaryAsync(
if (!ReferenceEquals(policy, _flushPolicyState.Capture()))
continue;

// Preserve the pre-runtime static pump contract: a data wake returns to the outer
// loop. If the queue was already drained by the time it is observed, the outer
// queue check falls through to the same immediate flush behavior as before #590.
// A wake is a request to recheck state, not an independent flush boundary.
// Delayed signals for already consumed frames must not flush a timed batch.
if (woke)
return true;
continue;

if (remaining <= MaximumTimerDelay)
return false;
Expand Down
5 changes: 5 additions & 0 deletions src/SharpLink.Runtime/Transport/SharedMemoryMapping.cs
Original file line number Diff line number Diff line change
Expand Up @@ -251,6 +251,11 @@ private void Dispose()
_view.SafeMemoryMappedViewHandle.ReleasePointer();
_pointer = null;
}
// This mapping carries transient IPC data; peers observe the shared pages directly.
// Close the view handle before the accessor so its Dispose does not explicitly flush
// the backing file. On Windows that flush can block/retry for seconds and exhaust
// the server shutdown budget, even though this file is deleted when peers close.
_view.SafeMemoryMappedViewHandle.Dispose();
_view.Dispose();
_mappedFile.Dispose();
_file.Dispose();
Expand Down
51 changes: 38 additions & 13 deletions test/SharpLink.ChaosTests/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -153,10 +153,21 @@ public static async Task<int> Main(string[] args)
await restarter.ConfigureAwait(false);
await memorySampler.ConfigureAwait(false);

await AwaitDiagnosticCaptureAsync().ConfigureAwait(false);

phase = "StoppingClient";
await client.StopAsync().ConfigureAwait(false);
phase = "StoppingServer";
serverStops.Enqueue(await server.StopAsync("FinalStop").ConfigureAwait(false));
try
{
serverStops.Enqueue(await server.StopAsync("FinalStop").ConfigureAwait(false));
}
catch (Exception exception)
{
RecordUnexpectedFailure(exception);
await AwaitDiagnosticCaptureAsync().ConfigureAwait(false);
throw;
}
phase = "DrainingMetrics";
var drain = await metrics.WaitForZeroAsync(TimeSpan.FromSeconds(10)).ConfigureAwait(false);
if (options.InjectUnobservedTaskException)
Expand All @@ -176,11 +187,7 @@ public static async Task<int> Main(string[] args)
clientLogs.InjectErrorForGateProbe("client");
if (options.InjectServerError)
serverLogs.InjectErrorForGateProbe("server");
Task<ChaosDiagnosticArtifact>? activeDiagnosticCapture;
lock (diagnosticGate)
activeDiagnosticCapture = diagnosticCaptureTask;
if (activeDiagnosticCapture is not null)
diagnosticArtifact = await activeDiagnosticCapture.ConfigureAwait(false);
await AwaitDiagnosticCaptureAsync().ConfigureAwait(false);
var exitCode = 0;
ChaosFailure? terminalFailure = null;
if (!drain.Drained)
Expand Down Expand Up @@ -282,6 +289,15 @@ public static async Task<int> Main(string[] args)
Console.WriteLine($"CHAOS_SERVER_ERROR {error}");
return exitCode;

async Task AwaitDiagnosticCaptureAsync()
{
Task<ChaosDiagnosticArtifact>? activeDiagnosticCapture;
lock (diagnosticGate)
activeDiagnosticCapture = diagnosticCaptureTask;
if (activeDiagnosticCapture is not null)
diagnosticArtifact = await activeDiagnosticCapture.ConfigureAwait(false);
}

async Task RestartLoopAsync()
{
try
Expand Down Expand Up @@ -849,7 +865,10 @@ private static async Task<ChaosDiagnosticArtifact> CaptureProcessDumpAsync(strin
: Path.ChangeExtension(Path.GetFullPath(reportPath), ".dmp");
Directory.CreateDirectory(Path.GetDirectoryName(dumpPath)!);
var executableName = OperatingSystem.IsWindows() ? "createdump.exe" : "createdump";
var toolPath = Path.Combine(
var externalTool = OperatingSystem.IsWindows()
? Environment.GetEnvironmentVariable("SHARPLINK_CHAOS_DUMP_TOOL")
: null;
var toolPath = externalTool ?? Path.Combine(
System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(),
executableName);
if (!File.Exists(toolPath))
Expand All @@ -870,12 +889,18 @@ private static async Task<ChaosDiagnosticArtifact> CaptureProcessDumpAsync(strin
UseShellExecute = false,
CreateNoWindow = true
};
info.ArgumentList.Add("--withheap");
info.ArgumentList.Add("--crashreport");
info.ArgumentList.Add("--name");
info.ArgumentList.Add(dumpPath);
info.ArgumentList.Add(Environment.ProcessId.ToString(
System.Globalization.CultureInfo.InvariantCulture));
var processId = Environment.ProcessId.ToString(
System.Globalization.CultureInfo.InvariantCulture);
if (externalTool is not null)
{
foreach (var argument in new[] { "collect", "--process-id", processId, "--type", "Heap", "--output", dumpPath })
info.ArgumentList.Add(argument);
}
else
{
foreach (var argument in new[] { "--withheap", "--crashreport", "--name", dumpPath, processId })
info.ArgumentList.Add(argument);
}
using var process = Process.Start(info);
if (process is null)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ public async Task RuntimeAssembliesShouldRegisterTransactionallyAndSupportEveryC
Ensure(duplicate.Error?.IncomingLoadContext?.Contains("dynamic-call-shapes", StringComparison.Ordinal) == true,
"duplicate diagnostics contain ALC identity");

await WaitForRemoteContractManifestAsync(harness.Client, plugin.ContractType);
object? proxy = GetProxy(harness.Client, plugin.ContractType);
var unary = await InvokeValueTaskAsync<int>(proxy, plugin.ContractType, "UnaryAsync", 7, CancellationToken.None);
Ensure(unary == 8, "dynamic unary");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,10 @@ public async Task ConcurrentPublicationMustNotResurrectClientAfterStop()
{
const int attempts = 100_000;
using var start = new Barrier(3);
using var workersCancellation = new CancellationTokenSource(TimeSpan.FromSeconds(30));
// Keep all 100,000 race rounds. Hosted macOS runners can spend over 30 seconds
// scheduling these barriers alongside the full suite; this bounds test workers,
// not any production lifecycle operation.
using var workersCancellation = new CancellationTokenSource(TimeSpan.FromMinutes(2));
var accessor = new SharpLinkClientAccessor();
var client = new FakeSharpLinkClient();
Exception? publicationFailure = null;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,11 +1,64 @@
using System.Diagnostics;
using System.IO.Pipelines;
using System.Reflection;

namespace SharpLink.UnitTests.Runtime;

[NotInParallel]
public sealed class RpcSessionRuntimeFlushPolicyTests
{
[Test]
[Arguments(false)]
[Arguments(true)]
public async Task TimedBatchShouldIgnoreDelayedDataWakeAfterQueueWasDrained(bool publishRuntimePolicy)
{
var clock = new ManualTimeProvider();
var provider = new TimerCountingTimeProvider(clock);
var initial = new RpcSessionFlushOptions(1024, TimeSpan.FromSeconds(30));
using var context = new SharpLinkRuntimeContextBuilder()
.UseTimeProvider(provider)
.Build(includeGeneratedAssemblyCatalog: false);
var owner = CompressionSendPolicyState.CreateInitial(new SharpLinkCompressionSendPolicy());
var policy = owner.GetOrCreateSessionFlushPolicyState(initial, context.PerformanceProfile);
if (publishRuntimePolicy)
Ensure(policy.Publish(4096, TimeSpan.FromSeconds(30)), "runtime threshold publication");
var input = new Pipe();
var output = new Pipe();
var session = CreateSession("delayed-data-wake", context, owner, initial, input, output);
try
{
var readTask = output.Reader.ReadAsync().AsTask();
session.SendPacket(CreateFrame(session, 64, 1));
await WaitUntilAsync(() => provider.TimerCount > 0);
var armedCount = provider.TimerCount;
var pump = typeof(RpcSession).GetField("_pump", BindingFlags.Instance | BindingFlags.NonPublic)!
.GetValue(session)!;
var wakeup = (WakeupSignal)pump.GetType()
.GetField("_wakeup", BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(pump)!;

// Model an enqueuer delayed between queue publication and Signal: the
// pump already consumed its frame through a previous wake.
wakeup.Signal();
await WaitUntilAsync(() => readTask.IsCompleted || provider.TimerCount > armedCount);
Ensure(!readTask.IsCompleted && session.QueuedSendBytes > 0,
"a delayed data wake must preserve the timed batch and retained frame");

clock.Advance(TimeSpan.FromSeconds(30));
var read = await readTask.WaitAsync(TimeSpan.FromSeconds(2));
Ensure(read.Buffer.Length == ProtocolV2Constants.HeaderBytes + 64,
"the original deadline must publish the complete retained frame");
output.Reader.AdvanceTo(read.Buffer.End);
await session.FlushSendQueueAsync();
Ensure(session.QueuedSendBytes == 0, "the deadline flush must release queued byte ownership");
}
finally
{
await session.DisposeAsync();
await output.Reader.CompleteAsync();
await input.Writer.CompleteAsync();
}
}

[Test]
public async Task ThresholdDecreaseShouldWakeArmedExistingSessionAndFutureSessionShouldShareGeneration()
{
Expand Down Expand Up @@ -176,6 +229,20 @@ public async Task LatencyIncreaseShouldIgnoreOldTimerBoundary()
}
}

private sealed class TimerCountingTimeProvider(ManualTimeProvider clock) : TimeProvider
{
private int _timerCount;
internal int TimerCount => Volatile.Read(ref _timerCount);
public override long TimestampFrequency => clock.TimestampFrequency;
public override long GetTimestamp() => clock.GetTimestamp();
public override ITimer CreateTimer(TimerCallback callback, object? state, TimeSpan dueTime, TimeSpan period)
{
var timer = clock.CreateTimer(callback, state, dueTime, period);
Interlocked.Increment(ref _timerCount);
return timer;
}
}

private static RpcSession CreateSession(
string id,
SharpLinkRuntimeContext context,
Expand Down
41 changes: 41 additions & 0 deletions test/SharpLink.UnitTests/Runtime/SharedMemoryLayoutTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,47 @@ public async Task MappingFileShouldDisappearAfterBothSidesClose()
}
}

[Test]
[Arguments(true)]
[Arguments(false)]
[NotInParallel]
public async Task DisposingOneMappingShouldPreservePeerMemoryAndInvalidateOwnedMemory(bool closeServerFirst)
{
const int capacity = 64 * 1024;
var baseline = SharedMemoryMapping.ActiveMappingCount;
var nonce = RandomNumberGenerator.GetBytes(SharedMemoryLayout.NonceBytes);
var server = SharedMemoryMapping.CreateServer(capacity, nonce, out var path);
var client = SharedMemoryMapping.OpenClient(path, capacity, nonce);
try
{
server.UnlinkAfterClientOpened();
var owner = closeServerFirst ? server : client;
var peer = closeServerFirst ? client : server;
var ownedMemory = owner.Memory.Slice(SharedMemoryLayout.HeaderBytes, 64);
var peerMemory = peer.Memory.Slice(SharedMemoryLayout.HeaderBytes, 64);
var expected = Enumerable.Range(1, 64).Select(value => (byte)value).ToArray();
expected.CopyTo(ownedMemory);

await owner.DisposeAsync();

await Assert.That(peerMemory.ToArray().SequenceEqual(expected)).IsTrue();
peerMemory.Span[0] = 123;
await Assert.That(peerMemory.Span[0]).IsEqualTo((byte)123);
await Assert.That(() => ownedMemory.Span[0]).Throws<ObjectDisposedException>();
await Assert.That(SharedMemoryMapping.ActiveMappingCount).IsEqualTo(baseline + 1);

await peer.DisposeAsync();
await owner.DisposeAsync();
await Assert.That(SharedMemoryMapping.ActiveMappingCount).IsEqualTo(baseline);
await Assert.That(File.Exists(path)).IsFalse();
}
finally
{
await client.DisposeAsync();
await server.DisposeAsync();
}
}

[Test]
public async Task MappingPathShouldRejectLocationsOutsideTransportDirectory()
{
Expand Down
Loading