Skip to content

test(reminders): stabilize Azure Table reminder tests #10442

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ReubenBond merged 1 commit into from
Aug 11, 2026
Merged

test(reminders): stabilize Azure Table reminder tests #10442

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 24 additions & 30 deletions test/Extensions/Orleans.Azure.Tests/Reminder/ReminderTests_AzureTable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,29 +110,26 @@ public async Task Rem_Azure_Basic()
Assert.Equal(last, curr);
}

[SkippableFact(Skip = "https://github.com/dotnet/orleans/issues/9557"), TestCategory("Functional")]
[SkippableFact, TestCategory("Functional")]
public async Task Rem_Azure_Basic_Restart()
{
IReminderTestGrain2 grain = this.GrainFactory.GetGrain<IReminderTestGrain2>(Guid.NewGuid());
TimeSpan period = await grain.GetReminderPeriod(DR);
using var cts = new CancellationTokenSource(ENDWAIT);

await grain.StartReminder(DR);
await WaitForReminderCounterAsync(grain, DR, () => grain.GetCounter(DR), 2);
long last = await grain.GetCounter(DR);
Assert.Equal(2, last);
await AdvanceRemindersByTicksAsync(2, cts.Token, (grain, DR));
await AssertReminderCountersAsync([grain], (DR, 2));

await StopReminderAndWaitForQuiescenceAsync(grain, DR, grain.StopReminder);
TimeSpan sleepFor = period.Multiply(1) + LEEWAY;
await AdvanceReminderTimeAsync(sleepFor);
long curr = await grain.GetCounter(DR);
Assert.Equal(last, curr);
AssertIsInRange(curr, last, last + 1, grain, DR, sleepFor);
await StopReminderAndWaitForQuiescenceAsync(grain, DR, grain.StopReminder, cts.Token);
await AdvanceReminderTimeAsync(period, cts.Token);
await AssertReminderCountersAsync([grain], (DR, 2));

// start the same reminder again
await grain.StartReminder(DR);
sleepFor = period.Multiply(2) + LEEWAY;
curr = await WaitForAdditionalReminderCounterAsync(grain, DR, () => grain.GetCounter(DR), 1);
AssertIsInRange(curr, 2, 3, grain, DR, sleepFor);
await StopReminderAndWaitForQuiescenceAsync(grain, DR, grain.StopReminder); // cleanup
await AdvanceRemindersByTicksAsync(2, cts.Token, (grain, DR));
await AssertReminderCountersAsync([grain], (DR, 2));
await StopReminderAndWaitForQuiescenceAsync(grain, DR, grain.StopReminder, cts.Token);
}

[SkippableFact, TestCategory("Functional")]
Expand Down Expand Up @@ -286,29 +283,26 @@ public async Task Rem_Azure_RegisterSameReminderTwice()
// TODO: write tests where period of a reminder is changed
}

[SkippableFact(Skip = "https://github.com/dotnet/orleans/issues/9557"), TestCategory("Functional")]
[SkippableFact, TestCategory("Functional")]
public async Task Rem_Azure_GT_Basic()
{
IReminderTestGrain2 g1 = this.GrainFactory.GetGrain<IReminderTestGrain2>(Guid.NewGuid());
IReminderTestCopyGrain g2 = this.GrainFactory.GetGrain<IReminderTestCopyGrain>(Guid.NewGuid());
using var cts = new CancellationTokenSource(ENDWAIT);

await g1.StartReminder(DR);
await WaitForReminderCounterAsync(g1, DR, () => g1.GetCounter(DR), 2);
await AdvanceRemindersByTicksAsync(2, cts.Token, (g1, DR));
await AssertReminderCountersAsync([g1], (DR, 2));

await g2.StartReminder(DR);
await WaitForReminderCounterAsync(g1, DR, () => g1.GetCounter(DR), 4);
await WaitForReminderCounterAsync(g2, DR, () => g2.GetCounter(DR), 2);
long last1 = await g1.GetCounter(DR);
Assert.Equal(4, last1);
long last2 = await g2.GetCounter(DR);
Assert.Equal(2, last2); // CopyGrain fault

await StopReminderAndWaitForQuiescenceAsync(g1, DR, g1.StopReminder);
await WaitForReminderCounterAsync(g2, DR, () => g2.GetCounter(DR), 4);
await StopReminderAndWaitForQuiescenceAsync(g2, DR, g2.StopReminder);
long curr1 = await g1.GetCounter(DR);
Assert.Equal(last1, curr1);
long curr2 = await g2.GetCounter(DR);
Assert.Equal(4, curr2); // CopyGrain fault
await AdvanceRemindersByTicksAsync(2, cts.Token, (g1, DR), (g2, DR));
await AssertReminderCountersAsync([g1], (DR, 4));
await AssertReminderCountersAsync([g2], (DR, 2));

await StopReminderAndWaitForQuiescenceAsync(g1, DR, g1.StopReminder, cts.Token);
await AdvanceRemindersByTicksAsync(2, cts.Token, (g2, DR));
await AssertReminderCountersAsync([g1, g2], (DR, 4));
await StopReminderAndWaitForQuiescenceAsync(g2, DR, g2.StopReminder, cts.Token);
}

[SkippableFact(Skip = "https://github.com/dotnet/orleans/issues/4319"), TestCategory("Functional")]
Expand Down
51 changes: 51 additions & 0 deletions test/Orleans.Reminders.Tests/TimerTests/ReminderTestsBase.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Microsoft.Extensions.Logging;
using Orleans.Internal;
using Orleans.Runtime;
using Orleans.Runtime.Messaging;
using Orleans.Testing.Reminders;
using Orleans.TestingHost;
using Orleans.TestingHost.Utils;
Expand Down Expand Up @@ -334,6 +335,9 @@ protected async Task Test_Reminders_MultiGrainMultiReminders(
protected async Task PrepareForGrainFailureAsync(CancellationToken cancellationToken, params IAddressable[] grains)
{
ArgumentNullException.ThrowIfNull(grains);
Assert.NotEmpty(grains);

await WaitForGrainsReachableAsync(cancellationToken, grains);

foreach (var grain in grains)
{
Expand All @@ -351,6 +355,7 @@ protected async Task CompleteGrainFailureTestAsync(CancellationToken cancellatio
ArgumentNullException.ThrowIfNull(grains);
Assert.NotEmpty(grains);

await WaitForGrainsReachableAsync(cancellationToken, grains);
await AdvanceRemindersByTicksAsync((int)(failCheckAfter - failAfter), cancellationToken, GetReminderIdentities(grains, DR));
await AssertReminderCountersAsync(grains, (DR, failCheckAfter));

Expand All @@ -359,6 +364,52 @@ protected async Task CompleteGrainFailureTestAsync(CancellationToken cancellatio
await AssertReminderCountersAsync(grains, (DR, failCheckAfter));
}

private async Task WaitForGrainsReachableAsync(CancellationToken cancellationToken, params IAddressable[] grains)
{
Exception? lastException = null;
try
{
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await Task.WhenAll(grains.Select(grain => GetReminderPeriodAsync(grain, DR))).WaitAsync(cancellationToken);
return;
}
catch (Exception exception) when (IsTransientLifecycleException(exception))
{
lastException = exception;
log.LogInformation(
exception,
"Waiting for reminder grains to become reachable after topology change: {Grains}",
string.Join(", ", grains.Select(grain => grain.GetGrainId())));
}

try
{
await WaitForLivenessToStabilizeAsync().WaitAsync(cancellationToken);
}
catch (Exception exception) when (IsTransientLifecycleException(exception))
{
lastException = exception;
}
}
}
catch (OperationCanceledException exception) when (cancellationToken.IsCancellationRequested)
{
throw new InvalidOperationException(
$"Timed out waiting for reminder grains to become reachable after a topology change: {string.Join(", ", grains.Select(grain => grain.GetGrainId()))}.",
lastException ?? exception);
}
}

private static bool IsTransientLifecycleException(Exception exception)
{
return exception is SiloUnavailableException or OrleansMessageRejectionException or ConnectionFailedException
|| exception.InnerException is not null && IsTransientLifecycleException(exception.InnerException);
}

protected async Task AdvanceRemindersByTicksAsync(
int tickCount,
CancellationToken cancellationToken,
Expand Down
Loading