Skip to content

Commit bc29489

Browse files
authored
fix: admit and drain descendants of invocation-owned tasks
1 parent a9739d5 commit bc29489

3 files changed

Lines changed: 279 additions & 42 deletions

File tree

‎sdk-integration-tests/src/test/java/software/amazon/lambda/durable/execution/InvocationDrainingIntegrationTest.java‎

Lines changed: 94 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
package software.amazon.lambda.durable.execution;
44

55
import static org.junit.jupiter.api.Assertions.assertEquals;
6+
import static org.junit.jupiter.api.Assertions.assertFalse;
67
import static org.junit.jupiter.api.Assertions.assertTrue;
78

89
import java.util.List;
@@ -11,8 +12,10 @@
1112
import java.util.concurrent.ExecutorService;
1213
import java.util.concurrent.Executors;
1314
import java.util.concurrent.TimeUnit;
15+
import java.util.concurrent.atomic.AtomicInteger;
1416
import java.util.concurrent.atomic.AtomicReference;
15-
import java.util.function.Function;
17+
import java.util.function.BiFunction;
18+
import org.junit.jupiter.api.Test;
1619
import org.junit.jupiter.params.ParameterizedTest;
1720
import org.junit.jupiter.params.provider.CsvSource;
1821
import software.amazon.awssdk.services.lambda.model.OperationStatus;
@@ -28,8 +31,12 @@
2831

2932
class InvocationDrainingIntegrationTest {
3033
@ParameterizedTest
31-
@CsvSource({"false, FLAT", "false, NESTED", "true, FLAT", "true, NESTED"})
32-
void handlerReturnDrainsQueuedBranches(boolean parallel, NestingType nestingType) throws Exception {
34+
@CsvSource({
35+
"false, FLAT, false", "false, NESTED, false", "true, FLAT, false", "true, NESTED, false",
36+
"false, FLAT, true", "false, NESTED, true", "true, FLAT, true", "true, NESTED, true"
37+
})
38+
void handlerReturnDrainsQueuedBranches(boolean parallel, NestingType nestingType, boolean nestedSteps)
39+
throws Exception {
3340
var users = Executors.newCachedThreadPool();
3441
var runtime = Executors.newSingleThreadExecutor();
3542
var firstStarted = new CountDownLatch(1);
@@ -41,11 +48,17 @@ void handlerReturnDrainsQueuedBranches(boolean parallel, NestingType nestingType
4148
String.class,
4249
(input, context) -> {
4350
manager.set(((DurableContextImpl) context).getExecutionManager());
44-
startQueuedWork(context, parallel, nestingType, item -> {
51+
startQueuedWork(context, parallel, nestingType, (item, child) -> {
4552
if (item.equals("a")) {
4653
firstStarted.countDown();
4754
await(releaseFirst);
4855
}
56+
if (nestedSteps) {
57+
return child.step("nested-step", String.class, step -> {
58+
completed.add(item);
59+
return item;
60+
});
61+
}
4962
completed.add(item);
5063
return item;
5164
});
@@ -66,15 +79,89 @@ void handlerReturnDrainsQueuedBranches(boolean parallel, NestingType nestingType
6679
assertEquals(
6780
OperationStatus.SUCCEEDED,
6881
result.getOperation("queued-work").getStatus());
82+
if (nestedSteps) {
83+
var steps = result.getOperations().stream()
84+
.filter(op -> "nested-step".equals(op.getName()))
85+
.toList();
86+
assertEquals(3, steps.size());
87+
assertTrue(steps.stream().allMatch(op -> op.getStatus() == OperationStatus.SUCCEEDED));
88+
var replay = runner.runUntilComplete("input");
89+
assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus());
90+
assertEquals(List.of("a", "b", "c"), completed, "Replay must not repeat nested step bodies");
91+
assertEquals(
92+
steps.stream().map(op -> op.getId()).toList(),
93+
replay.getOperations().stream()
94+
.filter(op -> "nested-step".equals(op.getName()))
95+
.map(op -> op.getId())
96+
.toList());
97+
}
6998
} finally {
7099
releaseFirst.countDown();
71100
stop(runtime);
72101
stop(users);
73102
}
74103
}
75104

105+
@Test
106+
void drainsAsyncDescendantsCreatedAfterTheRootReturns() throws Exception {
107+
var users = Executors.newCachedThreadPool();
108+
var runtime = Executors.newSingleThreadExecutor();
109+
var childEntered = new CountDownLatch(1);
110+
var releaseChild = new CountDownLatch(1);
111+
var leafEntered = new CountDownLatch(1);
112+
var releaseLeaf = new CountDownLatch(1);
113+
var effects = new AtomicInteger();
114+
var manager = new AtomicReference<ExecutionManager>();
115+
var runner = LocalDurableTestRunner.create(
116+
String.class,
117+
(input, context) -> {
118+
manager.set(((DurableContextImpl) context).getExecutionManager());
119+
context.runInChildContextAsync("child", String.class, child -> {
120+
childEntered.countDown();
121+
await(releaseChild);
122+
child.runInChildContextAsync("grandchild", String.class, grandchild -> {
123+
grandchild.stepAsync("leaf", String.class, step -> {
124+
leafEntered.countDown();
125+
await(releaseLeaf);
126+
effects.incrementAndGet();
127+
return "leaf-result";
128+
});
129+
return "grandchild-result";
130+
});
131+
return "child-result";
132+
});
133+
await(childEntered);
134+
return "done";
135+
},
136+
DurableConfig.builder().withExecutorService(users).build());
137+
try {
138+
var invocation = runtime.submit(() -> runner.runUntilComplete("input"));
139+
await(childEntered);
140+
awaitDraining(manager.get());
141+
releaseChild.countDown();
142+
await(leafEntered);
143+
assertFalse(invocation.isDone(), "Shutdown must wait for the newly registered async leaf");
144+
releaseLeaf.countDown();
145+
var result = invocation.get(10, TimeUnit.SECONDS);
146+
assertEquals(ExecutionStatus.SUCCEEDED, result.getStatus());
147+
assertEquals(OperationStatus.SUCCEEDED, result.getOperation("leaf").getStatus());
148+
assertEquals("leaf-result", result.getOperation("leaf").getStepResult(String.class));
149+
var replay = runner.runUntilComplete("input");
150+
assertEquals(ExecutionStatus.SUCCEEDED, replay.getStatus());
151+
assertEquals(1, effects.get(), "The checkpointed descendant must not repeat on replay");
152+
} finally {
153+
releaseChild.countDown();
154+
releaseLeaf.countDown();
155+
stop(runtime);
156+
stop(users);
157+
}
158+
}
159+
76160
private static void startQueuedWork(
77-
DurableContext context, boolean parallel, NestingType nestingType, Function<String, String> action) {
161+
DurableContext context,
162+
boolean parallel,
163+
NestingType nestingType,
164+
BiFunction<String, DurableContext, String> action) {
78165
// Default allCompleted requires a caller to join. Use an explicit threshold to complete without joining.
79166
var completion = CompletionConfig.minSuccessful(3);
80167
if (parallel) {
@@ -86,14 +173,14 @@ private static void startQueuedWork(
86173
.completionConfig(completion)
87174
.build());
88175
for (var item : List.of("a", "b", "c")) {
89-
operation.branch(item, String.class, child -> action.apply(item));
176+
operation.branch(item, String.class, child -> action.apply(item, child));
90177
}
91178
} else {
92179
context.mapAsync(
93180
"queued-work",
94181
List.of("a", "b", "c"),
95182
String.class,
96-
(item, index, child) -> action.apply(item),
183+
(item, index, child) -> action.apply(item, child),
97184
MapConfig.builder()
98185
.maxConcurrency(1)
99186
.nestingType(nestingType)

‎sdk/src/main/java/software/amazon/lambda/durable/execution/ExecutionManager.java‎

Lines changed: 66 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
import java.util.Objects;
1616
import java.util.Optional;
1717
import java.util.Set;
18-
import java.util.concurrent.CancellationException;
1918
import java.util.concurrent.CompletableFuture;
2019
import java.util.concurrent.ConcurrentHashMap;
20+
import java.util.concurrent.ExecutionException;
2121
import java.util.concurrent.ExecutorService;
2222
import java.util.concurrent.RejectedExecutionException;
2323
import java.util.concurrent.ThreadPoolExecutor;
@@ -71,8 +71,8 @@ enum LifecycleState {
7171
OPEN,
7272

7373
/**
74-
* Invocation shutdown has started. New operations and root tasks are rejected, while registered operations may
75-
* still submit tasks, checkpoint, or poll as they finish.
74+
* Invocation shutdown has started. Admitted child/coordinator tasks may create descendants, and registered
75+
* operations may submit tasks, checkpoint, or poll as they finish. New root and unrelated work is rejected.
7676
*/
7777
DRAINING,
7878

@@ -98,6 +98,8 @@ enum LifecycleState {
9898
private final Long deadlineNanos;
9999
private final AtomicLong taskSequence = new AtomicLong();
100100
private final Map<Long, ExecutorTaskHandle<?>> activeExecutorTasks = new ConcurrentHashMap<>();
101+
// The current task is shared across managers on a thread; admission verifies the exact handle's owner.
102+
private static final ThreadLocal<ExecutorTaskHandle<?>> currentExecutorTask = new ThreadLocal<>();
101103
private LifecycleState lifecycleState = LifecycleState.OPEN;
102104
// Guarded by admissionLock. Counts enqueue calls, not the lifetime of their returned futures.
103105
private int checkpointAdmissionsInFlight;
@@ -228,7 +230,7 @@ public Collection<Operation> getUpdatedOperationsSnapshot() {
228230
/** Registers an operation so it can receive checkpoint completion notifications. */
229231
public void registerOperation(BaseDurableOperation operation) {
230232
synchronized (admissionLock) {
231-
requireOpen("durable operation");
233+
requireOperationAdmission();
232234
registeredOperations.put(operation.getOperationId(), operation);
233235
}
234236
}
@@ -267,14 +269,28 @@ private <T> CompletableFuture<T> submitExecutorTask(
267269
}
268270

269271
try {
270-
task.bindExecution(executor.submit(() -> task.run(action)));
272+
task.bindExecution(executor.submit(() -> runExecutorTask(task, action)));
271273
return task.completion();
272274
} catch (RuntimeException | Error failure) {
273275
task.submissionFailed(failure);
274276
throw failure;
275277
}
276278
}
277279

280+
private <T> void runExecutorTask(ExecutorTaskHandle<T> task, Supplier<T> action) {
281+
var previous = currentExecutorTask.get();
282+
currentExecutorTask.set(task);
283+
try {
284+
task.run(action);
285+
} finally {
286+
if (previous == null) {
287+
currentExecutorTask.remove();
288+
} else {
289+
currentExecutorTask.set(previous);
290+
}
291+
}
292+
}
293+
278294
String getInvocationId() {
279295
return invocationId;
280296
}
@@ -570,6 +586,20 @@ private <T> T admitCheckpoint(Supplier<T> request) {
570586
}
571587
}
572588

589+
private void requireOperationAdmission() {
590+
var task = currentExecutorTask.get();
591+
if (lifecycleState == LifecycleState.DRAINING
592+
&& task != null
593+
&& task.state() == ExecutorTaskHandle.State.RUNNING
594+
&& activeExecutorTasks.get(task.id()) == task
595+
&& (task.role() == ExecutorTaskHandle.Role.CHILD_CONTEXT
596+
|| task.role() == ExecutorTaskHandle.Role.COORDINATOR)) {
597+
// Descendants are part of the admitted task's work, including branches that start during draining.
598+
return;
599+
}
600+
requireOpen("durable operation");
601+
}
602+
573603
private void requireTaskAdmission(BaseDurableOperation operation) {
574604
if (lifecycleState == LifecycleState.DRAINING
575605
&& operation != null
@@ -592,22 +622,13 @@ private RejectedExecutionException rejected(String workType) {
592622
}
593623

594624
private void validateRunningThreads() {
595-
// This will detect stuck user thread and thread leaks in the thread pool
596-
for (BaseDurableOperation op : registeredOperations.values()) {
597-
var userHandlerFuture = op.getRunningUserHandler();
598-
if (userHandlerFuture != null && !userHandlerFuture.isDone()) {
599-
// Some user threads can still be running because
600-
// the operations that run them have never been waiting for and the execution has completed.
601-
logger.info("Waiting for operation to complete before shutting down: {}", op.getOperationId());
602-
try {
603-
userHandlerFuture.get();
604-
} catch (InterruptedException | CancellationException e) {
605-
// if the user handler is stuck
606-
throw new IllegalStateException(
607-
"Stuck running user handler when shutting down: " + op.getOperationId());
608-
} catch (Exception e) {
609-
// ok if the future completed exceptionally
610-
}
625+
while (true) {
626+
var tasks = operationTasksToDrain();
627+
if (tasks.isEmpty()) {
628+
break;
629+
}
630+
for (var task : tasks) {
631+
awaitTaskExit(task);
611632
}
612633
}
613634

@@ -621,6 +642,30 @@ private void validateRunningThreads() {
621642
}
622643
}
623644

645+
private List<ExecutorTaskHandle<?>> operationTasksToDrain() {
646+
synchronized (admissionLock) {
647+
var tasks = activeExecutorTasks.values().stream()
648+
.filter(task -> task.role() != ExecutorTaskHandle.Role.ROOT)
649+
.toList();
650+
if (tasks.isEmpty()) {
651+
// Close under the submission lock so no task can enter after the final empty snapshot.
652+
lifecycleState = LifecycleState.CLOSED;
653+
}
654+
return tasks;
655+
}
656+
}
657+
658+
private void awaitTaskExit(ExecutorTaskHandle<?> task) {
659+
try {
660+
task.exit().get();
661+
} catch (InterruptedException e) {
662+
Thread.currentThread().interrupt();
663+
throw new IllegalStateException("Interrupted while draining operation task: " + task.operationId(), e);
664+
} catch (ExecutionException e) {
665+
throw new IllegalStateException("Could not drain operation task: " + task.operationId(), e);
666+
}
667+
}
668+
624669
/** Returns {@code true} if the given status represents a terminal (final) operation state. */
625670
public static boolean isTerminalStatus(OperationStatus status) {
626671
return status == OperationStatus.SUCCEEDED

0 commit comments

Comments
 (0)