From 9d53081e9b30f0ba4fbdc85b4c39217be1451ad7 Mon Sep 17 00:00:00 2001 From: yaythomas Date: Sun, 1 Nov 2026 05:31:59 +0000 Subject: [PATCH] fix(testing): derive updated operation IDs The local runner kept UpdatedOperationIds in a list that every completion path had to append to, and cleared it when it built the invocation input. Callback completions never appended, so a callback that completed while suspended looked like a replay. A failed invocation lost the list, while the service lists every change since the last successful invocation again. The runner also did not record that the input delivered the state, so the first checkpoint response of an invocation sent those operations again. Derive the list from the touch sequences the runner already keeps. Building the input sets handler_seen_seq. A valid response saves it as last_successful_invocation_seq. The next input lists every operation touched after that point, or every operation before the first success. The stored list, its producer calls and the invocation_seq baseline are removed. The retry behavior matches the Lambda measurements in #892. Fixes #860 Fixes #861 Co-authored-by: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> --- .../docs/architecture.md | 38 ++- .../execution.py | 83 ++++-- .../executor.py | 41 +-- .../e2e/callback_updated_operations_test.py | 146 +++++++++ .../tests/e2e/updated_operation_ids_test.py | 219 ++++++++++++++ .../tests/execution_test.py | 278 ++++++++++++++++-- .../tests/executor_checkpoint_test.py | 37 +++ .../tests/executor_pause_resume_test.py | 6 +- .../tests/executor_test.py | 68 ++--- .../tests/invoker_test.py | 5 +- .../plugin.py | 26 +- .../aws_durable_execution_sdk_python/state.py | 8 +- 12 files changed, 805 insertions(+), 150 deletions(-) create mode 100644 packages/aws-durable-execution-sdk-python-testing/tests/e2e/callback_updated_operations_test.py create mode 100644 packages/aws-durable-execution-sdk-python-testing/tests/e2e/updated_operation_ids_test.py diff --git a/packages/aws-durable-execution-sdk-python-testing/docs/architecture.md b/packages/aws-durable-execution-sdk-python-testing/docs/architecture.md index 0a33e4ec1..fbcf455b8 100644 --- a/packages/aws-durable-execution-sdk-python-testing/docs/architecture.md +++ b/packages/aws-durable-execution-sdk-python-testing/docs/architecture.md @@ -163,13 +163,13 @@ the handler would wait on a lane it needs in order to make progress. `_invoke_handler` splits the invocation into three steps. 1. Begin. A task on the lane claims the gate from `PRE_INVOKE` to - `INVOKING` and builds the invocation input. + `INVOKING`, builds the invocation input, and sets `handler_seen_seq`. 2. Invoke. The blocking call to the function runs off the lane, bounded by the invocation timeout. While it blocks, the handler's checkpoint and get state tasks run on the now free lane. 3. Finish. A task on the lane records the response and releases the gate. It moves `PENDING` to `PRE_INVOKE`, or a terminal result to - `COMPLETED`. + `COMPLETED`. A valid response saves `last_successful_invocation_seq`. ```mermaid sequenceDiagram @@ -225,19 +225,27 @@ the checkpoint write transaction. - Idempotent replays advance neither. - Pure reads advance neither. -## The watermark and delta - -`Execution.handler_seen_seq` records the highest `seq_counter` the -handler has been told about. The handler learns of operations through -`InitialExecutionState` on an invocation, or through the `operations` -list in a checkpoint response. The delta for the next response is the -operations whose `operation_last_touched_seq` is greater than -`handler_seen_seq`. - -The watermark advances only when a checkpoint response is built. It never -advances when invocation input is built. That is what makes retries -correct. A failed invocation never moves the watermark. So the retry -delivers the same state again. +## The watermarks and the delta + +The runner keeps two watermarks over `seq_counter`. + +1. `Execution.handler_seen_seq` is the highest sequence the handler has + observed. Building the invocation input sets it to `seq_counter`, + because the input carries the state as of that point. Each checkpoint + response raises it to the highest touch it returns. A paused checkpoint + returns no operations and leaves it. The delta for the next checkpoint + response is the operations whose `operation_last_touched_seq` is + greater than `handler_seen_seq`. +2. `Execution.last_successful_invocation_seq` is `handler_seen_seq` as it + stood when the last successful invocation ended. A successful + invocation returned a valid `PENDING`, `SUCCEEDED` or `FAILED` output. + A failed invocation leaves the value unchanged. + +The next input's `UpdatedOperationIds` lists every operation touched after +`last_successful_invocation_seq`, or every operation while it is `None`. +The list is derived from these values and is not stored. So every change +that touches an operation is listed, and a retry lists again every change +since the last successful invocation, as the service does. ``` seq_counter handler_seen_seq delta next checkpoint diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py index 5190b9b69..69e8c0149 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/execution.py @@ -132,7 +132,6 @@ def __init__( self.updates: list[OperationUpdate] = [] self.update_timestamps: list[datetime] = [] self.invocation_completions: list[InvocationCompletedDetails] = [] - self.updated_operation_ids: list[str] = [] # Two monotonic counters. `token_sequence` is the checkpoint-response # version carried on the wire; `seq_counter` is the internal # event counter bumped on every state-affecting mutation. @@ -142,6 +141,12 @@ def __init__( # about (via InitialExecutionState or a successful checkpoint # response). Filters the "unseen" delta on the next checkpoint. self.handler_seen_seq: int = 0 + # `handler_seen_seq` as it stood when the last successful invocation + # ended. The next input lists every operation touched after it. A + # failed invocation leaves it unchanged, so the retry lists those + # operations again. None means that no invocation has succeeded yet, + # so the input lists every operation. + self.last_successful_invocation_seq: int | None = None # Per-op "last touched at `seq_counter`" bookkeeping. Sparse: # ops not in the dict are treated as touch-seq 0. self.operation_last_touched_seq: dict[str, int] = {} @@ -271,10 +276,10 @@ def to_json_dict(self) -> dict[str, Any]: "InvocationCompletions": [ completion.to_json_dict() for completion in self.invocation_completions ], - "UpdatedOperationIds": self.updated_operation_ids, "TokenSequence": self.token_sequence, "SeqCounter": self.seq_counter, "HandlerSeenSeq": self.handler_seen_seq, + "LastSuccessfulInvocationSeq": self.last_successful_invocation_seq, "OperationLastTouchedSeq": dict(self.operation_last_touched_seq), "OperationSizeBytes": dict(self.operation_size_bytes), "ChainedInvokeChildren": dict(self.chained_invoke_children), @@ -321,7 +326,6 @@ def from_json_dict(cls, data: dict[str, Any]) -> Execution: InvocationCompletedDetails.from_json_dict(item) for item in data.get("InvocationCompletions", []) ] - execution.updated_operation_ids = list(data.get("UpdatedOperationIds", [])) # NOTE: prior format included a "UsedTokens" set. Current # removed it (the Executor no longer consults it — token # validity is via _is_current_token against token_sequence). @@ -330,6 +334,11 @@ def from_json_dict(cls, data: dict[str, Any]) -> Execution: # Safe defaults for fields added after the original schema. execution.seq_counter = data.get("SeqCounter", 0) execution.handler_seen_seq = data.get("HandlerSeenSeq", 0) + # A file from an older runner has no saved watermark. Its next input + # then lists every operation once, as after a failed first invocation. + execution.last_successful_invocation_seq = data.get( + "LastSuccessfulInvocationSeq" + ) execution.operation_last_touched_seq = dict( data.get("OperationLastTouchedSeq", {}) ) @@ -458,12 +467,9 @@ def has_pending_operations(self, execution: Execution) -> bool: def has_unseen_changes(self) -> bool: """True if any operation changed after the handler's last observed sequence watermark.""" - return self.has_changes_after(self.handler_seen_seq) - - def has_changes_after(self, seq: int) -> bool: - """True if any operation changed after ``seq_counter`` was ``seq``.""" return any( - touched > seq for touched in self.operation_last_touched_seq.values() + touched > self.handler_seen_seq + for touched in self.operation_last_touched_seq.values() ) def record_invocation_completion( @@ -479,23 +485,46 @@ def record_invocation_completion( ) def mark_state_delivered(self) -> None: - """Reset the list of changed operations once an invocation's input - carries them. - - The service reports, on each invocation, the operations that - changed after the last state the handler observed. The runner - delivers state in the invocation input. So the list is reset when - that input is built, not when the invocation completes: an - operation that completes while the handler is still running is - reported on the next invocation. + """Record that the handler has observed the state up to ``seq_counter``. + + The runner calls this when it builds an invocation input. The input + carries the state as of ``seq_counter``. So the next checkpoint + response returns only the operations touched after it. + """ + self.handler_seen_seq = self.seq_counter + + def record_successful_invocation(self) -> None: + """Record that an invocation returned a valid response. + + A valid response is a valid ``PENDING``, ``SUCCEEDED`` or ``FAILED`` + output. It resets the failed-attempt count and saves the handler's + watermark. An invocation that raised, timed out or returned an invalid + output leaves the watermark unchanged. So the next input lists again + every change since the last successful invocation, including the + changes the failed invocation observed or made. """ - self.updated_operation_ids = [] + self.consecutive_failed_invocation_attempts = 0 + self.last_successful_invocation_seq = self.handler_seen_seq - def _record_updated_operation(self, operation_id: str) -> None: - """Remember an operation that changed since the handler last saw - the state.""" - if operation_id not in self.updated_operation_ids: - self.updated_operation_ids.append(operation_id) + @property + def updated_operation_ids(self) -> list[str]: + """The ``UpdatedOperationIds`` of the next invocation input. + + Until an invocation succeeds, every operation, the EXECUTION operation + first. After that, every operation touched after the watermark the + last successful invocation saved. A change the handler observed in a + successful invocation, in its input or in a checkpoint response, is + not listed again. The list is derived from the touch sequences, so + every state change is covered once it is touched. + """ + watermark = self.last_successful_invocation_seq + return [ + operation.operation_id + for operation in self.operations + if watermark is None + or self.operation_last_touched_seq.get(operation.operation_id, 0) + > watermark + ] def complete_success(self, result: str | None, now: datetime | None = None) -> None: """Complete execution successfully (DecisionType.COMPLETE_WORKFLOW_EXECUTION).""" @@ -580,7 +609,6 @@ def complete_wait( status=OperationStatus.SUCCEEDED, end_timestamp=now if now is not None else real_now(), ) - self._record_updated_operation(operation_id) return self.operations[index] def complete_retry(self, operation_id: str) -> Operation: @@ -614,7 +642,6 @@ def complete_retry(self, operation_id: str) -> Operation: # Assign self.operations[index] = updated_operation - self._record_updated_operation(operation_id) return updated_operation def complete_due_operations(self, now: datetime) -> bool: @@ -678,13 +705,16 @@ def complete_callback_success( raise IllegalStateException(msg) with self._state_lock: - self.touch_operation(operation.operation_id) + # Decode first: a result that fails to decode must leave no + # touch behind, or the next input would list a change that + # never happened. updated_callback_details = None if operation.callback_details: updated_callback_details = replace( operation.callback_details, result=result.decode() if result else None, ) + self.touch_operation(operation.operation_id) self.operations[index] = replace( operation, @@ -792,7 +822,6 @@ def complete_chained_invoke( self.operation_size_bytes[operation_id] = len( (result if result is not None else "").encode() ) + (len(json.dumps(error.to_dict())) if error is not None else 0) - self._record_updated_operation(operation_id) return self.operations[index] def executed_version(self) -> str: diff --git a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/executor.py b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/executor.py index 4c3a0a71f..48c4bbea9 100644 --- a/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/executor.py +++ b/packages/aws-durable-execution-sdk-python-testing/src/aws_durable_execution_sdk_python_testing/executor.py @@ -1403,13 +1403,9 @@ def _validate_invocation_response_and_store( execution_arn: str, response: DurableExecutionInvocationOutput, execution: Execution, - invocation_seq: int | None = None, ): """Validate response status and save it to the store if fine. - ``invocation_seq`` is the execution's ``seq_counter`` when the - invocation's input was built. - Raises: InvalidParameterValueException: If the response status is invalid. IllegalStateException: If the response status is valid but the execution is already completed. @@ -1474,12 +1470,7 @@ def _validate_invocation_response_and_store( if ( not execution.has_deferred_invocation and not execution.has_pending_operations(execution) - and not ( - invocation_seq is not None - and execution.has_changes_after( - max(invocation_seq, execution.handler_seen_seq) - ) - ) + and not execution.has_unseen_changes() ): msg_pending_ops: str = ( "Cannot return PENDING status with no pending operations." @@ -1495,18 +1486,17 @@ def _validate_invocation_response_and_store( def _begin_invocation( self, execution_arn: str - ) -> tuple[Execution, DurableExecutionInvocationInput, int] | None: + ) -> tuple[Execution, DurableExecutionInvocationInput] | None: """Claim the invocation gate and build the handler input. - Returns the execution, its invocation input, and the execution's - ``seq_counter`` at that moment when this call claims the gate - (PRE_INVOKE -> INVOKING); returns None when the execution is - already complete, the gate is COMPLETED, or another invocation - is in flight (recording a deferred re-invoke). + Returns the execution and its invocation input when this call + claims the gate (PRE_INVOKE -> INVOKING); returns None when the + execution is already complete, the gate is COMPLETED, or another + invocation is in flight (recording a deferred re-invoke). - The counter is captured here, on the execution's serial lane, - because a completion queued behind this call mutates the same - object before the caller resumes. + Building the input records, on the execution's serial lane, the + sequence the handler has now observed. A completion queued behind + this call is touched after it, so it counts as unseen. """ execution = self._store.load(execution_arn) @@ -1553,7 +1543,7 @@ def _begin_invocation( invocation_input = self._invoker.create_invocation_input(execution=execution) execution.mark_state_delivered() self._store.save(execution) - return execution, invocation_input, execution.seq_counter + return execution, invocation_input def _finish_invocation( self, @@ -1561,7 +1551,6 @@ def _finish_invocation( invoke_response: InvokeResponse, invocation_start: datetime, invocation_end: datetime, - invocation_seq: int | None = None, ) -> None: """Apply a completed handler invocation. @@ -1593,7 +1582,7 @@ def _finish_invocation( response = invoke_response.invocation_output try: self._validate_invocation_response_and_store( - execution_arn, response, execution, invocation_seq + execution_arn, response, execution ) except (InvalidParameterValueException, IllegalStateException) as e: logger.warning( @@ -1615,9 +1604,10 @@ def _finish_invocation( self._fail_workflow(execution_arn, error_obj) return - # A clean invocation resets the retry counter. + # A clean invocation resets the retry counter and saves the + # handler's watermark for the next input's updated operations. reset_target = self._store.load(execution_arn) - reset_target.consecutive_failed_invocation_attempts = 0 + reset_target.record_successful_invocation() self._store.save(reset_target) # Release the gate: terminal -> COMPLETED, otherwise PRE_INVOKE @@ -1725,7 +1715,7 @@ async def invoke() -> None: ) if claim is None: return - execution, invocation_input, invocation_seq = claim + execution, invocation_input = claim invocation_start = self._clock.now() invoke_response = await asyncio.wait_for( @@ -1757,7 +1747,6 @@ async def invoke() -> None: invoke_response, invocation_start, invocation_end, - invocation_seq, ) ), ) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/callback_updated_operations_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/callback_updated_operations_test.py new file mode 100644 index 000000000..2ebe0e127 --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/callback_updated_operations_test.py @@ -0,0 +1,146 @@ +"""The first externally completed callback is visible before dependent code.""" + +import json +import time +from pathlib import Path +from typing import Any + +import pytest +from aws_durable_execution_sdk_python import DurableContext, durable_execution +from aws_durable_execution_sdk_python.config import CallbackConfig, Duration +from aws_durable_execution_sdk_python.exceptions import CallbackError +from aws_durable_execution_sdk_python.execution import InvocationStatus +from aws_durable_execution_sdk_python.lambda_service import ErrorObject, OperationStatus +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + InvocationStartInfo, + OperationEndInfo, +) +from aws_durable_execution_sdk_python.serdes import JsonSerDes + +from aws_durable_execution_sdk_python_testing.runner import DurableFunctionTestRunner +from aws_durable_execution_sdk_python_testing.stores.filesystem import ( + FileSystemExecutionStore, +) + + +def _suspended_callback(runner: DurableFunctionTestRunner, arn: str) -> str: + deadline = time.monotonic() + 10 + while time.monotonic() < deadline: + events = runner.get_execution_history(arn, include_execution_data=True).events + starts = [ + event + for event in events + if event.event_type == "CallbackStarted" and event.name == "target" + ] + if starts and any( + event.event_type == "InvocationCompleted" + and event.event_id > starts[0].event_id + for event in events + ): + details = starts[0].callback_started_details + assert details is not None and details.callback_id is not None + return details.callback_id + time.sleep(0.01) + raise AssertionError("The target callback did not suspend") + + +@pytest.mark.parametrize("outcome", ["success", "failure", "timeout"]) +@pytest.mark.parametrize("filesystem", [False, True]) +def test_callback_first_completion_precedes_dependent_step( + outcome: str, filesystem: bool, tmp_path: Path +) -> None: + invocations: list[InvocationStartInfo] = [] + ends: list[OperationEndInfo] = [] + saved_calls: list[str] = [] + observed_calls: list[str] = [] + completions_before_step: list[OperationEndInfo] = [] + + class Observer(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + invocations.append(info) + + def on_operation_end(self, info: OperationEndInfo) -> None: + if info.name == "target": + ends.append(info) + + class ObserverFactory: + def create_plugin(self, _info: InvocationStartInfo) -> Observer: + return Observer() + + def saved_step(_context: Any) -> str: + saved_calls.append("saved") + return "saved" + + def observed_step(_context: Any, value: str) -> str: + completions_before_step.extend(ends) + observed_calls.append(value) + return value + + def body(_event: Any, context: DurableContext) -> str: + saved = context.step(saved_step, name="saved") + config = CallbackConfig( + timeout=Duration.from_seconds(1 if outcome == "timeout" else 30), + serdes=JsonSerDes(), + ) + try: + value = context.create_callback(name="target", config=config).result() + except CallbackError: + value = outcome + assert isinstance(value, str) + observed = context.step( + lambda step_context: observed_step(step_context, value), name="observed" + ) + return saved + "/" + observed + + with DurableFunctionTestRunner( + handler=durable_execution(body, plugins=[ObserverFactory()]), + store=FileSystemExecutionStore.create(tmp_path) if filesystem else None, + skip_time=False, + poll_interval=0.01, + execution_timeout=20, + ) as runner: + arn = runner.run_async(input="{}") + callback_id = _suspended_callback(runner, arn) + if outcome == "success": + runner.send_callback_success( + callback_id, result=json.dumps("target").encode() + ) + elif outcome == "failure": + runner.send_callback_failure( + callback_id, error=ErrorObject.from_message("explicit failure") + ) + # Timeout exercises the scheduled public-runner completion path. + result = runner.wait_for_result(arn, timeout=10) + + expected = "target" if outcome == "success" else outcome + assert result.status is InvocationStatus.SUCCEEDED + assert ( + result.result is not None and json.loads(result.result) == "saved/" + expected + ) + assert saved_calls == ["saved"] + assert observed_calls == [expected] + assert len(invocations) == 2 + assert completions_before_step, "The terminal hook must precede dependent code" + terminal = completions_before_step[0] + assert terminal.is_replayed is False + assert ( + terminal.status + is { + "success": OperationStatus.SUCCEEDED, + "failure": OperationStatus.FAILED, + "timeout": OperationStatus.TIMED_OUT, + }[outcome] + ) + snapshot = invocations[1].updated_operations[terminal.operation_id] + assert snapshot.status is terminal.status + assert snapshot.result == terminal.result + assert snapshot.error == terminal.error + if outcome == "success": + assert terminal.result == json.dumps("target") + assert terminal.error is None + elif outcome == "failure": + assert terminal.error is not None + assert terminal.error.message == "explicit failure" + else: + assert terminal.error is not None diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/e2e/updated_operation_ids_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/updated_operation_ids_test.py new file mode 100644 index 000000000..7f9445e0b --- /dev/null +++ b/packages/aws-durable-execution-sdk-python-testing/tests/e2e/updated_operation_ids_test.py @@ -0,0 +1,219 @@ +"""The operations each invocation input lists as updated. + +Until an invocation succeeds, the input lists every operation. After that, it +lists every operation that changed after the state the last successful +invocation observed. A failed invocation does not count, so its retry lists +the same changes again, including the ones the failed invocation made. +""" + +from __future__ import annotations + +import json +import threading +from pathlib import Path +from typing import Any + +import pytest +from aws_durable_execution_sdk_python import DurableContext, durable_execution +from aws_durable_execution_sdk_python.config import CallbackConfig, Duration +from aws_durable_execution_sdk_python.exceptions import InvocationError +from aws_durable_execution_sdk_python.execution import InvocationStatus +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + InvocationStartInfo, + OperationEndInfo, + OperationType, +) +from aws_durable_execution_sdk_python.serdes import JsonSerDes + +from aws_durable_execution_sdk_python_testing.executor import Executor +from aws_durable_execution_sdk_python_testing.runner import DurableFunctionTestRunner +from aws_durable_execution_sdk_python_testing.stores.base import ExecutionStore +from aws_durable_execution_sdk_python_testing.stores.filesystem import ( + FileSystemExecutionStore, +) + +EXECUTION = "" + + +@pytest.fixture(autouse=True) +def _no_retry_backoff(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(Executor, "RETRY_BACKOFF_SECONDS", 0) + + +@pytest.fixture(params=["memory", "filesystem"]) +def store(request: pytest.FixtureRequest, tmp_path: Path) -> ExecutionStore | None: + if request.param == "filesystem": + return FileSystemExecutionStore.create(tmp_path) + return None + + +def _listed(info: InvocationStartInfo) -> set[str]: + return { + EXECUTION + if operation.operation_type is OperationType.EXECUTION + else operation.name or operation.operation_id + for operation in info.updated_operations.values() + } + + +def _run( + body: Any, + store: ExecutionStore | None, + fail_before_sdk_on_attempt: int | None = None, +) -> list[set[str]]: + """Run ``body`` and return what each invocation input listed as updated. + + ``fail_before_sdk_on_attempt`` makes that Lambda attempt fail before the + SDK receives its input, so that attempt records no invocation start. + """ + starts: list[InvocationStartInfo] = [] + attempts: list[int] = [] + + class Observer(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + starts.append(info) + + class ObserverFactory: + def create_plugin(self, _info: InvocationStartInfo) -> Observer: + return Observer() + + durable_handler = durable_execution(body, plugins=[ObserverFactory()]) + + def entrypoint(event: Any, context: Any) -> Any: + attempts.append(1) + if len(attempts) == fail_before_sdk_on_attempt: + msg = "the invocation failed before the SDK received its input" + raise RuntimeError(msg) + return durable_handler(event, context) + + with DurableFunctionTestRunner( + handler=entrypoint, + store=store, + poll_interval=0.01, + execution_timeout=30, + ) as runner: + arn = runner.run_async(input="{}") + result = runner.wait_for_result(arn, timeout=25) + assert result.status is InvocationStatus.SUCCEEDED, result + return [_listed(info) for info in starts] + + +def test_retry_after_failed_first_invocation_lists_execution_and_step( + store: ExecutionStore | None, +) -> None: + attempts: list[int] = [] + + def body(_event: Any, context: DurableContext) -> str: + attempts.append(1) + context.step(lambda _c: "a", name="A") + if len(attempts) == 1: + msg = "fail once" + raise InvocationError(msg) + return "done" + + assert _run(body, store) == [{EXECUTION}, {EXECUTION, "A"}] + + +def test_resume_after_successful_invocation_lists_only_the_wait( + store: ExecutionStore | None, +) -> None: + def body(_event: Any, context: DurableContext) -> str: + context.step(lambda _c: "a", name="A") + context.wait(Duration.from_seconds(1), name="W") + return "done" + + assert _run(body, store) == [{EXECUTION}, {"W"}] + + +def test_retry_after_failed_resume_lists_its_changes_again( + store: ExecutionStore | None, +) -> None: + attempts: list[int] = [] + + def body(_event: Any, context: DurableContext) -> str: + attempts.append(1) + context.step(lambda _c: "a", name="A") + context.wait(Duration.from_seconds(1), name="W") + context.step(lambda _c: "b", name="B") + if len(attempts) == 2: + msg = "fail the resume once" + raise InvocationError(msg) + return "done" + + assert _run(body, store) == [{EXECUTION}, {"W"}, {"W", "B"}] + + +def test_failed_delivery_lists_the_wait_again(store: ExecutionStore | None) -> None: + steps: list[str] = [] + + def record(name: str) -> str: + steps.append(name) + return name + + def body(_event: Any, context: DurableContext) -> str: + context.step(lambda _c: record("A"), name="A") + context.wait(Duration.from_seconds(1), name="W") + context.step(lambda _c: record("B"), name="B") + return "done" + + assert _run(body, store, fail_before_sdk_on_attempt=2) == [{EXECUTION}, {"W"}] + assert steps == ["A", "B"] + + +def test_callback_consumed_mid_invocation_is_not_listed_again() -> None: + """A callback that completes while the handler runs reaches it in a + checkpoint response. The next invocation does not list it again, so the + callback ends once.""" + starts: list[InvocationStartInfo] = [] + ends: list[OperationEndInfo] = [] + callback_ids: list[str] = [] + callback_created = threading.Event() + callback_sent = threading.Event() + + class Observer(DurableInstrumentationPlugin): + def on_invocation_start(self, info: InvocationStartInfo) -> None: + starts.append(info) + + def on_operation_end(self, info: OperationEndInfo) -> None: + if info.name == "target": + ends.append(info) + + class ObserverFactory: + def create_plugin(self, _info: InvocationStartInfo) -> Observer: + return Observer() + + def blocking_step(_context: Any) -> str: + assert callback_sent.wait(timeout=10) + return "stepped" + + def body(_event: Any, context: DurableContext) -> str: + callback = context.create_callback( + name="target", + config=CallbackConfig( + timeout=Duration.from_seconds(60), serdes=JsonSerDes() + ), + ) + callback_ids.append(callback.callback_id) + callback_created.set() + context.step(blocking_step, name="blocker") + value = callback.result() + context.wait(Duration.from_seconds(1), name="pause") + return str(value) + + with DurableFunctionTestRunner( + handler=durable_execution(body, plugins=[ObserverFactory()]), + skip_time=False, + poll_interval=0.01, + execution_timeout=30, + ) as runner: + arn = runner.run_async(input="{}") + assert callback_created.wait(timeout=10) + runner.send_callback_success(callback_ids[0], result=json.dumps("cb").encode()) + callback_sent.set() + result = runner.wait_for_result(arn, timeout=20) + + assert result.status is InvocationStatus.SUCCEEDED + assert [_listed(info) for info in starts] == [{EXECUTION}, {"pause"}] + assert len(ends) == 1 + assert ends[0].is_replayed is False diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/execution_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/execution_test.py index bde5e3d4b..7daa0a623 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/execution_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/execution_test.py @@ -3,6 +3,8 @@ import json from dataclasses import replace from datetime import datetime, timezone +from pathlib import Path +from typing import Literal from unittest.mock import patch, Mock import pytest @@ -28,6 +30,17 @@ OperationPaginatorState, ) from aws_durable_execution_sdk_python_testing.model import StartDurableExecutionInput +from aws_durable_execution_sdk_python_testing.invoker import ( + InProcessInvoker, + LambdaInvoker, +) +from aws_durable_execution_sdk_python_testing.stores.base import ExecutionStore +from aws_durable_execution_sdk_python_testing.stores.filesystem import ( + FileSystemExecutionStore, +) +from aws_durable_execution_sdk_python_testing.stores.memory import ( + InMemoryExecutionStore, +) def test_execution_init(): @@ -1638,8 +1651,7 @@ def test_start_without_invocation_id_raises(): execution.start() -def test_complete_wait_records_updated_operation_id(): - """Wait completion happens outside the invocation and is reported on the next input.""" +def _updated_ids_execution() -> Execution: start_input = StartDurableExecutionInput( account_id="123456789012", function_name="test-function", @@ -1649,47 +1661,259 @@ def test_complete_wait_records_updated_operation_id(): execution_retention_period_days=7, invocation_id="invocation-id", ) - execution = Execution( - "test-arn", - start_input, - [ + execution = Execution("test-arn", start_input, []) + execution.start() + for operation_id in ("step-1", "wait-1"): + execution.operations.append( Operation( - operation_id="wait-1", + operation_id=operation_id, operation_type=OperationType.WAIT, status=OperationStatus.STARTED, ) - ], - ) + ) + return execution + + +def test_updated_operation_ids_list_every_operation_before_any_success(): + """Before any invocation succeeds, the service lists the invocation id + and every operation. A failed first invocation keeps that rule.""" + execution = _updated_ids_execution() + execution.touch_operation("step-1") + + execution.mark_state_delivered() + + assert execution.updated_operation_ids == [ + "invocation-id", + "step-1", + "wait-1", + ] + + +def test_updated_operation_ids_list_changes_after_the_last_success(): + """After a successful invocation, the list holds the operations touched + after the watermark it saved: a wait that completes later is listed, an + operation the handler already observed is not.""" + execution = _updated_ids_execution() + execution.touch_operation("step-1") + execution.mark_state_delivered() + execution.record_successful_invocation() execution.complete_wait("wait-1") assert execution.updated_operation_ids == ["wait-1"] -def test_record_invocation_completion_keeps_updated_operation_ids(): - """An operation that changes while the handler runs is reported on the - next invocation. So completing the invocation keeps the list; only - delivering the state in an input resets it.""" - start_input = StartDurableExecutionInput( - account_id="123456789012", - function_name="test-function", - function_qualifier="$LATEST", - execution_name="test-execution", - execution_timeout_seconds=300, - execution_retention_period_days=7, - invocation_id="invocation-id", - ) - execution = Execution("test-arn", start_input, []) - execution.updated_operation_ids = ["wait-1"] +def test_a_failed_invocation_keeps_the_watermark(): + """A failed invocation does not save its watermark. So the retry lists + again what the failed invocation observed, as the service does.""" + execution = _updated_ids_execution() + execution.mark_state_delivered() + execution.record_successful_invocation() + execution.complete_wait("wait-1") - now = datetime(2023, 1, 1, 12, 0, 0, tzinfo=timezone.utc) - execution.record_invocation_completion(now, now, "request-1") - assert execution.updated_operation_ids == ["wait-1"] + # The next invocation observes the wait and checkpoints step-1, then fails. + execution.mark_state_delivered() + execution.touch_operation("step-1") + execution.handler_seen_seq = execution.seq_counter + + assert execution.updated_operation_ids == [ + "step-1", + "wait-1", + ] + + +def test_mark_state_delivered_records_the_handler_watermark(): + """Building the input records that the handler observed the state up to + the current sequence.""" + execution = _updated_ids_execution() + execution.touch_operation("step-1") + execution.touch_operation("wait-1") + + execution.mark_state_delivered() + + assert execution.handler_seen_seq == execution.seq_counter == 2 + assert not execution.has_unseen_changes() + + +def test_record_successful_invocation_saves_the_watermark_and_resets_attempts(): + execution = _updated_ids_execution() + execution.consecutive_failed_invocation_attempts = 2 + execution.touch_operation("step-1") + execution.handler_seen_seq = execution.seq_counter + + execution.record_successful_invocation() + assert execution.last_successful_invocation_seq == 1 + assert execution.consecutive_failed_invocation_attempts == 0 + + +@pytest.mark.parametrize("watermark", [None, 0, 3]) +def test_last_successful_invocation_seq_survives_json_round_trip(watermark): + execution = _updated_ids_execution() + execution.last_successful_invocation_seq = watermark + + restored = Execution.from_json_dict(execution.to_json_dict()) + + assert restored.last_successful_invocation_seq == watermark + + +def test_legacy_json_without_watermark_lists_every_operation_once(): + """A file from an older runner has no saved watermark. Its next input + lists every operation once, as after a failed first invocation.""" + data = _updated_ids_execution().to_json_dict() + data["HandlerSeenSeq"] = 5 + data["UpdatedOperationIds"] = ["wait-1"] + del data["LastSuccessfulInvocationSeq"] + + restored = Execution.from_json_dict(data) + + assert restored.last_successful_invocation_seq is None + assert restored.updated_operation_ids == [ + "invocation-id", + "step-1", + "wait-1", + ] + + +def test_paused_first_invocation_lists_its_checkpoint_on_resume(): + """A first invocation that a pause interrupts succeeds without seeing its + last checkpoint, because a paused checkpoint returns no operations. So the + resume lists that checkpoint's operation and not the EXECUTION operation, + as the service does.""" + execution = _updated_ids_execution() execution.mark_state_delivered() + execution.touch_operation("step-1") # the paused checkpoint delivers nothing + execution.record_successful_invocation() + + assert execution.last_successful_invocation_seq == 0 + assert execution.updated_operation_ids == ["step-1"] + + +def test_undecodable_callback_result_leaves_no_change(): + execution = _updated_ids_execution() + execution.operations.append( + Operation( + operation_id="callback-1", + operation_type=OperationType.CALLBACK, + status=OperationStatus.STARTED, + callback_details=CallbackDetails(callback_id="cb-1"), + ) + ) + execution.mark_state_delivered() + execution.record_successful_invocation() + before = execution.to_json_dict() + + with pytest.raises(UnicodeDecodeError): + execution.complete_callback_success("cb-1", b"\xff") + + assert execution.to_json_dict() == before assert execution.updated_operation_ids == [] +CallbackOutcome = Literal["success", "failure", "timeout"] +CALLBACK_NOW = datetime(2026, 10, 27, tzinfo=timezone.utc) +CALLBACK_ERROR = ErrorObject( + "callback failure", "CallbackRejected", "details", ["original frame"] +) + + +def _complete_callback( + execution: Execution, outcome: CallbackOutcome, callback_id: str +) -> Operation: + if outcome == "success": + return execution.complete_callback_success( + callback_id, b'"result"', now=CALLBACK_NOW + ) + if outcome == "failure": + return execution.complete_callback_failure( + callback_id, CALLBACK_ERROR, now=CALLBACK_NOW + ) + return execution.complete_callback_timeout( + callback_id, CALLBACK_ERROR, now=CALLBACK_NOW + ) + + +@pytest.mark.parametrize("outcome", ["success", "failure", "timeout"]) +@pytest.mark.parametrize("filesystem", [False, True]) +@pytest.mark.parametrize("other_changed", [False, True]) +def test_callback_completion_is_listed_after_the_last_success( + outcome: CallbackOutcome, filesystem: bool, other_changed: bool, tmp_path: Path +) -> None: + """Each callback outcome touches the callback, so the next input lists it + once. A rejected completion changes nothing.""" + target = Operation( + operation_id="callback", + operation_type=OperationType.CALLBACK, + status=OperationStatus.STARTED, + callback_details=CallbackDetails(callback_id="callback-id"), + ) + other = Operation( + operation_id="other", + operation_type=OperationType.CALLBACK, + status=OperationStatus.SUCCEEDED, + callback_details=CallbackDetails(callback_id="other-id", result='"other"'), + ) + execution = Execution("test-arn", _make_start_input(), [target, other]) + # The previous invocation succeeded after observing the state so far. + execution.mark_state_delivered() + execution.record_successful_invocation() + if other_changed: + execution.touch_operation("other") + expected = ["callback", "other"] if other_changed else ["callback"] + touched = 2 if other_changed else 1 + + before = execution.to_json_dict() + with pytest.raises(IllegalStateException): + _complete_callback(execution, outcome, "missing") + assert execution.to_json_dict() == before + + completed = _complete_callback(execution, outcome, "callback-id") + assert execution.updated_operation_ids == expected + assert execution.operation_last_touched_seq["callback"] == touched + assert execution.token_sequence == 0 + assert ( + completed.status + is { + "success": OperationStatus.SUCCEEDED, + "failure": OperationStatus.FAILED, + "timeout": OperationStatus.TIMED_OUT, + }[outcome] + ) + assert completed.end_timestamp == CALLBACK_NOW + assert completed.callback_details is not None + assert completed.callback_details.result == ( + '"result"' if outcome == "success" else None + ) + assert completed.callback_details.error == ( + None if outcome == "success" else CALLBACK_ERROR + ) + assert execution.operations[1] == other + + store: ExecutionStore = ( + FileSystemExecutionStore.create(tmp_path) + if filesystem + else InMemoryExecutionStore() + ) + store.save(execution) + restored = store.load(execution.durable_execution_arn) + assert restored.updated_operation_ids == expected + assert restored.operations == execution.operations + for invoker in [InProcessInvoker(Mock(), Mock()), LambdaInvoker(Mock())]: + invocation_input = invoker.create_invocation_input(restored) + assert invocation_input.updated_operation_ids == expected + assert invocation_input.initial_execution_state.operations == [completed, other] + + # After the next successful invocation, a rejected second completion + # lists nothing. + restored.mark_state_delivered() + restored.record_successful_invocation() + before = restored.to_json_dict() + with pytest.raises(IllegalStateException, match="not in STARTED state"): + _complete_callback(restored, outcome, "callback-id") + assert restored.to_json_dict() == before + assert restored.updated_operation_ids == [] + + def test_function_arn_is_qualified_with_the_executed_version(): execution = Execution.new(_make_start_input()) execution.region = "ap-southeast-2" diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/executor_checkpoint_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/executor_checkpoint_test.py index 08edbac70..3b9569281 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/executor_checkpoint_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/executor_checkpoint_test.py @@ -166,6 +166,43 @@ def test_non_empty_checkpoint_returns_only_new_ops(): assert op_ids_r2 == {"C"} +def test_first_checkpoint_response_omits_operations_the_input_carried(): + """ + Building the invocation input records that the handler observed the + state up to that point. So the first checkpoint response of the next + invocation returns only B, not the wait W that changed while suspended + and that the input carried. + """ + executor, store, execution, token_0 = _make_executor_with_started_execution() + arn = execution.durable_execution_arn + executor.checkpoint_execution( + execution_arn=arn, checkpoint_token=token_0, updates=[_step_start_update("A")] + ) + + suspended = store.load(arn) + suspended.operations.append( + SvcOperation( + operation_id="W", + operation_type=OperationType.WAIT, + status=OperationStatus.SUCCEEDED, + ) + ) + suspended.touch_operation("W") + suspended.mark_state_delivered() + store.save(suspended) + + response = executor.checkpoint_execution( + execution_arn=arn, + checkpoint_token=CheckpointToken( + execution_arn=arn, token_sequence=suspended.token_sequence + ).to_str(), + updates=[_step_start_update("B")], + ) + + assert response.new_execution_state is not None + assert {op.operation_id for op in response.new_execution_state.operations} == {"B"} + + # endregion # region: Async completion between polls surfaces in next poll diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/executor_pause_resume_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/executor_pause_resume_test.py index 7158913c6..17bfd7104 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/executor_pause_resume_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/executor_pause_resume_test.py @@ -360,7 +360,7 @@ def test_paused_pending_is_accepted_only_when_a_token_was_withheld(): with pytest.raises(InvalidParameterValueException) as exc_info: executor._validate_invocation_response_and_store( # noqa: SLF001 - execution.durable_execution_arn, pending, execution, execution.seq_counter + execution.durable_execution_arn, pending, execution ) assert str(exc_info.value) == ( "Cannot return PENDING status with no pending operations." @@ -375,14 +375,14 @@ def test_paused_pending_is_accepted_only_when_a_token_was_withheld(): assert response.checkpoint_token is None assert execution.pause_state is PauseState.PAUSED_INVOCATION_DEFERRED executor._validate_invocation_response_and_store( # noqa: SLF001 - execution.durable_execution_arn, pending, execution, execution.seq_counter + execution.durable_execution_arn, pending, execution ) executor.resume_execution(execution.durable_execution_arn) with pytest.raises(InvalidParameterValueException, match="no pending operations"): executor._validate_invocation_response_and_store( # noqa: SLF001 - execution.durable_execution_arn, pending, execution, execution.seq_counter + execution.durable_execution_arn, pending, execution ) diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/executor_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/executor_test.py index 21993649e..155ecf9bb 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/executor_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/executor_test.py @@ -686,7 +686,7 @@ def test_should_retry_when_pending_response_has_no_operations( mock_execution.seq_counter = 0 mock_execution.has_pending_operations.return_value = False # No pending operations mock_execution.handler_seen_seq = 0 - mock_execution.has_changes_after.return_value = False + mock_execution.has_unseen_changes.return_value = False # Mock invoker to return pending response mock_invocation_input = Mock() @@ -734,10 +734,7 @@ def test_pending_response_is_valid_when_an_operation_completed_after_the_handler mock_execution.is_complete = False mock_execution.start_input = start_input mock_execution.consecutive_failed_invocation_attempts = 0 - mock_execution.seq_counter = 4 - mock_execution.handler_seen_seq = 2 mock_execution.has_pending_operations.return_value = False - mock_execution.has_changes_after.side_effect = lambda seq: seq == 4 mock_execution.has_unseen_changes.return_value = True mock_execution.needs_reinvoke = True @@ -759,18 +756,18 @@ def test_pending_response_is_valid_when_an_operation_completed_after_the_handler handler = mock_scheduler.call_later.call_args_list[-1][0][0] asyncio.run(handler()) - mock_execution.has_changes_after.assert_called_once_with(4) + mock_execution.record_successful_invocation.assert_called_once_with() assert mock_execution.consecutive_failed_invocation_attempts == 0 # timeout + initial invocation + the re-invoke, with no retry delay assert mock_scheduler.call_later.call_count == 3 assert mock_scheduler.call_later.call_args_list[-1].kwargs["delay"] == 0 -def test_begin_invocation_baseline_predates_a_completion_queued_behind_it( +def test_begin_invocation_watermark_predates_a_completion_queued_behind_it( executor, mock_store, mock_invoker, start_input ): - """The baseline is the counter when the input was built. A completion - applied to the same object afterward lies past it.""" + """Building the input sets the handler's watermark to the counter. A + completion applied to the same object afterward lies past it.""" execution = Execution.new(start_input) execution.operations.append( Operation( @@ -784,20 +781,21 @@ def test_begin_invocation_baseline_predates_a_completion_queued_behind_it( claim = executor._begin_invocation(execution.durable_execution_arn) # noqa: SLF001 assert claim is not None - _, _, baseline = claim - assert baseline == execution.seq_counter + watermark = execution.handler_seen_seq + assert watermark == execution.seq_counter + assert not execution.has_unseen_changes() execution.complete_chained_invoke("invoke-1", OperationStatus.SUCCEEDED, result="1") - assert execution.seq_counter > baseline - assert execution.has_changes_after(baseline) + assert execution.handler_seen_seq == watermark + assert execution.has_unseen_changes() def test_pending_response_with_only_input_time_state_is_still_an_error( executor, mock_store, start_input ): - """Operations the invocation input already delivered lie past the - handler's watermark but not past the invocation's baseline, so a - no-pending PENDING is rejected rather than accepted without a re-invoke.""" + """Operations the invocation input already delivered are covered by the + watermark the input set, so a no-pending PENDING is rejected rather than + accepted without a re-invoke.""" execution = Execution.new(start_input) execution.operations.append( Operation( @@ -808,7 +806,7 @@ def test_pending_response_with_only_input_time_state_is_still_an_error( ) execution.complete_chained_invoke("invoke-1", OperationStatus.SUCCEEDED, result="1") assert execution.has_unseen_changes() - baseline = execution.seq_counter + execution.mark_state_delivered() with pytest.raises( InvalidParameterValueException, @@ -818,14 +816,13 @@ def test_pending_response_with_only_input_time_state_is_still_an_error( execution.durable_execution_arn, DurableExecutionInvocationOutput(status=InvocationStatus.PENDING), execution, - baseline, ) def test_pending_response_is_an_error_when_the_handler_saw_the_completion( executor, mock_store, start_input ): - """A completion after the baseline that a checkpoint response already + """A completion after the input that a checkpoint response already delivered leaves nothing to re-invoke for, so PENDING is rejected.""" execution = Execution.new(start_input) execution.operations.append( @@ -835,7 +832,7 @@ def test_pending_response_is_an_error_when_the_handler_saw_the_completion( status=OperationStatus.STARTED, ) ) - baseline = execution.seq_counter + execution.mark_state_delivered() execution.complete_chained_invoke("invoke-1", OperationStatus.SUCCEEDED, result="1") execution.handler_seen_seq = execution.seq_counter @@ -847,7 +844,6 @@ def test_pending_response_is_an_error_when_the_handler_saw_the_completion( execution.durable_execution_arn, DurableExecutionInvocationOutput(status=InvocationStatus.PENDING), execution, - baseline, ) @@ -3680,10 +3676,9 @@ def test_a_completion_during_the_invocation_reaches_the_next_input( mock_scheduler, mock_invoker, mock_checkpoint_processor ): """A chained target completes while the parent handler is still - running. The service reports it in the next invocation's - UpdatedOperationIds, because the handler never observed it. So must - the runner: completing the current invocation keeps the id, and the - next input carries it.""" + running, and no checkpoint response delivers it. The service lists it + in the next invocation's UpdatedOperationIds, because the handler never + observed it. So must the runner.""" store = InMemoryExecutionStore() parent, _child = _linked_parent_and_child(store) real_executor = Executor( @@ -3692,25 +3687,22 @@ def test_a_completion_during_the_invocation_reaches_the_next_input( arn = parent.durable_execution_arn now = datetime.now(UTC) - # The parent handler is running: the child's completion lands first. + # The parent's input is built, then the child's completion lands. + running = store.load(arn) + running.mark_state_delivered() + store.save(running) real_executor._apply_chained_invoke_completion( # noqa: SLF001 arn, "invoke-1", OperationStatus.SUCCEEDED, '{"squared": 16}', None ) - assert store.load(arn).updated_operation_ids == ["invoke-1"] - # Then the current invocation returns. - store.load(arn).record_invocation_completion(now, now, "request-1") - store.save(store.load(arn)) - assert store.load(arn).updated_operation_ids == ["invoke-1"] + # Then the current invocation returns successfully. + finished = store.load(arn) + finished.record_invocation_completion(now, now, "request-1") + finished.record_successful_invocation() + store.save(finished) - # The next input carries the id, and delivering it resets the list. - execution = store.load(arn) - execution.begin_new_invocation() - invocation_input = mock_invoker.create_invocation_input(execution=execution) - mock_invoker.create_invocation_input.assert_called_once() - assert invocation_input is mock_invoker.create_invocation_input.return_value - execution.mark_state_delivered() - assert execution.updated_operation_ids == [] + # The next input lists the completion. + assert store.load(arn).updated_operation_ids == ["invoke-1"] def test_a_target_exception_that_lands_after_shutdown_is_dropped(executor): diff --git a/packages/aws-durable-execution-sdk-python-testing/tests/invoker_test.py b/packages/aws-durable-execution-sdk-python-testing/tests/invoker_test.py index 0286c1c00..9b164e8bb 100644 --- a/packages/aws-durable-execution-sdk-python-testing/tests/invoker_test.py +++ b/packages/aws-durable-execution-sdk-python-testing/tests/invoker_test.py @@ -245,7 +245,7 @@ def test_in_process_invoker_create_invocation_input(): invocation_id="test-invocation-id", ) execution = Execution.new(input_data) - execution.updated_operation_ids = ["wait-1"] + execution.start() invocation_input = invoker.create_invocation_input(execution) @@ -253,7 +253,8 @@ def test_in_process_invoker_create_invocation_input(): assert invocation_input.durable_execution_arn == execution.durable_execution_arn assert invocation_input.checkpoint_token is not None assert isinstance(invocation_input.initial_execution_state, InitialExecutionState) - assert invocation_input.updated_operation_ids == ["wait-1"] + # The first invocation lists the execution operation, as the service does. + assert invocation_input.updated_operation_ids == ["test-invocation-id"] assert invocation_input.service_client is service_client diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py index d23b6aad9..5015faced 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/plugin.py @@ -311,13 +311,17 @@ class InvocationStartInfo(InvocationInfo): compare=False, hash=False, ) - """Operations updated externally while this execution was suspended. - - A wait timer that expired, a callback that was delivered, or a chained - invoke that completed between the previous invocation and this one. This is - the subset of :attr:`InvocationInfo.operations` named by the durable - invocation input's ``UpdatedOperationIds``, so it is empty on the first - invocation. + """Operations updated since the last successful invocation. + + This is the subset of :attr:`InvocationInfo.operations` named by the + durable invocation input's ``UpdatedOperationIds``. Until an invocation + succeeds, the service lists every operation, starting with the EXECUTION + operation. After that, it lists the operations that changed after the + state the last successful invocation observed: a wait timer that expired, + a callback that was delivered, or a chained invoke that completed. A failed + invocation does not move that point, so its retry lists again every change + since the last successful invocation, including the changes the failed + invocation observed or made. Excluded from ``repr``, ``__eq__`` and ``__hash__`` like :attr:`InvocationInfo.operations`. @@ -1612,7 +1616,8 @@ def on_invocation_start( operations_provider: Returns the current checkpointed operation map, converted here into the plugin's ``OperationInfo`` view. updated_operation_ids: Operation ids from the invocation input's - ``UpdatedOperationIds`` -- those updated while suspended. + ``UpdatedOperationIds`` -- those updated since the last + successful invocation. """ aws_request_id = lambda_context.aws_request_id if lambda_context else None self._operations_provider = ( @@ -1818,8 +1823,9 @@ def on_operation_update( ): """Run the end hooks for operation updates here, and queue the change hook. - Called for an operation that changed while the execution was suspended, - by the thread that replays it; the end hook runs on that thread. The + Called for an operation that the invocation input lists in + ``UpdatedOperationIds``, by the thread that replays it; the end hook + runs on that thread. The change hook, built only when both operation maps are given, goes to the plugin change thread. Checkpoint responses during the invocation take the other path: the checkpoint thread records their hooks with diff --git a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py index ecf4e1b7f..0518f0f87 100644 --- a/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py +++ b/packages/aws-durable-execution-sdk-python/src/aws_durable_execution_sdk_python/state.py @@ -682,11 +682,15 @@ def emit_child_context_end_hook( ) def is_operation_updated_since_last_invocation(self, operation_id: str) -> bool: - """Return True if an operation changed while this execution was suspended.""" + """Return True if the invocation input lists the operation as updated. + + ``UpdatedOperationIds`` names the operations that changed since the + last successful invocation. + """ return operation_id in self._updated_operation_ids def emit_operation_update_hook(self, operation: Operation) -> None: - """Fire the plugin update hook for an operation changed during suspend. + """Fire the plugin update hook for an operation the input lists as updated. This method is safe to call for any operation. It emits only for operations listed in UpdatedOperationIds.