Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -163,13 +163,13 @@ the handler would wait on a lane it needs in order to make progress.
`_invoke_handler` splits the invocation into three steps.

1. Begin. A task on the lane claims the gate from `PRE_INVOKE` to
`INVOKING` and builds the invocation input.
`INVOKING`, builds the invocation input, and sets `handler_seen_seq`.
2. Invoke. The blocking call to the function runs off the lane, bounded
by the invocation timeout. While it blocks, the handler's checkpoint
and get state tasks run on the now free lane.
3. Finish. A task on the lane records the response and releases the
gate. It moves `PENDING` to `PRE_INVOKE`, or a terminal result to
`COMPLETED`.
`COMPLETED`. A valid response saves `last_successful_invocation_seq`.

```mermaid
sequenceDiagram
Expand Down Expand Up @@ -225,19 +225,27 @@ the checkpoint write transaction.
- Idempotent replays advance neither.
- Pure reads advance neither.

## The watermark and delta

`Execution.handler_seen_seq` records the highest `seq_counter` the
handler has been told about. The handler learns of operations through
`InitialExecutionState` on an invocation, or through the `operations`
list in a checkpoint response. The delta for the next response is the
operations whose `operation_last_touched_seq` is greater than
`handler_seen_seq`.

The watermark advances only when a checkpoint response is built. It never
advances when invocation input is built. That is what makes retries
correct. A failed invocation never moves the watermark. So the retry
delivers the same state again.
## The watermarks and the delta

The runner keeps two watermarks over `seq_counter`.

1. `Execution.handler_seen_seq` is the highest sequence the handler has
observed. Building the invocation input sets it to `seq_counter`,
because the input carries the state as of that point. Each checkpoint
response raises it to the highest touch it returns. A paused checkpoint
returns no operations and leaves it. The delta for the next checkpoint
response is the operations whose `operation_last_touched_seq` is
greater than `handler_seen_seq`.
2. `Execution.last_successful_invocation_seq` is `handler_seen_seq` as it
stood when the last successful invocation ended. A successful
invocation returned a valid `PENDING`, `SUCCEEDED` or `FAILED` output.
A failed invocation leaves the value unchanged.

The next input's `UpdatedOperationIds` lists every operation touched after
`last_successful_invocation_seq`, or every operation while it is `None`.
The list is derived from these values and is not stored. So every change
that touches an operation is listed, and a retry lists again every change
since the last successful invocation, as the service does.

```
seq_counter handler_seen_seq delta next checkpoint
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,6 @@ def __init__(
self.updates: list[OperationUpdate] = []
self.update_timestamps: list[datetime] = []
self.invocation_completions: list[InvocationCompletedDetails] = []
self.updated_operation_ids: list[str] = []
# Two monotonic counters. `token_sequence` is the checkpoint-response
# version carried on the wire; `seq_counter` is the internal
# event counter bumped on every state-affecting mutation.
Expand All @@ -142,6 +141,12 @@ def __init__(
# about (via InitialExecutionState or a successful checkpoint
# response). Filters the "unseen" delta on the next checkpoint.
self.handler_seen_seq: int = 0
# `handler_seen_seq` as it stood when the last successful invocation
# ended. The next input lists every operation touched after it. A
# failed invocation leaves it unchanged, so the retry lists those
# operations again. None means that no invocation has succeeded yet,
# so the input lists every operation.
self.last_successful_invocation_seq: int | None = None
# Per-op "last touched at `seq_counter`" bookkeeping. Sparse:
# ops not in the dict are treated as touch-seq 0.
self.operation_last_touched_seq: dict[str, int] = {}
Expand Down Expand Up @@ -271,10 +276,10 @@ def to_json_dict(self) -> dict[str, Any]:
"InvocationCompletions": [
completion.to_json_dict() for completion in self.invocation_completions
],
"UpdatedOperationIds": self.updated_operation_ids,
"TokenSequence": self.token_sequence,
"SeqCounter": self.seq_counter,
"HandlerSeenSeq": self.handler_seen_seq,
"LastSuccessfulInvocationSeq": self.last_successful_invocation_seq,
"OperationLastTouchedSeq": dict(self.operation_last_touched_seq),
"OperationSizeBytes": dict(self.operation_size_bytes),
"ChainedInvokeChildren": dict(self.chained_invoke_children),
Expand Down Expand Up @@ -321,7 +326,6 @@ def from_json_dict(cls, data: dict[str, Any]) -> Execution:
InvocationCompletedDetails.from_json_dict(item)
for item in data.get("InvocationCompletions", [])
]
execution.updated_operation_ids = list(data.get("UpdatedOperationIds", []))
# NOTE: prior format included a "UsedTokens" set. Current
# removed it (the Executor no longer consults it — token
# validity is via _is_current_token against token_sequence).
Expand All @@ -330,6 +334,11 @@ def from_json_dict(cls, data: dict[str, Any]) -> Execution:
# Safe defaults for fields added after the original schema.
execution.seq_counter = data.get("SeqCounter", 0)
execution.handler_seen_seq = data.get("HandlerSeenSeq", 0)
# A file from an older runner has no saved watermark. Its next input
# then lists every operation once, as after a failed first invocation.
execution.last_successful_invocation_seq = data.get(
"LastSuccessfulInvocationSeq"
)
execution.operation_last_touched_seq = dict(
data.get("OperationLastTouchedSeq", {})
)
Expand Down Expand Up @@ -458,12 +467,9 @@ def has_pending_operations(self, execution: Execution) -> bool:
def has_unseen_changes(self) -> bool:
"""True if any operation changed after the handler's last
observed sequence watermark."""
return self.has_changes_after(self.handler_seen_seq)

def has_changes_after(self, seq: int) -> bool:
"""True if any operation changed after ``seq_counter`` was ``seq``."""
return any(
touched > seq for touched in self.operation_last_touched_seq.values()
touched > self.handler_seen_seq
for touched in self.operation_last_touched_seq.values()
)

def record_invocation_completion(
Expand All @@ -479,23 +485,46 @@ def record_invocation_completion(
)

def mark_state_delivered(self) -> None:
"""Reset the list of changed operations once an invocation's input
carries them.

The service reports, on each invocation, the operations that
changed after the last state the handler observed. The runner
delivers state in the invocation input. So the list is reset when
that input is built, not when the invocation completes: an
operation that completes while the handler is still running is
reported on the next invocation.
"""Record that the handler has observed the state up to ``seq_counter``.

The runner calls this when it builds an invocation input. The input
carries the state as of ``seq_counter``. So the next checkpoint
response returns only the operations touched after it.
"""
self.handler_seen_seq = self.seq_counter

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Codex AI review · Finding arf_v1_6huvwhctwuu2th554g3cfkucyi

[P2] Establish evidence for the service-parity assumptions

Rule 6, “Mark unverified assumptions,” requires cited evidence or an explicit unverified label. The description says input-state omission from the first checkpoint response, first-invocation EXECUTION inclusion, and callback inclusion were checked against the service implementation, but supplies no identifiable source reference or validation artifact for those checks. The new local tests encode these same assumptions, so they cannot establish service parity. Per Rule 9, return to the design phase (Rule 8, step 6): cite reviewable evidence for these behaviors, or mark them unverified and obtain a maintainer decision on the necessary validation.

To suppress repeats, reply /ai verdict deferred, /ai verdict false-positive, or /ai verdict out-of-scope on the first nonblank line. Add rationale on later lines. A new /ai verdict accepted allows future repeats again.


def record_successful_invocation(self) -> None:
"""Record that an invocation returned a valid response.

A valid response is a valid ``PENDING``, ``SUCCEEDED`` or ``FAILED``
output. It resets the failed-attempt count and saves the handler's
watermark. An invocation that raised, timed out or returned an invalid
output leaves the watermark unchanged. So the next input lists again
every change since the last successful invocation, including the
changes the failed invocation observed or made.
"""
self.updated_operation_ids = []
self.consecutive_failed_invocation_attempts = 0
self.last_successful_invocation_seq = self.handler_seen_seq

def _record_updated_operation(self, operation_id: str) -> None:
"""Remember an operation that changed since the handler last saw
the state."""
if operation_id not in self.updated_operation_ids:
self.updated_operation_ids.append(operation_id)
@property
def updated_operation_ids(self) -> list[str]:
"""The ``UpdatedOperationIds`` of the next invocation input.

Until an invocation succeeds, every operation, the EXECUTION operation
first. After that, every operation touched after the watermark the
last successful invocation saved. A change the handler observed in a
successful invocation, in its input or in a checkpoint response, is
not listed again. The list is derived from the touch sequences, so
every state change is covered once it is touched.
"""
watermark = self.last_successful_invocation_seq
return [
operation.operation_id
for operation in self.operations
if watermark is None
or self.operation_last_touched_seq.get(operation.operation_id, 0)
> watermark
]

def complete_success(self, result: str | None, now: datetime | None = None) -> None:
"""Complete execution successfully (DecisionType.COMPLETE_WORKFLOW_EXECUTION)."""
Expand Down Expand Up @@ -580,7 +609,6 @@ def complete_wait(
status=OperationStatus.SUCCEEDED,
end_timestamp=now if now is not None else real_now(),
)
self._record_updated_operation(operation_id)
return self.operations[index]

def complete_retry(self, operation_id: str) -> Operation:
Expand Down Expand Up @@ -614,7 +642,6 @@ def complete_retry(self, operation_id: str) -> Operation:

# Assign
self.operations[index] = updated_operation
self._record_updated_operation(operation_id)
return updated_operation

def complete_due_operations(self, now: datetime) -> bool:
Expand Down Expand Up @@ -678,13 +705,16 @@ def complete_callback_success(
raise IllegalStateException(msg)

with self._state_lock:
self.touch_operation(operation.operation_id)
# Decode first: a result that fails to decode must leave no
# touch behind, or the next input would list a change that
# never happened.
updated_callback_details = None
if operation.callback_details:
updated_callback_details = replace(
operation.callback_details,
result=result.decode() if result else None,
)
self.touch_operation(operation.operation_id)

self.operations[index] = replace(
operation,
Expand Down Expand Up @@ -792,7 +822,6 @@ def complete_chained_invoke(
self.operation_size_bytes[operation_id] = len(
(result if result is not None else "").encode()
) + (len(json.dumps(error.to_dict())) if error is not None else 0)
self._record_updated_operation(operation_id)
return self.operations[index]

def executed_version(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1403,13 +1403,9 @@ def _validate_invocation_response_and_store(
execution_arn: str,
response: DurableExecutionInvocationOutput,
execution: Execution,
invocation_seq: int | None = None,
):
"""Validate response status and save it to the store if fine.

``invocation_seq`` is the execution's ``seq_counter`` when the
invocation's input was built.

Raises:
InvalidParameterValueException: If the response status is invalid.
IllegalStateException: If the response status is valid but the execution is already completed.
Expand Down Expand Up @@ -1474,12 +1470,7 @@ def _validate_invocation_response_and_store(
if (
not execution.has_deferred_invocation
and not execution.has_pending_operations(execution)
and not (
invocation_seq is not None
and execution.has_changes_after(
max(invocation_seq, execution.handler_seen_seq)
)
)
and not execution.has_unseen_changes()
):
msg_pending_ops: str = (
"Cannot return PENDING status with no pending operations."
Expand All @@ -1495,18 +1486,17 @@ def _validate_invocation_response_and_store(

def _begin_invocation(
self, execution_arn: str
) -> tuple[Execution, DurableExecutionInvocationInput, int] | None:
) -> tuple[Execution, DurableExecutionInvocationInput] | None:
"""Claim the invocation gate and build the handler input.

Returns the execution, its invocation input, and the execution's
``seq_counter`` at that moment when this call claims the gate
(PRE_INVOKE -> INVOKING); returns None when the execution is
already complete, the gate is COMPLETED, or another invocation
is in flight (recording a deferred re-invoke).
Returns the execution and its invocation input when this call
claims the gate (PRE_INVOKE -> INVOKING); returns None when the
execution is already complete, the gate is COMPLETED, or another
invocation is in flight (recording a deferred re-invoke).

The counter is captured here, on the execution's serial lane,
because a completion queued behind this call mutates the same
object before the caller resumes.
Building the input records, on the execution's serial lane, the
sequence the handler has now observed. A completion queued behind
this call is touched after it, so it counts as unseen.
"""
execution = self._store.load(execution_arn)

Expand Down Expand Up @@ -1553,15 +1543,14 @@ def _begin_invocation(
invocation_input = self._invoker.create_invocation_input(execution=execution)
execution.mark_state_delivered()
self._store.save(execution)
return execution, invocation_input, execution.seq_counter
return execution, invocation_input

def _finish_invocation(
self,
execution_arn: str,
invoke_response: InvokeResponse,
invocation_start: datetime,
invocation_end: datetime,
invocation_seq: int | None = None,
) -> None:
"""Apply a completed handler invocation.

Expand Down Expand Up @@ -1593,7 +1582,7 @@ def _finish_invocation(
response = invoke_response.invocation_output
try:
self._validate_invocation_response_and_store(
execution_arn, response, execution, invocation_seq
execution_arn, response, execution
)
except (InvalidParameterValueException, IllegalStateException) as e:
logger.warning(
Expand All @@ -1615,9 +1604,10 @@ def _finish_invocation(
self._fail_workflow(execution_arn, error_obj)
return

# A clean invocation resets the retry counter.
# A clean invocation resets the retry counter and saves the
# handler's watermark for the next input's updated operations.
reset_target = self._store.load(execution_arn)
reset_target.consecutive_failed_invocation_attempts = 0
reset_target.record_successful_invocation()
self._store.save(reset_target)

# Release the gate: terminal -> COMPLETED, otherwise PRE_INVOKE
Expand Down Expand Up @@ -1725,7 +1715,7 @@ async def invoke() -> None:
)
if claim is None:
return
execution, invocation_input, invocation_seq = claim
execution, invocation_input = claim

invocation_start = self._clock.now()
invoke_response = await asyncio.wait_for(
Expand Down Expand Up @@ -1757,7 +1747,6 @@ async def invoke() -> None:
invoke_response,
invocation_start,
invocation_end,
invocation_seq,
)
),
)
Expand Down
Loading
Loading