From 02c24a410689b07d4118cb1e743d1c990fa42b63 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Tue, 3 Nov 2026 21:57:50 +0000 Subject: [PATCH 1/9] feat(extras): suspend from the durable function vm.invoke(..., suspend_after=True) now calls SuspendMicrovm itself, inside the job's child context, after the job's result arrives (success or failure). A failed call logs a warning and the job still returns. The session then marks the MicroVM as possibly suspended, so the next job checks the state and resumes it. - No new step and no new checkpoint. - The job request no longer carries suspendAfter, and the session no longer reads the X-Microvm-Suspend-After answer header. - The function's role needs lambda:SuspendMicrovm. The MicroVM's role needs it only for auto_suspend_on_idle. --- .../examples-catalog.json | 2 +- .../microvm/microvm_session_suspend_after.py | 10 +- .../test/README.md | 2 +- .../README.md | 33 +-- .../microvm/config.py | 68 +++--- .../microvm/job.py | 19 +- .../microvm/lifecycle.py | 4 +- .../microvm/operation.py | 40 ++-- .../microvm/request.py | 33 +-- .../microvm/session.py | 50 +---- .../tests/conftest.py | 22 +- .../tests/microvm/job_test.py | 37 +--- .../tests/microvm/request_test.py | 55 +---- .../tests/microvm/session_test.py | 205 ++++++++++-------- .../tests/microvm/worker_composed_test.py | 131 ++--------- 15 files changed, 251 insertions(+), 460 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-examples/examples-catalog.json b/packages/aws-durable-execution-sdk-python-examples/examples-catalog.json index e6c3ce461..8a8169353 100644 --- a/packages/aws-durable-execution-sdk-python-examples/examples-catalog.json +++ b/packages/aws-durable-execution-sdk-python-examples/examples-catalog.json @@ -837,7 +837,7 @@ }, { "name": "MicroVM Session Suspend After", - "description": "Runs two jobs in one Lambda MicroVM that its worker suspends after the first job", + "description": "Runs two jobs in one Lambda MicroVM that the function suspends after the first job", "handler": "microvm_session_suspend_after.handler", "integration": true, "usesMicrovm": true, diff --git a/packages/aws-durable-execution-sdk-python-examples/src/microvm/microvm_session_suspend_after.py b/packages/aws-durable-execution-sdk-python-examples/src/microvm/microvm_session_suspend_after.py index acf42d44d..6f5ccda4d 100644 --- a/packages/aws-durable-execution-sdk-python-examples/src/microvm/microvm_session_suspend_after.py +++ b/packages/aws-durable-execution-sdk-python-examples/src/microvm/microvm_session_suspend_after.py @@ -1,4 +1,4 @@ -"""Run two jobs in one Lambda MicroVM that the worker suspends after the first. +"""Run two jobs in one Lambda MicroVM that the function suspends after the first. The MicroVM runs the image that scripts/ensure_microvm_image.py builds from microvm-image/. The image's routes /write and /read write and read a file in @@ -6,7 +6,8 @@ next job as soon as the MicroVM resumes. The session does not suspend when idle. The first job sets suspend_after, so -the worker suspends the MicroVM after that job. A callback named gate holds +the durable function calls SuspendMicrovm after that job's result arrives. +The function's role needs lambda:SuspendMicrovm on the image. A callback named gate holds the second job. The caller completes the callback with SendDurableExecutionCallbackSuccess to start the second job. """ @@ -64,8 +65,7 @@ def handler(event: dict[str, Any], context: DurableContext) -> Any: key: str = event["key"] def write_gate_read(vm: MicrovmSession, session_context: DurableContext) -> Any: - # The worker suspends the MicroVM when this job has ended and its - # result is reported. + # The session suspends the MicroVM when this job's result arrives. written = vm.invoke( session_context, "write", @@ -83,7 +83,7 @@ def write_gate_read(vm: MicrovmSession, session_context: DurableContext) -> Any: # 1. The first job has ended before the gate starts. # 2. The second job starts only after the caller completes the gate. # 3. So no job runs while the gate is pending. - # 4. So the worker suspends the MicroVM after the first job. + # 4. So the session suspends the MicroVM after the first job. # 5. So a caller that sees the MicroVM SUSPENDED while the gate is # pending sees a suspend between the two jobs. # The callback result carries no data. So the handler ignores it. diff --git a/packages/aws-durable-execution-sdk-python-examples/test/README.md b/packages/aws-durable-execution-sdk-python-examples/test/README.md index ec4ab5079..bbf236050 100644 --- a/packages/aws-durable-execution-sdk-python-examples/test/README.md +++ b/packages/aws-durable-execution-sdk-python-examples/test/README.md @@ -155,7 +155,7 @@ def test_my_example(durable_runner): - Pass a cloud-test input as an object, such as a `dict`. The cloud runner encodes the input with `json.dumps`. So a string that already holds JSON is encoded again, and the handler receives a `str` instead of a `dict`. - Start a MicroVM test with `run_async`, and read its result with `wait_for_result`. The synchronous `run` uses botocore's 60-second read timeout, and botocore sends the Invoke again after a read timeout. Each resend starts a new execution, which launches its own MicroVM. - `test_microvm_session_suspend.py` checks that an idle session MicroVM suspends and resumes. The function's role needs `lambda:GetMicrovm` and `lambda:ResumeMicrovm`, and the MicroVM execution role needs `lambda:SuspendMicrovm` on the image. -- `test_microvm_session_suspend_after.py` checks that the worker suspends a session MicroVM after a job with `suspend_after=True`, in a session without auto-suspend, and that the next job resumes it. It needs the same permissions as `test_microvm_session_suspend.py`. +- `test_microvm_session_suspend_after.py` checks that the durable function suspends a session MicroVM after a job with `suspend_after=True`, in a session without auto-suspend, and that the next job resumes it. The function's role needs `lambda:GetMicrovm`, `lambda:ResumeMicrovm`, and `lambda:SuspendMicrovm` on the image. The MicroVM execution role needs no suspend permission. ### Cloud-Only Tests `@pytest.mark.durable_execution(..., cloud_only=True)` marks a test that needs a service the local runner cannot provide, such as a MicroVM. Local mode skips the test with the reason `cloud_only: runs only against the deployed function`. Cloud mode runs it. diff --git a/packages/aws-durable-execution-sdk-python-extras/README.md b/packages/aws-durable-execution-sdk-python-extras/README.md index 6b1e287cf..9b67f0c3a 100644 --- a/packages/aws-durable-execution-sdk-python-extras/README.md +++ b/packages/aws-durable-execution-sdk-python-extras/README.md @@ -272,7 +272,7 @@ The worker must support `auto_suspend_on_idle`. The JavaScript worker [`@aws/dur ### Suspending after a job -`MicrovmInvokeOptions(suspend_after=True)` asks the worker to suspend the MicroVM after this job. It is an option of one job, not of the session. `microvm` raises `ValidationError` for it, because `microvm` terminates its MicroVM after its one job. A value that is not a `bool` raises `ValidationError` too, before the job's durable operations start. +`MicrovmInvokeOptions(suspend_after=True)` makes the durable function suspend the MicroVM after this job. It is an option of one job, not of the session. `microvm` raises `ValidationError` for it, because `microvm` terminates its MicroVM after its one job. A value that is not a `bool` raises `ValidationError` too, before the job's durable operations start. ```python build = vm.invoke( @@ -299,13 +299,20 @@ vm.invoke( What happens: -1. The job request carries `"suspendAfter": true`. The job document stays at version 1, and the field is omitted for every other job. The run hook payload never carries it. -2. The worker acknowledges the field with the header `X-Microvm-Suspend-After: accepted` in its 202 answer. -3. The worker suspends the MicroVM when the job has ended and its outcome is reported, and no other job runs. A job that ends while another job runs leaves the request pending. The worker then suspends the MicroVM when the last running job ends. -4. The worker suspends the MicroVM that its `run` hook named. It never uses the MicroVM ID of a job request. -5. Each later job of the session handle checks the MicroVM state before delivery, and resumes a suspended MicroVM, as with `auto_suspend_on_idle`. The `suspend_after` job itself checks the state too, because the record is set before the job starts. +1. The session calls `SuspendMicrovm` with the session's MicroVM ID when the job's callback result arrives, whether the job succeeded or failed. The call runs inside the job's child context, before the child context returns. A wait for the callback does not suspend the MicroVM. +2. It adds no step and no checkpoint. The child context's own completion checkpoint covers it. On replay, a completed child context does not run again, so the session sends no second suspend. +3. A failed `SuspendMicrovm` call logs a warning and does not fail the job. The next job already checks the state and resumes the MicroVM when needed. +4. The job request carries no field for it, and the worker does nothing for it. The worker needs no change, and the MicroVM's role needs no suspend permission. +5. The session records that the MicroVM may be suspended. Each later job of the session handle checks the MicroVM state before delivery, and resumes a suspended MicroVM, as with `auto_suspend_on_idle`. The `suspend_after` job itself checks the state too, because the record is set before the job starts. -The setting works with or without `auto_suspend_on_idle`. With both, the worker suspends right after the job, and the idle time applies after each resume. +The setting works with or without `auto_suspend_on_idle`. With both, the function suspends right after the job, and the worker's idle time applies after each resume. + +The call is not a durable operation, so these rules are yours to follow. The session adds no guard for them: + +1. Do not use it on a job that runs next to other jobs of the session (`map`, `parallel`, promise combinators). Those jobs freeze until the next job resumes the MicroVM, and can hit their heartbeat timeout. +2. The suspend runs inside the job's child context, so it follows the child context replay rules. A result larger than the child context checkpoint limit makes the child context run again on replay, and each run suspends again. +3. A retry after the Lambda invocation fails can send the suspend again. Suspending a suspended MicroVM succeeds. +4. Adding or removing `suspend_after` changes what the session does on replay. Change it only through a new function version. The record that a job set `suspend_after` is kept in memory only. A replay of a completed job calls `vm.invoke` again, so it sets the record again. A replay of a completed `map` item, `parallel` branch, or child context does not run its function again. So the next job can skip the state check of a suspended MicroVM. For that case, every session job checks the state after a sign of a suspended MicroVM: @@ -321,22 +328,20 @@ A session with `auto_suspend_on_idle` or an `idle_policy`, and a handle after a Use `suspend_after` only on the job before a long wait, such as a durable wait or a callback: - Not on back-to-back jobs. Each suspend writes a snapshot, and each resume adds about 1 second before the next job. -- Not on `map` items. A request waits until no job runs. So a `map` with `suspend_after` on its items suspends and resumes between items that do not overlap. +- Not on `map` items or other jobs that run at the same time. The suspend freezes the jobs that still run. - Not on the last job. The session terminates the MicroVM after its handler returns, so the snapshot is wasted. It needs: 1. The `resume` hook in the image. Without it, the worker refuses the next job with HTTP 503 for up to 30 seconds after the suspend. 2. A `heartbeat_timeout` on the next job of at least its `retry_window` plus 30 seconds. The heartbeat timeout starts before delivery, so it includes the resume and the refusal. -3. `lambda:GetMicrovm` and `lambda:ResumeMicrovm` in the function's role. -4. `lambda:SuspendMicrovm` on the image in the MicroVM's role. That permission lets code in any MicroVM of the image suspend any other. So use one image per trust boundary. -5. A worker that supports the field. An older worker ignores it, and the MicroVM keeps running. Each execution environment then logs one warning per session: `the worker in MicroVM does not support suspendAfter. The MicroVM keeps running after the job.` +3. `lambda:GetMicrovm`, `lambda:ResumeMicrovm`, and `lambda:SuspendMicrovm` on the image in the function's role. The MicroVM's role needs `lambda:SuspendMicrovm` only for `auto_suspend_on_idle`. -A suspend that the service refuses for now, such as a throttle or expired credentials, is not retried without `auto_suspend_on_idle`. The MicroVM then keeps running until a later suspend. With `auto_suspend_on_idle`, the worker tries again after the idle time, as it does for an idle suspend. +A failed suspend is not retried. The MicroVM then keeps running until the next job, the idle suspend, or the end of the session. The warning names `lambda:SuspendMicrovm` for a missing permission. ### Session permissions -A session needs the permissions of `microvm` with HTTP delivery: `lambda:RunMicrovm`, `lambda:TerminateMicrovm`, `lambda:CreateMicrovmAuthToken`, `lambda:PassNetworkConnector`, and `iam:PassRole`. Suspending adds the permissions listed above. Grant `lambda:GetMicrovm` to every session's function role. Without it, the check after a sign of a suspended MicroVM logs a warning, and the job keeps its retries. +A session needs the permissions of `microvm` with HTTP delivery: `lambda:RunMicrovm`, `lambda:TerminateMicrovm`, `lambda:CreateMicrovmAuthToken`, `lambda:PassNetworkConnector`, and `iam:PassRole`. Suspending adds the permissions listed above: `lambda:SuspendMicrovm` in the function's role for `suspend_after`, and in the MicroVM's role for `auto_suspend_on_idle`. Grant `lambda:GetMicrovm` to every session's function role. Without it, the check after a sign of a suspended MicroVM logs a warning, and the job keeps its retries. ### Subtypes @@ -406,7 +411,7 @@ The MicroVM operations handle the network, the service, and the worker lifecycle - The idle time is at most the session `timeout`. A value that breaks any one condition raises `ValidationError` when `microvm_session` is called, before any durable operation starts. See [Suspending an idle MicroVM](#suspending-an-idle-microvm). -4. Set `suspend_after` only on the job before a long wait, and give the next job a `heartbeat_timeout` of at least its `retry_window` plus 30 seconds. On back-to-back jobs and `map` items, each suspend and resume adds latency and a snapshot. A shorter heartbeat timeout can fail the next job with `MicrovmTimeoutError` while the MicroVM resumes. Keep `lambda:GetMicrovm` and `lambda:ResumeMicrovm` in the function's role after a deploy that removes `suspend_after`, until every session that ran such a job has ended. See [Suspending after a job](#suspending-after-a-job). +4. Set `suspend_after` only on the job before a long wait. Never set it on a job that runs next to other jobs of the session. Add `lambda:SuspendMicrovm` to the function's role, and change `suspend_after` only through a new function version. Give the next job a `heartbeat_timeout` of at least its `retry_window` plus 30 seconds. On back-to-back jobs, each suspend and resume adds latency and a snapshot. A shorter heartbeat timeout can fail the next job with `MicrovmTimeoutError` while the MicroVM resumes. Keep `lambda:GetMicrovm` and `lambda:ResumeMicrovm` in the function's role after a deploy that removes `suspend_after`, until every session that ran such a job has ended. See [Suspending after a job](#suspending-after-a-job). 5. Change `auto_suspend_on_idle` or `idle_policy` only through a new version, as in [Deploys and versions](#deploys-and-versions). The launch records the resume mode, and a deploy changes it only for sessions that launch after it. Every job of a session that launched with suspending needs `lambda:GetMicrovm` and `lambda:ResumeMicrovm`, also after a deploy turns suspending off. So remove them from the function's role only as [Suspending an idle MicroVM](#suspending-an-idle-microvm) describes. ### Retry strategies diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py index 14301c0f3..3fb29deda 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py @@ -360,38 +360,46 @@ class MicrovmInvokeOptions(MicrovmRequestConfig): """ suspend_after: bool = False - """Whether the worker suspends the MicroVM after this job. Defaults to - ``False``. - - The worker suspends the MicroVM when the job has ended and its outcome is - reported, and no other job runs. A job that ends while another job runs - leaves the request pending. The worker then suspends the MicroVM when - the last running job ends. The next job of the session resumes the - MicroVM before it is delivered. The ``suspend_after`` job itself checks - the state too, because the record is set before the job starts. The - setting works with or without - :attr:`MicrovmSessionConfig.auto_suspend_on_idle`. + """Whether the durable function suspends the MicroVM after this job. + Defaults to ``False``. + + The session calls ``SuspendMicrovm`` for the session's MicroVM when the + job's result arrives, whether the job succeeded or failed. The call runs + inside the job's child context, before the child context returns. It is + not a durable operation of its own. The next job of the session checks + the MicroVM state before delivery, and resumes a suspended MicroVM. + + Read these rules before you set it: + + 1. The durable function suspends the MicroVM after this job's result + arrives. + 2. Do not use it on a job that runs next to other jobs of the session + (``map``, ``parallel``, promise combinators). Those jobs freeze until + the next job resumes the MicroVM, and can hit their heartbeat + timeout. + 3. The suspend runs inside the job's child context, so it follows the + child context replay rules. A result larger than the child context + checkpoint limit makes the child context run again on replay, and + each run suspends again. + 4. A retry after the Lambda invocation fails can send the suspend again. + Suspending a suspended MicroVM succeeds. + 5. The function's execution role needs ``lambda:SuspendMicrovm`` on the + MicroVM image. + 6. Adding or removing ``suspend_after`` changes what the session does on + replay. Change it only through a new function version. + + A failed ``SuspendMicrovm`` call logs a warning and does not fail the + job. The MicroVM then keeps running until the next job or the end of + the session. Set it on the job before a long wait, such as a durable wait or a - callback. Do not set it on back-to-back jobs, on ``map`` items, or on - the last job of the session. Each suspend and resume costs a snapshot - and about 1 second of latency for the next job. The session terminates - the MicroVM after its last job anyway. - - It needs more than a session without suspending: - - 1. The image enables the ``resume`` hook. Without it, the next job waits - up to 30 seconds of HTTP 503 answers. - 2. The function's role has ``lambda:GetMicrovm`` and - ``lambda:ResumeMicrovm``. - 3. The MicroVM's role has ``lambda:SuspendMicrovm`` on the image. - 4. The worker supports the setting. An older worker ignores it, and the - MicroVM keeps running. Each execution environment then logs one - warning per session. - - A suspend that the service refuses for now is not retried, unless - :attr:`MicrovmSessionConfig.auto_suspend_on_idle` is ``True``. Only - :meth:`MicrovmSession.invoke` accepts it. :func:`microvm` raises + callback. Do not set it on back-to-back jobs or on the last job of the + session. Each suspend and resume costs a snapshot and about 1 second of + latency for the next job. The session terminates the MicroVM after its + last job anyway. The image must enable the ``resume`` hook, and the + function's role needs ``lambda:GetMicrovm`` and ``lambda:ResumeMicrovm``. + + Only :meth:`MicrovmSession.invoke` accepts it. :func:`microvm` raises ``ValidationError`` for it. A value that is not a ``bool`` raises ``ValidationError`` too. """ diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/job.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/job.py index cee564937..1a9bcfaa6 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/job.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/job.py @@ -36,9 +36,8 @@ 1. The worker accepts a job request body of at most 6 MiB. 2. The body holds the input as the same JSON that :func:`dumps` writes. 3. The body also holds fixed fields: the version, the Region, the MicroVM - ID, the suspend-after flag, the callback ID, and the heartbeat timeout. - Their names and the JSON punctuation take 112 bytes. The suspend-after - flag, ``,"suspendAfter":true``, takes 20 of them. + ID, the callback ID, and the heartbeat timeout. Their names and the JSON + punctuation take 92 bytes. 4. The botocore ``lambda-microvms`` model allows a MicroVM ID of 1 to 256 characters, with no pattern. JSON writes a character in at most 6 bytes, as a ``\\u00XX`` escape. So the MicroVM ID takes at most 1,536 bytes. @@ -62,9 +61,6 @@ class RunHookPayload: A job that goes over HTTP needs a ``runHookPayload`` too, because the worker rejects a payload without ``version`` and ``region``. See :func:`run_hook_payload_without_job`. - - The payload never carries ``suspendAfter``. Only a session job can set - it, and every session job goes over HTTP. """ region: str @@ -103,7 +99,7 @@ def to_json(self) -> str: class JobRequest: """The body of the HTTP request that delivers one job to a route. - Wire names: ``version``, ``region``, ``microvmId``, ``suspendAfter``, + Wire names: ``version``, ``region``, ``microvmId``, ``callbackId``, ``heartbeatTimeoutSeconds``, and ``input``. The job fields sit at the top level, next to ``version``, ``region``, and ``microvmId``. @@ -111,11 +107,6 @@ class JobRequest: The endpoint can accept a request before Lambda has sent the ``run`` hook. The worker learns the MicroVM ID from the ``run`` hook. So the request carries the MicroVM ID too. - - ``suspendAfter`` is ``true`` only for a session job that asks the worker - to suspend the MicroVM after the job. The field is omitted otherwise. An - older worker ignores a field that it does not know. So the version stays - 1, because a new version would make every older worker fail the job. """ region: str @@ -128,8 +119,6 @@ class JobRequest: """The caller's input. It must be JSON-serializable.""" heartbeat_timeout_seconds: int | None = None """The heartbeat timeout, when the caller set one.""" - suspend_after: bool = False - """Whether the worker suspends the MicroVM after the job.""" def to_json(self) -> str: """Serialize the request as compact JSON, as :meth:`RunHookPayload.to_json` does. @@ -143,8 +132,6 @@ def to_json(self) -> str: "region": self.region, "microvmId": self.microvm_id, } - if self.suspend_after: - document["suspendAfter"] = True document.update( _job_fields( self.callback_id, self.job_input, self.heartbeat_timeout_seconds diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/lifecycle.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/lifecycle.py index 11ef931b3..eb69f8858 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/lifecycle.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/lifecycle.py @@ -161,8 +161,8 @@ def resume_if_suspended(options: EnsureRunningOptions) -> str | None: The request step of a session without the resume mode calls it after a sign that the MicroVM may be suspended: an HTTP 502 from the endpoint with ``x-aws-proxy-error: MICROVM_CONNECT_FAILED``, or no answer. A - worker can suspend its MicroVM after a ``suspend_after`` job. The - session can lose the record of that job on replay. So this check runs + session can suspend its MicroVM after a ``suspend_after`` job, and can + lose the record of that job on replay. So this check runs for every session job. 1. ``GetMicrovm`` runs once. A failure other than diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py index 39b20b02a..1fa7dc6f5 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py @@ -499,8 +499,6 @@ def deliver( retry_window_seconds: int, resume: bool | Callable[[], bool] = False, suspend_check: bool = False, - suspend_after: bool = False, - claim_suspend_after_warning: Callable[[], bool] | None = None, ) -> None: """Run the request step, ``.request``: POST the job to the MicroVM's endpoint. @@ -518,10 +516,6 @@ def deliver( sets it for every job. A failed ``GetMicrovm`` call in that check logs one warning per job, and the attempt keeps sending the job. - ``suspend_after`` adds ``"suspendAfter": true`` to the job request. - When the answer lacks the worker's acknowledgement, the attempt - logs a warning if ``claim_suspend_after_warning`` returns ``True``. - ``GetMicrovm``, ``ResumeMicrovm``, and ``CreateMicrovmAuthToken`` answer ``ResourceNotFoundException`` for a removed MicroVM. An attempt raises ``MicrovmStateError`` for that answer of any of the @@ -538,7 +532,6 @@ def deliver( callback_id=callback_id, job_input=job_input, heartbeat_timeout_seconds=heartbeat_timeout_seconds, - suspend_after=suspend_after, ) .to_json() .encode() @@ -601,17 +594,6 @@ def check_state(max_wait_seconds: float) -> str | None: check = check_state - def ignored() -> None: - if ( - claim_suspend_after_warning is not None - and claim_suspend_after_warning() - ): - logger.warning( - f"the worker in MicroVM {microvm_id} does not support " - "suspendAfter. The MicroVM keeps running after the job.", - extra={"microvmId": microvm_id}, - ) - # Only CreateMicrovmAuthToken can raise ResourceNotFoundException # here: # @@ -644,7 +626,6 @@ def ignored() -> None: unavailable_note=lambda: ( _GET_DENIED_NOTE if get_denied else None ), - on_suspend_after_ignored=ignored if suspend_after else None, ) ) except ClientError as error: @@ -767,6 +748,27 @@ def _launch_call( resume_before_delivery=resume_before_delivery, ) + def suspend(self, child: DurableContext, microvm_id: str) -> None: + """Call ``SuspendMicrovm``, and log a failure as a warning instead of + raising. + + It runs inside a job's child context, after the job's outcome is + known, and it is not a durable operation. A failed call must not + fail the job. The next job of the session checks the MicroVM state + and resumes it when needed, and suspending a suspended MicroVM + succeeds. + """ + try: + self._client().suspend_microvm(microvmIdentifier=microvm_id) + except Exception as error: # noqa: BLE001 + child.logger.warning( + f'MicroVM "{self.name}": SuspendMicrovm failed for {microvm_id}. ' + "The MicroVM keeps running, and the next job checks its state. " + "A missing permission needs lambda:SuspendMicrovm in the " + "function's execution role.", + extra={"microvmId": microvm_id, "error": str(error)}, + ) + def _terminate_call(self, microvm_id: str) -> None: try: self._client().terminate_microvm(microvmIdentifier=microvm_id) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py index b2c28fe05..4b5b8f18d 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/request.py @@ -73,13 +73,6 @@ _AUTH_TOKEN_MINUTES = 5 _PORT_HEADER = "X-aws-proxy-port" -SUSPEND_AFTER_HEADER = "X-Microvm-Suspend-After" -"""The header with which a worker acknowledges ``suspendAfter`` in its 202 -answer. Its value is :data:`SUSPEND_AFTER_ACCEPTED`. The JavaScript SDK -uses the same header.""" - -SUSPEND_AFTER_ACCEPTED = "accepted" - _PROXY_ERROR_HEADER = "x-aws-proxy-error" _CONNECT_FAILED = "MICROVM_CONNECT_FAILED" """The endpoint answers HTTP 502 with ``x-aws-proxy-error: @@ -259,9 +252,9 @@ class SendJobOptions: """Resume the MicroVM if it is suspended, and return its current endpoint. - A session job without :attr:`recheck` sets it. A worker can suspend the - MicroVM after a ``suspend_after`` job, and the session can lose the - record of that job on replay. A suspended or suspending MicroVM gets no + A session job without :attr:`recheck` sets it. A session can suspend the + MicroVM after a ``suspend_after`` job, and can lose the record of that + job on replay. A suspended or suspending MicroVM gets no answer to a request, or its endpoint answers HTTP 502 with ``x-aws-proxy-error: MICROVM_CONNECT_FAILED``. So the next attempt calls it after either sign, at most once per 2 seconds. A booting MicroVM's @@ -274,13 +267,6 @@ class SendJobOptions: unavailable_note: Callable[[], str | None] | None = None """A sentence to add to the message of :class:`MicrovmEndpointUnavailableError`, or ``None``.""" - on_suspend_after_ignored: Callable[[], None] | None = None - """Called when the answer that accepted the job lacks the header - ``X-Microvm-Suspend-After: accepted``. - - A ``suspend_after`` job sets it. A worker that supports the field sends - the header. An older worker ignores the field, and the MicroVM keeps - running after the job.""" def require_time_for_call(remaining_seconds: Callable[[], float | None]) -> None: @@ -364,11 +350,6 @@ def send_job(options: SendJobOptions) -> int: answer, at most once per 2 seconds. The function then sends the job again to the endpoint that the check reports. - A ``suspend_after`` job also reads the answer that accepted it. A worker - that supports the field answers with ``X-Microvm-Suspend-After: - accepted``. Without that header, the function calls - ``on_suspend_after_ignored``. - The same job can reach the route more than once: 1. A request reaches the route, and its answer is lost. @@ -469,11 +450,6 @@ def new_token() -> Mapping[str, str]: options.logger.info( "MicroVM job accepted", extra={"attempt": attempt, "status": status} ) - if ( - options.on_suspend_after_ignored is not None - and answer.suspend_after != SUSPEND_AFTER_ACCEPTED - ): - options.on_suspend_after_ignored() return status if status is None: last_failure = answer.failure @@ -632,8 +608,6 @@ class _Answer: """Whether the endpoint answered 502 with ``x-aws-proxy-error: MICROVM_CONNECT_FAILED``. It answers so for a suspending or suspended MicroVM.""" - suspend_after: str | None = None - """The value of the ``X-Microvm-Suspend-After`` header of a 2xx answer.""" def _post_within( @@ -684,7 +658,6 @@ def _post( return _Answer(response.status, followed_redirect=True) return _Answer( response.status, - suspend_after=response.headers.get(SUSPEND_AFTER_HEADER), ) except urllib.error.HTTPError as error: has_location = bool(error.headers and error.headers.get("Location")) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py index 5a3f6a376..9e54299b2 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py @@ -6,9 +6,7 @@ from __future__ import annotations -import threading import uuid -from collections import OrderedDict from collections.abc import Callable, Mapping from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, TypeVar @@ -62,16 +60,6 @@ ``timeout`` is under 10 seconds does not suspend by default. """ -_WARNED_SESSIONS: OrderedDict[str, None] = OrderedDict() -"""The IDs of the sessions that warned about a worker that ignores -``suspend_after``, oldest first.""" - -_MAX_WARNED_SESSIONS = 1_000 -"""How many session IDs :data:`_WARNED_SESSIONS` keeps. A session ID is a -short string, so 1,000 of them use well under 1 MB.""" - -_WARNED_LOCK = threading.Lock() - _MIN_AUTO_SUSPEND_IDLE_SECONDS = 10 """The shortest idle time a session accepts. @@ -121,9 +109,9 @@ def microvm_session( records whether the MicroVM can be suspended. Replay follows that record, not the current config. - A job with ``suspend_after`` asks the worker to suspend the MicroVM after - the job. Each later job of the handle then checks the state before - delivery, as with ``auto_suspend_on_idle``. That record is kept in + A job with ``suspend_after`` makes the session call ``SuspendMicrovm`` + after the job's result arrives. Each later job of the handle then checks + the state before delivery, as with ``auto_suspend_on_idle``. That record is kept in memory only. So every job of every session also checks the state after a sign of a suspended MicroVM: no answer, or an HTTP 502 with ``x-aws-proxy-error: MICROVM_CONNECT_FAILED``. A boot gives no such @@ -226,7 +214,6 @@ def __init__( launched: LaunchResult, *, resume_before_delivery: bool, - session_id: str, ) -> None: self._scope = scope self._launched = launched @@ -237,7 +224,6 @@ def __init__( # its invoke() again, so it does not set the flag. The check after a # sign of a suspended MicroVM covers that case. self._may_be_suspended = False - self._session_id = session_id @property def microvm_id(self) -> str: @@ -254,22 +240,6 @@ def _resume_mode(self) -> bool: """ return self._resume_before_delivery or self._may_be_suspended - def _claim_suspend_after_warning(self) -> bool: - """Return ``True`` the first time, so the session warns once about a - worker that does not support ``suspend_after``. - - Each invocation creates a new handle. So the record is kept per - process, by session ID. A later invocation in the same execution - environment then logs no second warning. - """ - with _WARNED_LOCK: - if self._session_id in _WARNED_SESSIONS: - return False - _WARNED_SESSIONS[self._session_id] = None - if len(_WARNED_SESSIONS) > _MAX_WARNED_SESSIONS: - _WARNED_SESSIONS.popitem(last=False) - return True - def invoke( self, context: DurableContext, @@ -371,21 +341,22 @@ def run(job_context: DurableContext) -> Any: retry_window_seconds=retry_window_seconds, resume=self._resume_mode, suspend_check=True, - suspend_after=suspend_after, - claim_suspend_after_warning=( - self._claim_suspend_after_warning if suspend_after else None - ), ) try: - return callback.result() + result = callback.result() except (ExecutionError, InvocationError): # An SDK execution or invocation error is not a job outcome. # The core SDK records neither as the child context's failure. raise except Exception as error: # SuspendExecution is a BaseException, so a wait for the - # callback passes through here. + # callback passes through here, and no suspend runs. + if suspend_after: + scope.suspend(job_context, launched.microvm_id) raise job_error(name, callback, error) from error + if suspend_after: + scope.suspend(job_context, launched.microvm_id) + return result try: return context.run_in_child_context(run, name=name) @@ -485,7 +456,6 @@ def run_session( self, launched, resume_before_delivery=launched.resume_before_delivery is True, - session_id=session_id, ), child, ) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/conftest.py b/packages/aws-durable-execution-sdk-python-extras/tests/conftest.py index 5d9ec3699..06af7fce0 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/conftest.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/conftest.py @@ -31,9 +31,10 @@ def client_error(code: str, status: int = 400) -> ClientError: class FakeMicrovmsClient: """Records Lambda MicroVMs API calls, and answers from scripted lists. - ``launches``, ``tokens``, ``terminates``, and ``resumes`` hold the - answers of ``run_microvm``, ``create_microvm_auth_token``, - ``terminate_microvm``, and ``resume_microvm``, in order. Each answer is an + ``launches``, ``tokens``, ``terminates``, ``resumes``, and ``suspends`` + hold the answers of ``run_microvm``, ``create_microvm_auth_token``, + ``terminate_microvm``, ``resume_microvm``, and ``suspend_microvm``, in + order. Each answer is an exception to raise, a function that receives the request, or ``None`` for success. After a list ends, every call succeeds. @@ -54,6 +55,7 @@ def __init__( tokens: list[Answer] | None = None, terminates: list[Answer] | None = None, resumes: list[Answer] | None = None, + suspends: list[Answer] | None = None, states: list[str | BaseException] | None = None, endpoint: str | None = "mvm-1.microvm.example", suspended_until_resumed: bool = False, @@ -66,6 +68,7 @@ def __init__( "create_microvm_auth_token": list(tokens or []), "terminate_microvm": list(terminates or []), "resume_microvm": list(resumes or []), + "suspend_microvm": list(suspends or []), } self._states = list(states or []) self._lock = threading.Lock() @@ -101,6 +104,10 @@ def terminate_microvm(self, **request: Any) -> dict[str, Any]: self._answer("terminate_microvm", request) return {} + def suspend_microvm(self, **request: Any) -> dict[str, Any]: + self._answer("suspend_microvm", request) + return {} + def resume_microvm(self, **request: Any) -> dict[str, Any]: self._answer("resume_microvm", request) with self._lock: @@ -150,20 +157,11 @@ def connect_failed() -> Status: return CONNECT_FAILED -@pytest.fixture -def suspend_after_ack() -> dict[str, str]: - """The header of a worker that supports ``suspendAfter``.""" - return dict(SUSPEND_AFTER_ACK) - - Status = int | tuple[int, dict[str, str]] | None CONNECT_FAILED: Status = (502, {"x-aws-proxy-error": "MICROVM_CONNECT_FAILED"}) """The answer of the endpoint of a suspending or suspended MicroVM.""" -SUSPEND_AFTER_ACK = {"X-Microvm-Suspend-After": "accepted"} -"""The header with which a worker acknowledges ``suspendAfter``.""" - class FakeEndpoint: """A local HTTP server that stands in for a MicroVM endpoint. diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/job_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/job_test.py index f9198ac94..3b164b72d 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/job_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/job_test.py @@ -7,7 +7,6 @@ import json import math -from typing import Any import pytest from aws_durable_execution_sdk_python_microvm_worker import ( @@ -154,37 +153,24 @@ def test_the_worker_reads_the_idle_time_of_a_session() -> None: assert request.payload.auto_suspend_idle_seconds == 60 -def test_writes_suspend_after_only_when_it_is_set() -> None: - fields: dict[str, Any] = { - "region": "us-east-1", - "microvm_id": "mvm-1", - "callback_id": "cb-1", - "job_input": 1, - } +def test_the_job_request_has_no_suspend_after_field() -> None: + # The durable function suspends the MicroVM itself after the job, so the + # request carries no field that asks the worker to do it. + request = JobRequest( + region="us-east-1", microvm_id="mvm-1", callback_id="cb-1", job_input=1 + ) - assert JobRequest(**fields, suspend_after=True).to_json() == ( + assert request.to_json() == ( '{"version":1,"region":"us-east-1","microvmId":"mvm-1",' - '"suspendAfter":true,"callbackId":"cb-1","input":1}' + '"callbackId":"cb-1","input":1}' ) - assert "suspendAfter" not in JobRequest(**fields).to_json() - - -def test_the_worker_reads_suspend_after() -> None: - body = JobRequest( - region="us-east-1", - microvm_id="mvm-1", - callback_id="cb-1", - job_input=None, - suspend_after=True, - ).to_json() - - assert MicrovmJobRequest.from_dict(json.loads(body)).suspend_after is True + assert not hasattr(request, "suspend_after") def test_the_job_fields_fit_in_the_margin_of_the_input_limit() -> None: # 1. The longest MicroVM ID is 256 characters, each escaped in 6 bytes. # 2. The longest callback ID is 1,024 characters of 1 byte. - # 3. The heartbeat timeout is at most 8 hours, and suspendAfter is set. + # 3. The heartbeat timeout is at most 8 hours. # So the body of the largest input is at most the 6 MiB that the worker # reads. body = JobRequest( @@ -193,10 +179,9 @@ def test_the_job_fields_fit_in_the_margin_of_the_input_limit() -> None: callback_id="A" * 1024, job_input="x" * (MAX_JOB_INPUT_BYTES - 2), heartbeat_timeout_seconds=28_800, - suspend_after=True, ).to_json() fixed = len(body.encode()) - MAX_JOB_INPUT_BYTES - assert fixed == 112 + len("ap-southeast-5") + 256 * 6 + 1024 + len("28800") + assert fixed == 92 + len("ap-southeast-5") + 256 * 6 + 1024 + len("28800") assert fixed < 3 * 1024 assert len(body.encode()) <= 6 * 1024 * 1024 diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/request_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/request_test.py index fac5b92df..35a6da96a 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/request_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/request_test.py @@ -15,7 +15,6 @@ import pytest from aws_durable_execution_sdk_python_microvm_worker import MICROVM_JOB_PATH -from aws_durable_execution_sdk_python_microvm_worker import worker from aws_durable_execution_sdk_python_extras.microvm import ( DEFAULT_MICROVM_JOB_PATH, @@ -58,11 +57,6 @@ def test_matches_the_job_path_of_the_worker() -> None: assert DEFAULT_MICROVM_JOB_PATH == MICROVM_JOB_PATH -def test_matches_the_suspend_after_header_of_the_worker() -> None: - assert request.SUSPEND_AFTER_HEADER == worker.SUSPEND_AFTER_HEADER - assert request.SUSPEND_AFTER_ACCEPTED == "accepted" - - def test_posts_the_job_with_an_auth_token(fake_client: Any, fake_endpoint: Any) -> None: client = fake_client() endpoint = fake_endpoint() @@ -646,7 +640,7 @@ def test_sends_no_auth_token_on_a_redirect_that_the_callers_opener_follows( # --------------------------------------------------------------------------- -# The check for a suspended MicroVM, and the suspendAfter acknowledgement +# The check for a suspended MicroVM # --------------------------------------------------------------------------- @@ -814,50 +808,3 @@ def test_adds_no_note_when_the_note_is_none( ) assert str(raised.value).endswith("Last failure: HTTP 503.") - - -@pytest.mark.parametrize( - ("headers", "ignored"), - [ - ({"X-Microvm-Suspend-After": "accepted"}, 0), - ({}, 1), - ({"X-Microvm-Suspend-After": "yes"}, 1), - ], - ids=["acknowledged", "no-header", "other-value"], -) -def test_reports_a_suspend_after_job_that_the_worker_did_not_acknowledge( - fake_client: Any, fake_endpoint: Any, headers: dict[str, str], ignored: int -) -> None: - endpoint = fake_endpoint(accept_headers=headers) - calls: list[None] = [] - - status = send_job( - options( - fake_client(), - endpoint.url, - [], - on_suspend_after_ignored=lambda: calls.append(None), - ) - ) - - assert status == 202 - assert len(calls) == ignored - - -def test_reads_the_acknowledgement_of_the_answer_that_accepted_the_job( - fake_client: Any, fake_endpoint: Any, suspend_after_ack: dict[str, str] -) -> None: - # A refused answer that carries the header does not count. - endpoint = fake_endpoint([(503, suspend_after_ack)]) - calls: list[None] = [] - - send_job( - options( - fake_client(), - endpoint.url, - [], - on_suspend_after_ignored=lambda: calls.append(None), - ) - ) - - assert calls == [None] diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index fa5500eb2..d92e150a7 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -9,7 +9,6 @@ import hashlib import json import logging -from collections import OrderedDict from collections.abc import Callable, Iterator from types import SimpleNamespace from typing import Any @@ -50,7 +49,6 @@ microvm_session, operation, ) -from aws_durable_execution_sdk_python_extras.microvm import session as session_module WAIT = 30 @@ -1531,9 +1529,10 @@ def test_makes_no_state_check_for_a_launch_checkpoint_without_a_bool_resume_mode # suspend_after, and the check after a sign of a suspended MicroVM # --------------------------------------------------------------------------- -IGNORED_WARNING = ( - "the worker in MicroVM mvm-1 does not support suspendAfter. The MicroVM " - "keeps running after the job." +SUSPEND_WARNING = ( + 'MicroVM "pipeline": SuspendMicrovm failed for mvm-1. The MicroVM keeps ' + "running, and the next job checks its state. A missing permission needs " + "lambda:SuspendMicrovm in the function's execution role." ) STATE_CHECK_WARNING = ( "GetMicrovm failed, so the operation could not check whether the MicroVM " @@ -1561,10 +1560,10 @@ def suspend_then_wait_then_job( return [build, vm.invoke(ctx, "test", job_input=2, options=job())] -def test_asks_the_worker_to_suspend_after_the_job_only_when_it_is_set( - run: Any, fake_client: Any, fake_endpoint: Any, suspend_after_ack: Any +def test_suspends_the_microvm_after_the_job_only_when_it_is_set( + run: Any, fake_client: Any, fake_endpoint: Any ) -> None: - endpoint = fake_endpoint(accept_headers=suspend_after_ack) + endpoint = fake_endpoint() client = fake_client(endpoint=endpoint.url) result = run( @@ -1574,10 +1573,11 @@ def test_asks_the_worker_to_suspend_after_the_job_only_when_it_is_set( ) assert outcome(result) == {"result": [1, 2]} - first, second = (request["body"] for request in endpoint.requests) - assert first["suspendAfter"] is True - assert "suspendAfter" not in second - # The run hook payload never carries it. + # The session calls SuspendMicrovm once, for the first job only, and not + # in the invocation in which the job still waits for its callback. + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + # The job request does not ask the worker to suspend. + assert all("suspendAfter" not in request["body"] for request in endpoint.requests) [launch] = client.requests("run_microvm") assert json.loads(launch["runHookPayload"]) == {"version": 1, "region": "us-west-2"} # suspend_after is not a session setting. So the launch records no @@ -1585,10 +1585,101 @@ def test_asks_the_worker_to_suspend_after_the_job_only_when_it_is_set( assert launch_checkpoint(result)["resumeBeforeDelivery"] is False +def test_suspends_the_microvm_after_the_job_inside_its_child_context( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + # No step and no checkpoint of its own: the job's operations are the + # child context, the callback, and the request step. + client = fake_client(endpoint=fake_endpoint().url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + return vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + + result = run( + session_handler(session_config(client), body), + None, + [("build", succeed_with(1))], + ) + + assert outcome(result) == {"result": 1} + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + names = sorted(operation.name for operation in result.operations if operation.name) + assert not any("suspend" in name for name in names) + + +def test_suspends_the_microvm_after_a_failed_job_too( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + client = fake_client(endpoint=fake_endpoint().url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + try: + vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + except MicrovmJobFailedError as error: + return str(error) + return None + + result = run( + session_handler(session_config(client), body), + None, + [("build", fail_with("BuildError", "no compiler"))], + ) + + assert "no compiler" in outcome(result)["result"] + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + + +def test_logs_a_warning_and_returns_the_result_when_the_suspend_fails( + run: Any, + fake_client: Any, + fake_endpoint: Any, + make_error: Any, + caplog: pytest.LogCaptureFixture, +) -> None: + endpoint = fake_endpoint() + client = fake_client( + endpoint=endpoint.url, + suspends=[make_error("AccessDeniedException", 403)], + states=["RUNNING", "RUNNING"], + ) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + build = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + return [build, vm.invoke(ctx, "test", job_input=2, options=job())] + + with caplog.at_level(logging.WARNING): + result = run( + session_handler(session_config(client), body), + None, + [("build", succeed_with(1)), ("test", succeed_with(2))], + ) + + assert outcome(result) == {"result": [1, 2]} + assert warnings(caplog, SUSPEND_WARNING) == 1 + # The next job checks the state, as after a suspend that worked. + assert len(client.requests("get_microvm")) == 2 + assert len(endpoint.requests) == 2 + + +def test_does_not_suspend_a_job_that_did_not_ask_for_it( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + client = fake_client(endpoint=fake_endpoint().url) + + result = run( + session_handler(session_config(client), two_jobs), + None, + [("build", succeed_with(1)), ("test", succeed_with(2))], + ) + + assert outcome(result)["result"]["test"] == 2 + assert client.requests("suspend_microvm") == [] + + def test_checks_the_state_before_the_jobs_after_a_suspend_after_job( - run: Any, fake_client: Any, fake_endpoint: Any, suspend_after_ack: Any + run: Any, fake_client: Any, fake_endpoint: Any ) -> None: - endpoint = fake_endpoint(accept_headers=suspend_after_ack) + endpoint = fake_endpoint() client = fake_client( endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] ) @@ -1613,13 +1704,13 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: def test_sets_the_flag_again_when_a_completed_suspend_after_job_replays( - run: Any, fake_client: Any, fake_endpoint: Any, suspend_after_ack: Any + run: Any, fake_client: Any, fake_endpoint: Any ) -> None: # The second job runs in a later invocation. The first job's child # context is complete then, so its function does not run again. invoke() # sets the flag before the child context, so the second job still # checks the state before delivery. - endpoint = fake_endpoint(accept_headers=suspend_after_ack) + endpoint = fake_endpoint() client = fake_client( endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] ) @@ -1635,13 +1726,15 @@ def test_sets_the_flag_again_when_a_completed_suspend_after_job_replays( assert len(client.requests("get_microvm")) == 3 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert len(endpoint.requests) == 2 + # The completed job does not run again on replay. So the session does + # not send a second SuspendMicrovm. + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] def test_resumes_through_the_fallback_after_a_map_item_lost_the_flag( run: Any, fake_client: Any, fake_endpoint: Any, - suspend_after_ack: Any, connect_failed: Any, ) -> None: # 1. Item a asks for a suspend after its job. @@ -1651,7 +1744,7 @@ def test_resumes_through_the_fallback_after_a_map_item_lost_the_flag( # 4. So the job goes to the endpoint of a suspended MicroVM, which # answers 502 MICROVM_CONNECT_FAILED. The step checks the state, # resumes the MicroVM, and sends the job again. - endpoint = fake_endpoint([(202, suspend_after_ack), 202, connect_failed]) + endpoint = fake_endpoint([202, 202, connect_failed]) client = fake_client( endpoint=endpoint.url, states=["RUNNING", "RUNNING", "SUSPENDED", "RUNNING"], @@ -1889,79 +1982,3 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: assert error["error"] == "MicrovmDeliveryError" assert "AccessDeniedException message" in error["message"] assert len(endpoint.requests) == 1 - - -@pytest.mark.parametrize( - ("acknowledged", "expected"), [(False, 1), (True, 0)], ids=["old", "new"] -) -def test_warns_once_per_session_when_the_worker_ignores_suspend_after( - run: Any, - fake_client: Any, - fake_endpoint: Any, - suspend_after_ack: Any, - caplog: pytest.LogCaptureFixture, - acknowledged: bool, # noqa: FBT001 - a pytest parameter - expected: int, -) -> None: - endpoint = fake_endpoint(accept_headers=suspend_after_ack if acknowledged else {}) - client = fake_client(endpoint=endpoint.url) - - def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: - options = job(suspend_after=True) - return [ - vm.invoke(ctx, "build", job_input=1, options=options), - vm.invoke(ctx, "test", job_input=2, options=options), - ] - - with caplog.at_level(logging.WARNING): - result = run( - session_handler(session_config(client), body), - None, - [("build", succeed_with(1)), ("test", succeed_with(2))], - ) - - assert outcome(result) == {"result": [1, 2]} - assert len(endpoint.requests) == 2 - assert warnings(caplog, IGNORED_WARNING) == expected - - -def test_reads_no_acknowledgement_for_a_job_without_suspend_after( - run: Any, fake_client: Any, fake_endpoint: Any, caplog: pytest.LogCaptureFixture -) -> None: - client = fake_client(endpoint=fake_endpoint().url) - - with caplog.at_level(logging.WARNING): - result = run( - session_handler(session_config(client), two_jobs), - None, - [("build", succeed_with(1)), ("test", succeed_with(2))], - ) - - assert outcome(result)["result"]["test"] == 2 - assert warnings(caplog, IGNORED_WARNING) == 0 - - -def test_remembers_the_warning_of_the_latest_sessions_only( - monkeypatch: pytest.MonkeyPatch, -) -> None: - monkeypatch.setattr(session_module, "_WARNED_SESSIONS", OrderedDict()) - monkeypatch.setattr(session_module, "_MAX_WARNED_SESSIONS", 2) - a, b, c = ( - MicrovmSession( - None, # type: ignore[arg-type] - None, # type: ignore[arg-type] - resume_before_delivery=False, - session_id=session_id, - ) - for session_id in ["a", "b", "c"] - ) - - assert [handle._claim_suspend_after_warning() for handle in (a, b, c)] == [ # noqa: SLF001 - True, - True, - True, - ] - # Each session warns once. - assert c._claim_suspend_after_warning() is False # noqa: SLF001 - # The oldest session was forgotten. - assert a._claim_suspend_after_warning() is True # noqa: SLF001 diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/worker_composed_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/worker_composed_test.py index 3fa7a0896..ea83074f3 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/worker_composed_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/worker_composed_test.py @@ -29,7 +29,7 @@ from typing import Any import pytest -from aws_durable_execution_sdk_python.config import Duration, MapConfig +from aws_durable_execution_sdk_python.config import Duration from aws_durable_execution_sdk_python.context import DurableContext from aws_durable_execution_sdk_python.execution import ( InvocationStatus, @@ -572,16 +572,11 @@ def handler(event: Any, context: DurableContext) -> Any: # suspend_after with the real worker # --------------------------------------------------------------------------- -IGNORED_WARNING = ( - "the worker in MicroVM mvm-1 does not support suspendAfter. The MicroVM " - "keeps running after the job." -) - class SimulatedMicrovm: """A stand-in for Lambda that keeps the state of one MicroVM. - 1. The worker's SuspendMicrovm sends the ``suspend`` hook, and then sets + 1. The session's SuspendMicrovm sends the ``suspend`` hook, and then sets the state to ``SUSPENDED``. Lambda calls the hook before the freeze. 2. The session's ResumeMicrovm sends the ``resume`` hook, and then sets the state to ``RUNNING``. A probe saw the hook before ``RUNNING``. @@ -589,18 +584,14 @@ class SimulatedMicrovm: job to the worker while the MicroVM runs. While it is suspended, it answers HTTP 502 with ``x-aws-proxy-error: MICROVM_CONNECT_FAILED``, as the service does. - - ``old_worker`` removes ``suspendAfter`` from each forwarded job. The - worker then behaves as a release that does not know the field. """ - def __init__(self, *, old_worker: bool = False) -> None: + def __init__(self) -> None: self._lock = threading.Lock() self._state = "RUNNING" self.worker: MicrovmWorker | None = None self.suspended: list[str] = [] self.answers: list[int] = [] - self._old_worker = old_worker simulated = self class Endpoint(BaseHTTPRequestHandler): @@ -613,11 +604,9 @@ def do_POST(self) -> None: # noqa: N802 - the http.server name self.send_header("content-length", "0") self.end_headers() return - status, accepted, data = simulated.forward(self.path, body) + status, data = simulated.forward(self.path, body) simulated.record(status) self.send_response(status) - if accepted: - self.send_header("X-Microvm-Suspend-After", "accepted") self.send_header("content-length", str(len(data))) self.end_headers() self.wfile.write(data) @@ -644,15 +633,11 @@ def set_state(self, state: str) -> None: with self._lock: self._state = state - def forward(self, path: str, body: bytes) -> tuple[int, bool, bytes]: - """Send the job to the worker. Return the status, whether the worker - accepted ``suspendAfter``, and the body of the answer.""" + def forward(self, path: str, body: bytes) -> tuple[int, bytes]: + """Send the job to the worker. Return the status and the body of the + answer.""" worker = self.worker assert worker is not None - if self._old_worker: - document = json.loads(body) - document.pop("suspendAfter", None) - body = json.dumps(document).encode() connection = http.client.HTTPConnection( host_of(worker), worker.port, timeout=WAIT ) @@ -662,13 +647,7 @@ def forward(self, path: str, body: bytes) -> tuple[int, bool, bytes]: ) response = connection.getresponse() data = response.read() - # The endpoint sends one header with a fixed name and a fixed - # value. A value from the worker's answer never reaches - # send_header. The session accepts only this value. So an answer - # with another value reaches the session as an answer without - # the header, which the session treats the same way. - accepted = response.getheader("X-Microvm-Suspend-After") == "accepted" - return response.status, accepted, data + return response.status, data finally: connection.close() @@ -692,8 +671,6 @@ def suspend_after_session( fake_client: Any, make_pipeline: Callable[[SimulatedMicrovm], Pipeline], job: Callable[[Any, MicrovmJobContext], Any], - *, - old_worker: bool = False, ) -> tuple[Any, Any, SimulatedMicrovm, RecordingLogger]: """Run a pipeline in a session without auto-suspend, against a real worker behind a :class:`SimulatedMicrovm`. @@ -701,7 +678,7 @@ def suspend_after_session( ``make_pipeline`` receives the simulated MicroVM, so the pipeline can wait for its state. """ - microvm = SimulatedMicrovm(old_worker=old_worker) + microvm = SimulatedMicrovm() pipeline = make_pipeline(microvm) holder: dict[str, Any] = {} @@ -716,11 +693,9 @@ def resume_microvm(self, **request: Any) -> dict[str, Any]: microvm.set_state("RUNNING") return {} - class WorkerMicrovms: - """The worker's lambda-microvms client.""" - - def suspend_microvm(self, *, microvmIdentifier: str) -> dict[str, Any]: # noqa: N803 - microvm.suspended.append(microvmIdentifier) + def suspend_microvm(self, **request: Any) -> dict[str, Any]: + super().suspend_microvm(**request) + microvm.suspended.append(request["microvmIdentifier"]) post_hook(holder["worker"], "suspend") microvm.set_state("SUSPENDED") return {} @@ -738,7 +713,6 @@ def handler(event: Any, context: DurableContext) -> Any: lambda: start_microvm_worker( handler=job, create_client=lambda region: callbacks, # type: ignore[arg-type, return-value] - create_microvms_client=lambda region: WorkerMicrovms(), # type: ignore[arg-type, return-value] logger=logger, port=0, ) @@ -796,12 +770,10 @@ def pipeline(vm: MicrovmSession, ctx: DurableContext) -> Any: ) assert outcome == ["build done", "SUSPENDED", "test done"] - # The worker acknowledged suspendAfter. So the session logs no warning. - assert IGNORED_WARNING not in [record.getMessage() for record in caplog.records] + # The session suspended the MicroVM itself, and the worker did not. + assert not [record for record in caplog.records if record.levelname == "WARNING"] assert microvm.suspended == ["mvm-1"] - [line] = logger.extras("suspending the MicroVM after a job") - assert list(line) == ["microvmId", "callbackId"] - assert line["microvmId"] == "mvm-1" + assert logger.extras("suspending the MicroVM after a job") == [] # The flag makes the second job check the state before delivery. So it # resumes the MicroVM before it sends the job, and gets no 502. assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] @@ -810,47 +782,6 @@ def pipeline(vm: MicrovmSession, ctx: DurableContext) -> Any: assert client.requests("terminate_microvm") == [{"microvmIdentifier": "mvm-1"}] -def test_suspends_after_a_map_with_one_suspend_after_item(fake_client: Any) -> None: - def slow_echo(job_input: Any, context: MicrovmJobContext) -> Any: - # The items run at once. So the worker suspends only when the last - # of them ends. The sleep keeps each job running until every item - # has delivered its job. - time.sleep(1) - return f"{job_input} done" - - def make_pipeline(microvm: SimulatedMicrovm) -> Pipeline: - def pipeline(vm: MicrovmSession, ctx: DurableContext) -> Any: - shards: list[Any] = ctx.map( - ["a", "b", "c"], - lambda item_context, item, index, items: vm.invoke( - item_context, - f"shard-{item}", - job_input=item, - options=options(suspend_after=item == "a"), - ), - name="shards", - config=MapConfig(max_concurrency=3), - ).get_results() - between = ctx.step(lambda _: microvm.wait_for("SUSPENDED"), name="pause") - after = vm.invoke(ctx, "after", job_input="after", options=options()) - return [shards, between, after] - - return pipeline - - outcome, client, microvm, logger = suspend_after_session( - fake_client, make_pipeline, slow_echo - ) - - assert outcome == [["a done", "b done", "c done"], "SUSPENDED", "after done"] - # One request from item a, and one suspend. - assert microvm.suspended == ["mvm-1"] - assert len(logger.extras("suspending the MicroVM after a job")) == 1 - # The job after the map resumed the MicroVM, with the flag or after a - # 502, depending on which invocation ran item a. - assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] - assert microvm.state() == "RUNNING" - - def test_a_replay_that_lost_the_flag_resumes_through_the_check_after_a_502( fake_client: Any, ) -> None: @@ -894,38 +825,6 @@ def pipeline(vm: MicrovmSession, ctx: DurableContext) -> Any: assert microvm.state() == "RUNNING" -def test_an_old_worker_ignores_suspend_after_and_the_session_warns_once( - fake_client: Any, caplog: pytest.LogCaptureFixture -) -> None: - def make_pipeline(microvm: SimulatedMicrovm) -> Pipeline: - def pipeline(vm: MicrovmSession, ctx: DurableContext) -> Any: - return [ - vm.invoke( - ctx, name, job_input=name, options=options(suspend_after=True) - ) - for name in ["build", "test"] - ] - - return pipeline - - with caplog.at_level(logging.WARNING): - outcome, client, microvm, logger = suspend_after_session( - fake_client, make_pipeline, echo, old_worker=True - ) - - assert outcome == ["build done", "test done"] - # The worker sent no acknowledgement and never suspended the MicroVM. - assert microvm.suspended == [] - assert logger.extras("suspending the MicroVM after a job") == [] - warnings = [ - record - for record in caplog.records - if record.levelname == "WARNING" and record.getMessage() == IGNORED_WARNING - ] - assert len(warnings) == 1 - assert client.requests("resume_microvm") == [] - - class LosingProxy: """A stand-in for the MicroVM endpoint in front of a real worker. From 600dea0ae14778abc496aaccf9abe4c06c43015e Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Tue, 3 Nov 2026 21:57:56 +0000 Subject: [PATCH 2/9] refactor(worker): remove suspendAfter handling The durable function now suspends the MicroVM itself, so the worker no longer reads suspendAfter from a job request. Removed: the pending-suspend logic, the X-Microvm-Suspend-After 202 header, and the job trigger of the suspend-call warning. Kept: idle auto-suspend, the suspend, resume and terminate hooks. An older durable function can still send the field. The worker ignores it. --- .../README.md | 29 +- .../payload.py | 9 - .../worker.py | 285 +------- .../tests/lifecycle_hooks_test.py | 676 +----------------- .../tests/payload_test.py | 22 +- 5 files changed, 65 insertions(+), 956 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/README.md b/packages/aws-durable-execution-sdk-python-microvm-worker/README.md index dd15db8e0..cfa5bd992 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/README.md +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/README.md @@ -65,7 +65,7 @@ The image needs these hook settings in `CreateMicrovmImage`: ``` - The `run` hook delivers the job, or tells the worker that jobs arrive over HTTP. So it must be enabled. -- The `resume` hook tells a session's worker that its MicroVM runs again after a suspend. It is optional, and recommended for sessions. A session that uses `suspendAfter` needs it. See [Suspending when idle](#suspending-when-idle) and [Suspending after a job](#suspending-after-a-job). +- The `resume` hook tells a session's worker that its MicroVM runs again after a suspend. It is optional, and recommended for sessions that suspend. A session that suspends after a job, or when idle, needs it. See [Suspending when idle](#suspending-when-idle) and [Suspending after a job](#suspending-after-a-job). - The `suspend` hook tells the worker that Lambda is about to freeze the MicroVM. The worker then refuses new jobs until the `resume` hook. It is optional, and recommended for sessions. - The `terminate` hook tells the worker that Lambda is about to end the MicroVM. The worker then fails the callback of each running job, so the durable function fails the job at once instead of at its heartbeat timeout or its timeout. It is optional, and recommended. Give it at least 10 seconds, because the worker reports before it answers, for at most 5 seconds. A hook timeout left unset ended the hook after about 2 seconds in testing. - The service requires the `ready` hook whenever a lifecycle hook is enabled. The worker answers it with HTTP 200. @@ -83,7 +83,7 @@ On `POST /aws/lambda-microvms/runtime/v1/run` the worker: The worker acts only on the first valid `run` hook. The MicroVM endpoint forwards a request to a hook path from any caller that holds an auth token for the MicroVM. So a later `run` hook gets HTTP 200, and is ignored. -On `POST ` the worker validates the job request, answers HTTP 202 at once, and runs the route's handler. A job request with `"suspendAfter": true` gets the header `X-Microvm-Suspend-After: accepted` in its 202 answer. See [Suspending after a job](#suspending-after-a-job). `POST /durable-execution/v1/job` (`MICROVM_JOB_PATH`) runs `handler`. That path is reserved, so a route cannot use it. A job request can arrive before the `run` hook. Until the `run` hook arrives, the worker takes the MicroVM ID for `context.microvm_id` from the first job request that has a `microvmId`. The `run` hook's ID then replaces it for later jobs. Without either, `context.microvm_id` is `"unknown"`. +On `POST ` the worker validates the job request, answers HTTP 202 at once, and runs the route's handler. `POST /durable-execution/v1/job` (`MICROVM_JOB_PATH`) runs `handler`. That path is reserved, so a route cannot use it. A job request can arrive before the `run` hook. Until the `run` hook arrives, the worker takes the MicroVM ID for `context.microvm_id` from the first job request that has a `microvmId`. The `run` hook's ID then replaces it for later jobs. Without either, `context.microvm_id` is `"unknown"`. Each job runs in its own thread. For every job, the worker: @@ -116,9 +116,9 @@ A session's run hook payload can set `autoSuspendIdleSeconds`. The session sets 6. A call that the service rejected on its first attempt, such as `AccessDeniedException` for a missing permission, is logged as a warning, and the MicroVM keeps running. Missing credentials count as a rejection. A later call in the same MicroVM most likely finds no credentials either. The worker accepts jobs again at once and does not retry the call. The next job that ends starts the idle time again. The exception is a `resume` hook that arrived during the call. The idle time then starts when the call ends. So the worker retries the call one idle time after the call ends, if no job runs and no `suspend` hook refusal is in effect when the call ends. A second exception is a `suspend` hook that arrived during the call and whose refusal is still in effect when the call ends. The MicroVM is then suspending anyway. So the rejection does not end the hook's refusal. That refusal ends on the `resume` hook, or 30 seconds after the last `suspend` hook. The idle time then starts again. 7. A call that the service refused for now on its first attempt is logged as a warning too, and the worker accepts jobs again at once. This covers expired credentials and a signature error, with or without a clock offset, whatever the HTTP status. The service answers these before it acts. botocore retries none of these errors, so they reach the worker on the first attempt. The idle time then starts again, so the worker tries the call again. That idle time lasts at least 30 seconds, also when `autoSuspendIdleSeconds` is shorter. The same error can repeat on every call, for example with a wrong secret key or an expired session token from environment variables. So while no job ends and no lifecycle hook arrives, the worker makes at most one call per 30 seconds, and logs one warning per call. A job that ends after the refusal, or a `resume` hook after it, starts the normal idle time instead. A `resume` hook or a job that ends during the call does not shorten the idle time after the refusal, because the refused outcome arrives after it. A `suspend` hook whose refusal is still in effect when the call ends is the exception, as in item 6. Its refusal stays, and the idle time starts when it ends. The default client makes one attempt. So a throttle from it counts as refused for now. A client from `create_microvms_client` that retries a throttle turns it into an unknown outcome, as in item 8. The JavaScript worker classifies these errors the same way. 8. A call whose outcome is unknown is handled like a successful call, because the service may have suspended the MicroVM anyway. This covers a timeout, a conflict, a server error, and a dropped connection. An error after botocore retried an earlier attempt also counts as unknown, because the earlier attempt may have suspended the MicroVM. Only a client from `create_microvms_client` that retries can raise such an error. A call that the `before-send` handler stopped after a `resume` hook counts as unknown too. The handler can stop the first attempt, and then nothing was sent. An earlier attempt, if there was one, may have suspended the MicroVM. That `resume` hook has already ended the refusal. So the worker accepts jobs at once, and starts no new 30-second limit for that call. The refusal continues until the `resume` hook or the 30-second limit. The idle time then starts again, so the worker tries the call again. A `resume` hook during the call, or the end of a `suspend` hook's 30 seconds during the call, has already ended the refusal. The worker then accepts jobs at once, and tries the call again one idle time after the call and every job end. -9. The worker logs one warning for each failed call. The message names the outcome. The fields are, in order, `microvmId`, `error`, `outcome`, `suspendHookArrived`, `terminating`, `closed`, `refusingJobs`, and `trigger`. `microvmId` is the MicroVM ID of the call. `error` holds the name and the message of the error. `outcome` is `rejected`, `refusedForNow`, or `unknown`. `suspendHookArrived` states whether a `suspend` hook arrived since the worker decided to make the call. `terminating` states whether a `terminate` hook came before the worker acted on the outcome. `closed` states whether `worker.close()` came before the worker acted on the outcome. `refusingJobs` states whether the worker answers jobs with HTTP 503 after it has acted on the outcome. `trigger` is `idle` for a call after the idle time, and `job` for a call after a `suspendAfter` job. The worker also logs two info lines about the call. It logs `suspending the idle MicroVM` before each call, with `microvmId`, `idleSeconds`, and `waitedSeconds`. It logs `stopping the suspend call: the MicroVM resumed during it` for the first `resume` hook during a call, with `microvmId`. The JavaScript worker logs the same stop line. +9. The worker logs one warning for each failed call. The message names the outcome. The fields are, in order, `microvmId`, `error`, `outcome`, `suspendHookArrived`, `terminating`, `closed`, and `refusingJobs`. `microvmId` is the MicroVM ID of the call. `error` holds the name and the message of the error. `outcome` is `rejected`, `refusedForNow`, or `unknown`. `suspendHookArrived` states whether a `suspend` hook arrived since the worker decided to make the call. `terminating` states whether a `terminate` hook came before the worker acted on the outcome. `closed` states whether `worker.close()` came before the worker acted on the outcome. `refusingJobs` states whether the worker answers jobs with HTTP 503 after it has acted on the outcome. The worker also logs two info lines about the call. It logs `suspending the idle MicroVM` before each call, with `microvmId`, `idleSeconds`, and `waitedSeconds`. It logs `stopping the suspend call: the MicroVM resumed during it` for the first `resume` hook during a call, with `microvmId`. The JavaScript worker logs the same stop line. -`worker.close()` stops the idle time, and drops a pending suspend after a job. A closed worker never suspends its MicroVM again. A refusal that is already running still ends on the `resume` hook or after 30 seconds. +`worker.close()` stops the idle time. A closed worker never suspends its MicroVM again. A refusal that is already running still ends on the `resume` hook or after 30 seconds. The worker counts only jobs. It cannot see background threads or processes, or inbound traffic that is not a job. A session with such work leaves `auto_suspend_on_idle` off, and the payload then has no `autoSuspendIdleSeconds`. @@ -126,22 +126,11 @@ The worker creates a `lambda-microvms` client for each suspend, and closes it af ## Suspending after a job -A session job can carry `"suspendAfter": true`. The durable function sets it for `vm.invoke` with `suspend_after=True`. The worker then suspends its own MicroVM after that job. This works in every session, with or without `autoSuspendIdleSeconds`. +The durable function suspends a session's MicroVM after a job. `vm.invoke` with `suspend_after=True` calls `SuspendMicrovm` itself, from the durable function, after the job's result arrives. The job request carries no field for it, and the worker does nothing for it. A request from an older durable function can still carry `"suspendAfter": true`. The worker ignores the field, sends no `X-Microvm-Suspend-After` header, and does not suspend. -1. Only a JSON `true` turns it on. The worker ignores any other value, and does not reject the job. The `run` hook payload never carries the field. -2. The worker answers the job with HTTP 202 and the header `X-Microvm-Suspend-After: accepted`. A second delivery of the job that gets the duplicate HTTP 202 carries the header when the first delivery carried the field. A second delivery refused with HTTP 503 carries no header. An older worker ignores the field and sends no header. The durable function then logs a warning, and the MicroVM keeps running. -3. When the job has ended, and its outcome reports have ended, the worker sets a pending suspend request. This happens whether the handler succeeded or failed. A job that ends while the worker refuses jobs for a suspend sets no request. That refusal comes from the worker's own call, a `suspend` hook, or the 30 seconds that follow either. The MicroVM is suspending already. So the request is done. -4. The request fires when no job runs, no callback failure report runs, no `run` hook request is being handled, and no `SuspendMicrovm` call runs. The worker checks these conditions again when a job ends, a report ends, a `run` hook request ends, and a call ends. So a request from a job that ends while another job runs fires when the last job ends. -5. Firing clears the request before the call starts. A `resume` hook never sets it again. A call after the idle time clears it too, because that call suspends the MicroVM as well. A `suspend` hook clears it too, because Lambda suspends the MicroVM after that hook. -6. Firing stops the idle time, when the session also suspends when idle. -7. The `terminate` hook and `worker.close()` drop the request, and a job end after either records none. -8. The worker calls `SuspendMicrovm` with the MicroVM ID of the first accepted `run` hook with a payload. It never uses the MicroVM ID of a job request. A forged request could otherwise suspend another MicroVM of the same image. A request that comes before the first accepted run hook waits for it. The request fires when that run hook names a MicroVM, and is dropped with the warning when it names none. The warning is `cannot suspend the MicroVM after a job: the run hook named no MicroVM`, with `callbackId`. The worker then does not suspend. -9. The call follows items 3 to 9 of [Suspending when idle](#suspending-when-idle): the refusal of jobs from the moment the request fires, one attempt and none after a `resume` hook, the outcome classification, and the 30-second refusal after the call. -10. A call that the service refused for now is not retried without `autoSuspendIdleSeconds`. With it, the worker tries again after the idle time, at least 30 seconds later, as in item 7 of [Suspending when idle](#suspending-when-idle). A rejected call is not retried. +So the MicroVM's execution role needs no `lambda:SuspendMicrovm` for this case. The durable function's role needs it. See the [extras README](../aws-durable-execution-sdk-python-extras/README.md#suspending-after-a-job). -Before the call, the worker logs `suspending the MicroVM after a job` at info level, with `microvmId` and `callbackId`. When a request starts to wait because no `run` hook has been accepted yet, the worker logs the warning `a job asked to suspend the MicroVM before the run hook arrived. The worker waits for the run hook to name the MicroVM.`, with `callbackId`. It logs this warning once per worker, not once per job. An image without the `run` hook never ends the wait, and this warning is then the only sign of it. A failed call logs the warning of item 9 of [Suspending when idle](#suspending-when-idle), with `trigger` set to `job`. - -Enable the `resume` hook in the image. Without it, the worker refuses the next job with HTTP 503 until 30 seconds after the call. The next job's heartbeat timeout counts that time. The MicroVM's execution role needs `lambda:SuspendMicrovm` on the image. Without it, each call is rejected and logged, and the MicroVM keeps running. +Enable the `suspend` and `resume` hooks in the image. The worker answers the `suspend` hook, refuses jobs while the MicroVM freezes, and accepts them again on the `resume` hook. Without the `resume` hook, the worker refuses the next job with HTTP 503 until 30 seconds after the `suspend` hook. The next job's heartbeat timeout counts that time. ## Suspend and terminate hooks @@ -192,7 +181,7 @@ The `runHookPayload` string holds: } ``` -`job` is absent when the job arrives over HTTP. An HTTP job request body has the job fields at the top level, with `version`, `region`, an optional `microvmId`, and an optional `suspendAfter`. `MicrovmJobRequest` holds those job fields in its `job` attribute, and `suspendAfter` in its `suspend_after` attribute. +`job` is absent when the job arrives over HTTP. An HTTP job request body has the job fields at the top level, with `version`, `region`, and an optional `microvmId`. `MicrovmJobRequest` holds those job fields in its `job` attribute. The worker ignores a `suspendAfter` field. A document that does not match raises `InvalidRunHookPayloadError`. When the document named a non-empty callback ID, the error carries `callback_id` and `region`, so the worker can fail that callback at once. @@ -226,7 +215,7 @@ The worker logs to the standard library logger `aws_durable_execution_sdk_python The MicroVM's execution role needs `lambda:SendDurableExecutionCallbackSuccess`, `lambda:SendDurableExecutionCallbackFailure`, and `lambda:SendDurableExecutionCallbackHeartbeat` on the durable function's ARN (`arn:aws:lambda:::function::*`). -A session that suspends when idle, or that sends a job with `suspendAfter`, also needs `lambda:SuspendMicrovm` on the MicroVM image (`arn:aws:lambda:::microvm-image:`). The action authorizes on the image, not on the MicroVM. So a MicroVM with this permission can suspend any MicroVM from the same image, even during a job. That job then waits until its heartbeat timeout or its timeout. Use one image per trust boundary. +A session that suspends when idle also needs `lambda:SuspendMicrovm` on the MicroVM image (`arn:aws:lambda:::microvm-image:`). The action authorizes on the image, not on the MicroVM. So a MicroVM with this permission can suspend any MicroVM from the same image, even during a job. That job then waits until its heartbeat timeout or its timeout. Use one image per trust boundary. The worker needs boto3 1.43.35 or later, the first release with the `lambda-microvms` client. diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/payload.py b/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/payload.py index 30a780ada..18e047ced 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/payload.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/payload.py @@ -235,14 +235,6 @@ class MicrovmJobRequest: """The MicroVM identifier. A job request can reach the worker before the ``run`` hook, which also carries the identifier. Until the ``run`` hook arrives, the worker takes it from the first job request that has one.""" - suspend_after: bool = False - """Whether the worker suspends its MicroVM after the job. - - Wire name: ``suspendAfter``. Only a JSON ``true`` turns it on. The - worker ignores any other value, as the JavaScript worker does. So a - document from either SDK has the same meaning in both workers. The - worker suspends the MicroVM named by the ``run`` hook, never the - ``microvmId`` of the request.""" @classmethod def from_dict(cls, body: object) -> MicrovmJobRequest: @@ -275,7 +267,6 @@ def from_dict(cls, body: object) -> MicrovmJobRequest: region=region, job=job, microvm_id=microvm_id, - suspend_after=body.get("suspendAfter") is True, ) diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/worker.py b/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/worker.py index 27ed6a6bd..9e340d11c 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/worker.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/src/aws_durable_execution_sdk_python_microvm_worker/worker.py @@ -112,16 +112,6 @@ _UNKNOWN_MICROVM_ID = "unknown" -SUSPEND_AFTER_HEADER = "X-Microvm-Suspend-After" -"""The header of a 202 answer that acknowledges ``"suspendAfter": true``. - -Its value is ``accepted``. The durable function reads it for a -``suspendAfter`` job. An older worker ignores the field and sends no header. -The durable function then logs a warning. The JavaScript worker sends the -same header.""" - -_SUSPEND_AFTER_ACK = {SUSPEND_AFTER_HEADER: "accepted"} - # At most 19 digits, so that int() stays under Python's limit on digits, and # any declared length still exceeds every body limit. _CONTENT_LENGTH = re.compile(r"[0-9]{1,19}") @@ -230,8 +220,6 @@ class _SuspendCall: started the refusal decided it.""" target: _SuspendTarget - trigger: str - """``idle`` for the idle time, ``job`` for a ``suspendAfter`` job.""" hooks_before: int """The suspend hooks counted when the refusal started.""" resumes_before: int @@ -242,16 +230,6 @@ class _SuspendCall: """The fields of that line.""" -@dataclass(frozen=True, kw_only=True) -class _JobSuspend: - """A pending suspend after a ``suspendAfter`` job that fires now.""" - - callback_id: str - """The job whose end asked for the suspend.""" - call: _SuspendCall | None - """The call to make. ``None`` when no run hook named a MicroVM.""" - - @dataclass(kw_only=True) class _ActiveJob: """A job whose handler has not settled yet, with what the ``terminate`` @@ -402,7 +380,7 @@ class MicrovmWorkerListener: 9. The listener logs one warning for each failed call. The message names the outcome. The fields are, in order, ``microvmId``, ``error``, ``outcome``, ``suspendHookArrived``, ``terminating``, - ``closed``, ``refusingJobs``, and ``trigger``. ``microvmId`` is the + ``closed``, and ``refusingJobs``. ``microvmId`` is the MicroVM ID of the call. ``error`` holds the name and the message of the error. ``outcome`` is ``rejected``, ``refusedForNow``, or ``unknown``. ``suspendHookArrived`` states whether a ``suspend`` hook @@ -411,31 +389,7 @@ class MicrovmWorkerListener: on the outcome. ``closed`` states whether ``close()`` came before the listener acted on the outcome. ``refusingJobs`` states whether the listener answers jobs with HTTP 503 after it has acted on the - outcome. ``trigger`` is ``idle`` or ``job``. - - A job request with ``"suspendAfter": true`` asks the listener to - suspend its MicroVM after the job, in any session: - - 1. The 202 answer carries ``X-Microvm-Suspend-After: accepted``. A - second delivery that gets the duplicate 202 carries the header when - the first one carried the field. A 503 never carries it. Only a - JSON ``true`` counts. Any other value is ignored. - 2. When the job's outcome reports have ended, the listener sets a - pending request. A job that ends while the listener refuses jobs for - a suspend sets none, because the MicroVM is suspending already. The - request fires when no job, failure report, ``run`` hook request, or - SuspendMicrovm call runs. Each of those ends checks it again. - 3. Firing clears the request, stops the idle time, and makes the same - call as items 3 to 9, with the MicroVM ID of the first accepted - ``run`` hook with a payload. A request that comes before the first - accepted ``run`` hook waits for it. The request fires when that hook - names a MicroVM. When it names none, the listener logs a warning, - drops the request, and does not suspend. A ``resume`` hook never - repeats the request. A ``suspend`` hook drops it. The ``terminate`` - hook and ``close()`` drop it, and a job end after either records - none. - 4. A call refused for now is retried only when the session suspends - when idle. + outcome. When the image enables the ``suspend`` and ``terminate`` hooks, the listener acts on them: @@ -497,12 +451,8 @@ def __init__( self._locked_depth = 0 # The callback IDs of the jobs that run now. self._jobs: set[str] = set() - # The running jobs whose request carried "suspendAfter": true. - self._suspend_after_jobs: set[str] = set() - # The callback IDs of jobs that ended, oldest first. The value states - # whether the job's request carried "suspendAfter": true. A resent - # copy of the job then gets the same acknowledgement. - self._finished: OrderedDict[str, bool] = OrderedDict() + # The callback IDs of jobs that ended, oldest first. + self._finished: OrderedDict[str, None] = OrderedDict() # Callback failure reports in flight, from every place that fails a # callback. The terminate hook waits for them, because Lambda can end # the MicroVM as soon as it gets the answer. @@ -527,20 +477,6 @@ def __init__( self._reporting: set[str] = set() # Set from the run hook payload of a session that suspends when idle. self._auto_suspend: _AutoSuspend | None = None - # Set from the first accepted run hook that has a payload, in every - # session. Both triggers of a suspend call use it. - self._suspend_target: _SuspendTarget | None = None - # The callback ID of the suspendAfter job whose end asked for a - # suspend that has not started yet. The request fires when no work - # runs, and waits for the first accepted run hook, which sets the - # target. A call that starts clears it, so a resume hook never - # repeats it. A suspend hook, the terminate hook, and close() clear - # it too. - self._suspend_requested: str | None = None - # Set when the worker has logged that a request waits for the run - # hook. An image without the run hook would make every such job log - # it. So the worker logs it once. - self._warned_run_hook_wait = False # Each timer action carries the generation it was started with, and # acts only when it is still current. Cancelling a timer increments # the generation. @@ -621,9 +557,9 @@ def create( listener closes that client when the job ends. create_microvms_client: Creates the ``lambda-microvms`` client that suspends this MicroVM. The listener calls it only when - a session asks the worker to suspend when idle, or after a - ``suspendAfter`` job, once per suspend. It never closes a - client that this function returns. Defaults to a client of + a session asks the worker to suspend when idle, once per + suspend. It never closes a client that this function + returns. Defaults to a client of its own per suspend, from the default credential chain. The listener closes that client after the call. The listener stops the client's attempts once a ``resume`` hook arrived. @@ -748,8 +684,8 @@ def idle(self, timeout: float | None = None) -> bool: ) def close(self) -> None: - """Stop the idle time, and drop a pending suspend after a job. The - listener then never suspends the MicroVM again. + """Stop the idle time. The listener then never suspends the MicroVM + again. Running jobs continue. A refusal that a suspend started still ends on the ``resume`` hook or after 30 seconds. @@ -757,7 +693,6 @@ def close(self) -> None: with self._lock: self._closed = True self._stop_idle_time() - self._suspend_requested = None def _dispatch(self, exchange: _Exchange) -> None: path = exchange.path @@ -772,8 +707,6 @@ def _dispatch(self, exchange: _Exchange) -> None: with self._locked(): self._run_hooks -= 1 self._changed.notify_all() - decision = self._take_job_suspend() - self._act_on_job_suspend(decision) return if hook in {"suspend", "terminate", "resume"} and not _is_lambda_hook_call( exchange @@ -818,14 +751,12 @@ def _handle_suspend_hook(self) -> None: """Refuse jobs until the ``resume`` hook, or for 30 seconds. Lambda freezes the MicroVM after the answer. A job accepted now would - freeze with it. Lambda suspends the MicroVM after this hook. So a - pending suspend after a job is done, and it is cleared. + freeze with it. """ with self._locked(): self._suspend_hooks += 1 self._stop_idle_time() self._suspending = True - self._suspend_requested = None self._start_grace() def _handle_resume_hook(self) -> list[tuple[MicrovmJobDocument, str]]: @@ -890,7 +821,6 @@ def _terminate(self) -> None: with self._lock: self._terminating = True self._stop_idle_time() - self._suspend_requested = None stopped: list[tuple[str, _ActiveJob]] = [] for callback_id, job in self._active.items(): if job.cancelled.is_set(): @@ -958,16 +888,12 @@ def _handle_run_hook(self, exchange: _Exchange) -> None: self._run_hook_accepted = True self._microvm_id = parsed.microvm_id if payload is not None: - # Every session gets a target, also without auto-suspend. - # A suspendAfter job needs one. - target = _SuspendTarget( - microvm_id=parsed.microvm_id, region=payload.region - ) - self._suspend_target = target if payload.auto_suspend_idle_seconds is not None: self._auto_suspend = _AutoSuspend( idle_seconds=payload.auto_suspend_idle_seconds, - target=target, + target=_SuspendTarget( + microvm_id=parsed.microvm_id, region=payload.region + ), ) microvm_id = self._microvm_id duplicate = job is not None and self._is_known(job.callback_id) @@ -1118,22 +1044,9 @@ def _handle_job_request( terminating = not duplicate and self._terminating suspending = not duplicate and self._suspending if not (duplicate or terminating or suspending): - self._register_job( - job.callback_id, - request.region, - suspend_after=request.suspend_after, - ) - # A resent copy that gets the duplicate 202 carries the - # acknowledgement that the first request got, because only the - # first request decides the suspend. A 503 carries none. - acknowledge = ( - self._carried_suspend_after(job.callback_id) - if duplicate - else request.suspend_after - ) - headers = _SUSPEND_AFTER_ACK if acknowledge else None + self._register_job(job.callback_id, request.region) if duplicate: - exchange.respond(202, {}, headers) + exchange.respond(202, {}) self._logger.warning( "duplicate job ignored", extra={"microvmId": microvm_id, "callbackId": job.callback_id}, @@ -1155,7 +1068,7 @@ def _handle_job_request( # retries. After a 202, the durable function does not deliver the job # again. self._start_job(handler, job, request.region) - exchange.respond(202, {}, headers) + exchange.respond(202, {}) def _reject(self, error: BaseException, exchange: _Exchange) -> None: """Answer 400 to a request that the worker could not accept. @@ -1188,16 +1101,7 @@ def _is_known(self, callback_id: str) -> bool: """ return callback_id in self._jobs or callback_id in self._finished - def _carried_suspend_after(self, callback_id: str) -> bool: - """Whether the request of a known job carried ``"suspendAfter": - true``. The caller holds the lock.""" - return callback_id in self._suspend_after_jobs or self._finished.get( - callback_id, False - ) - - def _register_job( - self, callback_id: str, region: str, *, suspend_after: bool = False - ) -> None: + def _register_job(self, callback_id: str, region: str) -> None: """Register a job that is about to start. The caller holds the lock. The job is registered for the terminate hook before its thread @@ -1206,8 +1110,6 @@ def _register_job( handler. """ self._jobs.add(callback_id) - if suspend_after: - self._suspend_after_jobs.add(callback_id) self._active[callback_id] = _ActiveJob( region=region, cancelled=threading.Event() ) @@ -1229,16 +1131,12 @@ def _start_job( # The job never ran. A job request then answers 500, so the # durable function delivers the job again, and the retry may run # it. The run hook fails the callback instead, because Lambda - # sends it only once. The job never ran, so it asks for no - # suspend. A suspend that another job asked for can fire now. + # sends it only once. with self._locked(): self._jobs.discard(job.callback_id) - self._suspend_after_jobs.discard(job.callback_id) self._active.pop(job.callback_id, None) self._start_idle_time() self._changed.notify_all() - decision = self._take_job_suspend() - self._act_on_job_suspend(decision) raise def _run_job( @@ -1252,43 +1150,15 @@ def _run_job( extra={"callbackId": job.callback_id, "error": describe(error)}, ) finally: - # The job's report attempts have ended here, whether the handler - # succeeded or failed. So a suspendAfter job asks for the suspend - # now. The suspend runs after this block, so that no lock is - # held while it logs or starts its thread. During a suspend - # refusal, the MicroVM is suspending already. So a job that ends - # then asks for nothing. A request that comes before the first - # accepted run hook waits for it. An image without the run hook - # never ends that wait. So the worker warns the first time a - # request starts to wait. - warn_run_hook_wait = False with self._locked(): callback_id = job.callback_id - suspend_after = callback_id in self._suspend_after_jobs self._jobs.discard(callback_id) - self._suspend_after_jobs.discard(callback_id) self._active.pop(callback_id, None) - self._finished[callback_id] = suspend_after + self._finished[callback_id] = None if len(self._finished) > MAX_FINISHED_JOBS: self._finished.popitem(last=False) self._start_idle_time() - if suspend_after and not ( - self._terminating or self._closed or self._suspending - ): - self._suspend_requested = callback_id - if not (self._run_hook_accepted or self._warned_run_hook_wait): - self._warned_run_hook_wait = True - warn_run_hook_wait = True self._changed.notify_all() - decision = self._take_job_suspend() - if warn_run_hook_wait: - self._logger.warning( - "a job asked to suspend the MicroVM before the run hook " - "arrived. The worker waits for the run hook to name the " - "MicroVM.", - extra={"callbackId": callback_id}, - ) - self._act_on_job_suspend(decision) def _run_job_and_report( self, handler: MicrovmJobHandler, job: MicrovmJobDocument, region: str @@ -1518,8 +1388,6 @@ def _report_ended(self) -> None: with self._locked(): self._reports -= 1 self._changed.notify_all() - decision = self._take_job_suspend() - self._act_on_job_suspend(decision) def _reporter_for(self, callback_id: str, region: str) -> CallbackReporter: # Each job gets its own client. A client kept across jobs would keep @@ -1658,7 +1526,6 @@ def _idle_time_ended(self, generation: int, waited_seconds: float) -> None: # because that idle time lasts at least SUSPEND_GRACE_SECONDS. call = self._begin_suspend_call( auto_suspend.target, - "idle", "suspending the idle MicroVM", { "microvmId": auto_suspend.target.microvm_id, @@ -1666,14 +1533,11 @@ def _idle_time_ended(self, generation: int, waited_seconds: float) -> None: "waitedSeconds": waited_seconds, }, ) - self._start_suspend_thread( - call, "could not start the suspend of the idle MicroVM" - ) + self._start_suspend_thread(call) def _begin_suspend_call( self, target: _SuspendTarget, - trigger: str, message: str, fields: Mapping[str, object], ) -> _SuspendCall: @@ -1681,16 +1545,12 @@ def _begin_suspend_call( call. The caller holds the lock, and starts the call after it releases the lock. - The call suspends the MicroVM either way. So a pending suspend after - a job is done by this call too, and it is cleared. """ self._suspending = True self._suspend_calls += 1 - self._suspend_requested = None self._stoppable_suspend = target.microvm_id return _SuspendCall( target=target, - trigger=trigger, # A suspend hook can arrive at any point from here on: while the # thread starts, while the client is created, or during the # call. Each path that ends the refusal compares with this count. @@ -1702,75 +1562,7 @@ def _begin_suspend_call( fields=fields, ) - def _take_job_suspend(self) -> _JobSuspend | None: - """Decide whether the pending suspend after a ``suspendAfter`` job - fires now. - - The request fires when no job, failure report, run hook, or - SuspendMicrovm call runs. Each event that can change one of these - calls this method: a job end, a report end, a run hook end, and a - call end. Firing clears the request, and stops an idle timer that - auto-suspend armed. - - The terminate hook and ``close()`` drop the request, and a job end - after either records none. A suspend refusal drops the request too, - and a job end during a refusal records none. So these states need - no check here. - - HTTP jobs can arrive before the run hook. A request that is due - before the first accepted run hook stays pending, because that hook - sets the target. The end of that hook calls this method again. The - request then fires when the hook named a MicroVM, and is dropped - with a warning when it named none. - - The caller holds the lock through :meth:`_locked`. It passes the - result to :meth:`_act_on_job_suspend` after it releases the lock. So - no log line, ``describe()``, or caller code runs under the lock, and - no locking method runs inside the caller's locked block. - """ - callback_id = self._suspend_requested - if ( - callback_id is None - or self._jobs - or self._reports - or self._run_hooks - or self._suspend_calls - ): - return None - target = self._suspend_target - if target is None and not self._run_hook_accepted: - return None - self._suspend_requested = None - if target is None: - return _JobSuspend(callback_id=callback_id, call=None) - self._stop_idle_time() - call = self._begin_suspend_call( - target, - "job", - "suspending the MicroVM after a job", - {"microvmId": target.microvm_id, "callbackId": callback_id}, - ) - return _JobSuspend(callback_id=callback_id, call=call) - - def _act_on_job_suspend(self, decision: _JobSuspend | None) -> None: - """Start the suspend that :meth:`_take_job_suspend` decided. The - caller does not hold the lock.""" - if decision is None: - return - if decision.call is None: - # A job request carries a MicroVM ID, but a forged request could - # name another MicroVM of the image. So only the run hook names - # the MicroVM to suspend. - self._logger.warning( - "cannot suspend the MicroVM after a job: the run hook named no MicroVM", - extra={"callbackId": decision.callback_id}, - ) - return - self._start_suspend_thread( - decision.call, "could not start the suspend of the MicroVM after a job" - ) - - def _start_suspend_thread(self, call: _SuspendCall, failure: str) -> None: + def _start_suspend_thread(self, call: _SuspendCall) -> None: """Run a SuspendMicrovm call in its own thread. The caller started the refusal and counted the call, and does not hold the lock.""" try: @@ -1784,22 +1576,20 @@ def _start_suspend_thread(self, call: _SuspendCall, failure: str) -> None: # No call was made. So jobs can run again at once, unless a # suspend hook started its own refusal meanwhile. self._logger.error( - failure, + "could not start the suspend of the idle MicroVM", extra={"microvmId": call.target.microvm_id, "error": describe(error)}, ) with self._locked(): deferred = self._end_own_refusal(call.hooks_before) - decision = self._suspend_call_ended() + self._suspend_call_ended() self._launch_deferred(deferred) - self._act_on_job_suspend(decision) def _suspend_self(self, call: _SuspendCall) -> None: try: self._call_suspend(call) finally: with self._locked(): - decision = self._suspend_call_ended() - self._act_on_job_suspend(decision) + self._suspend_call_ended() def _end_own_refusal( self, hooks_before: int @@ -1814,13 +1604,11 @@ def _end_own_refusal( return [] return self._stop_refusing() - def _suspend_call_ended(self) -> _JobSuspend | None: + def _suspend_call_ended(self) -> None: """Count a SuspendMicrovm call as ended, and start the idle time that a resume hook, a job, or a refusal for now asked for during it. - A suspend after a job that ended during the call can fire now. The - caller holds the lock through :meth:`_locked`, and passes the result - to :meth:`_act_on_job_suspend` after it releases the lock. + The caller holds the lock through :meth:`_locked`. """ self._suspend_calls -= 1 # The thread or the client could fail before the call was sent. @@ -1829,7 +1617,6 @@ def _suspend_call_ended(self) -> _JobSuspend | None: self._idle_after_call = False self._start_idle_time(after_refusal=self._idle_after_call_floor) self._changed.notify_all() - return self._take_job_suspend() def _call_suspend(self, call: _SuspendCall) -> None: target = call.target.microvm_id @@ -1898,8 +1685,7 @@ def _call_suspend(self, call: _SuspendCall) -> None: # retried after the next idle time, only with # auto-suspend. That idle time lasts at least 30 # seconds, because the same error can repeat on every - # call. A suspendAfter job asked for one call, so - # without auto-suspend nothing retries it. + # call. if outcome is SuspendOutcome.REFUSED_FOR_NOW: self._start_idle_time(after_refusal=True) message, extra = self._failed_call_log(outcome, described, call) @@ -1961,9 +1747,7 @@ def _failed_call_log( The message names only the outcome. The fields state whether a suspend hook arrived since the call was decided, whether the - listener is terminating or closed, whether it refuses jobs, and what - started the call. ``trigger`` is the last field, so the first seven - keep their order. + listener is terminating or closed, and whether it refuses jobs. The caller holds the lock, and has already acted on the outcome in the same locked block. So the fields state what the listener does @@ -1985,7 +1769,6 @@ def _failed_call_log( "closed": self._closed, # The job path refuses a job with HTTP 503 in both states. "refusingJobs": self._terminating or self._suspending, - "trigger": call.trigger, } def _start_grace(self) -> None: @@ -2316,14 +2099,8 @@ def read_json(self, max_bytes: int) -> Any: msg = "request body is not valid JSON" raise InvalidRunHookPayloadError(msg) from error - def respond( - self, - status: int, - body: Mapping[str, object], - headers: Mapping[str, str] | None = None, - ) -> None: - """Send the answer, with ``headers`` added. A client that has gone - away is logged.""" + def respond(self, status: int, body: Mapping[str, object]) -> None: + """Send the answer. A client that has gone away is logged.""" self.responded = True data = json.dumps(body, separators=(",", ":")).encode() request = self._request @@ -2331,8 +2108,6 @@ def respond( request.send_response(status) request.send_header("Content-Type", "application/json") request.send_header("Content-Length", str(len(data))) - for name, value in (headers or {}).items(): - request.send_header(name, value) # The header also makes the base class close the connection. request.send_header("Connection", "close") request.end_headers() diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py index 548579a86..b66b3ebf1 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py @@ -64,8 +64,6 @@ "terminating", "closed", "refusingJobs", - # The last field, so the first seven keep their order. - "trigger", ] @@ -78,7 +76,6 @@ def assert_warning( suspend_hook_arrived: bool = False, terminating: bool = False, closed: bool = False, - trigger: str = "idle", ) -> None: """Assert that one warning for a failed SuspendMicrovm call has this outcome and these fields.""" @@ -92,7 +89,6 @@ def assert_warning( "terminating": terminating, "closed": closed, "refusingJobs": refusing_jobs, - "trigger": trigger, } @@ -104,7 +100,6 @@ def assert_one_warning( suspend_hook_arrived: bool = False, terminating: bool = False, closed: bool = False, - trigger: str = "idle", ) -> dict[str, Any]: """Assert that the worker logged exactly one warning, for one failed SuspendMicrovm call, with this outcome and these fields. Returns the @@ -120,7 +115,6 @@ def assert_one_warning( suspend_hook_arrived=suspend_hook_arrived, terminating=terminating, closed=closed, - trigger=trigger, ) return data @@ -3498,32 +3492,36 @@ def create(region: str) -> Any: # --------------------------------------------------------------------------- -# Suspend after a suspendAfter job +# A job request no longer asks the worker to suspend # --------------------------------------------------------------------------- -ACK_HEADER = "X-Microvm-Suspend-After" -JOB_SUSPEND_INFO = "suspending the MicroVM after a job" -NO_TARGET_WARNING = ( - "cannot suspend the MicroVM after a job: the run hook named no MicroVM" -) -RUN_HOOK_WAIT_WARNING = ( - "a job asked to suspend the MicroVM before the run hook arrived. " - "The worker waits for the run hook to name the MicroVM." -) + +def test_ignores_a_suspend_after_field_of_a_job_request(start: Any) -> None: + # The durable function suspends the MicroVM after a session job. A + # request from an older durable function can still carry the field. + h = start() + h.run_hook() + + assert h.job("cb-1", suspendAfter=True) == 202 + h.finish_job("cb-1") + h.settle() + + assert h.microvms.suspended == [] + assert h.job("cb-2", suspendAfter=True) == 202 -def post_job(h: Harness, callback_id: str, **fields: Any) -> tuple[int, str | None]: - """Send a job request, and return the status and the acknowledgement - header.""" +def test_answers_a_job_request_with_no_suspend_after_header(start: Any) -> None: + h = start() + h.run_hook() connection = http.client.HTTPConnection( host_of(h.worker), h.worker.port, timeout=WAIT ) body = { "version": 1, "region": "us-east-1", - "callbackId": callback_id, + "callbackId": "cb-1", "input": {}, - **fields, + "suspendAfter": True, } try: connection.request( @@ -3536,638 +3534,6 @@ def post_job(h: Harness, callback_id: str, **fields: Any) -> tuple[int, str | No response.read() finally: connection.close() - return response.status, response.getheader(ACK_HEADER) - - -def info_lines(logger: Any, message: str) -> list[dict[str, Any]]: - return [data for level, m, data in logger.lines if level == "info" and m == message] - - -def end_job(h: Harness, callback_id: str) -> None: - """Release a job, wait until it has ended, and let a suspend call that - its end started end too.""" - h.finish_job(callback_id) - h.settle() - - -@pytest.mark.parametrize( - ("value", "acknowledged"), - [(True, "accepted"), (False, None), ("true", None), (1, None), (None, None)], - ids=["true", "false", "string", "one", "null"], -) -def test_acknowledges_only_a_json_true_and_suspends_only_for_it( - start: Any, value: Any, acknowledged: str | None -) -> None: - h = start() - h.run_hook() - - assert post_job(h, "cb-1", suspendAfter=value) == (202, acknowledged) - end_job(h, "cb-1") - - assert h.microvms.suspended == (["mvm-1"] if acknowledged else []) - - -def test_acknowledges_no_job_without_the_field(start: Any) -> None: - h = start() - h.run_hook() - - assert post_job(h, "cb-1") == (202, None) - - -@pytest.mark.parametrize("finished", [False, True], ids=["running", "finished"]) -def test_acknowledges_a_resent_copy_when_the_first_request_carried_the_field( - start: Any, finished: bool -) -> None: - h = start() - h.run_hook() - h.microvms.failure = suspend_error("AccessDeniedException", 403) - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - assert post_job(h, "cb-2") == (202, None) - if finished: - end_job(h, "cb-1") - end_job(h, "cb-2") - - # Only the first request decides the suspend. So a copy gets the same - # acknowledgement, whatever it carries. - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - assert post_job(h, "cb-1") == (202, "accepted") - assert post_job(h, "cb-2", suspendAfter=True) == (202, None) - - -def test_sends_no_acknowledgement_with_a_refusal(start: Any) -> None: - h = start() - h.run_hook() - h.hook("suspend") - - assert post_job(h, "cb-1", suspendAfter=True) == (503, None) - - -def test_refuses_a_resent_running_suspend_after_job_without_acknowledgement( - start: Any, -) -> None: - # The suspend refusal answers the resend of a running job with 503, so - # the copy carries no acknowledgement. The delivery after the resume gets - # the duplicate 202 with the acknowledgement of the first request. - h = start() - h.run_hook() - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - assert eventually(lambda: h.started("cb-1")) - assert h.hook("suspend") == 200 - - assert post_job(h, "cb-1", suspendAfter=True) == (503, None) - assert post_job(h, "cb-1") == (503, None) - - assert h.hook("resume") == 200 - assert post_job(h, "cb-1") == (202, "accepted") - end_job(h, "cb-1") - - assert h.jobs.started() == ["cb-1"] - assert h.microvms.suspended == ["mvm-1"] - - -@pytest.mark.parametrize("refusal_end", ["resume", "grace"]) -def test_a_job_that_ends_during_a_suspend_hook_refusal_asks_for_nothing( - start: Any, refusal_end: str -) -> None: - # Lambda suspends the MicroVM after the suspend hook. So a suspendAfter - # job that ends during the refusal asks for no call of the worker's own, - # neither then nor after the refusal ends. - h = start() - h.run_hook() - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - assert eventually(lambda: h.started("cb-1")) - assert h.hook("suspend", LAMBDA_HEADERS) == 200 - - end_job(h, "cb-1") - assert h.microvms.suspended == [] - assert h.listener._suspend_requested is None # noqa: SLF001 - - if refusal_end == "resume": - assert h.hook("resume", LAMBDA_HEADERS) == 200 - else: - h.advance(GRACE) - post_job(h, "cb-2") - end_job(h, "cb-2") - - assert h.jobs.started() == ["cb-1", "cb-2"] - assert h.microvms.suspended == [] - - -def test_a_suspend_hook_clears_a_pending_suspend_after_a_job(start: Any) -> None: - # The request waits for the other job. Lambda then suspends the MicroVM - # after the suspend hook. So the request is done, and the other job's end - # after the resume fires nothing. - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - post_job(h, "cb-2") - end_job(h, "cb-1") - assert h.listener._suspend_requested == "cb-1" # noqa: SLF001 - - assert h.hook("suspend", LAMBDA_HEADERS) == 200 - assert h.listener._suspend_requested is None # noqa: SLF001 - assert h.hook("resume", LAMBDA_HEADERS) == 200 - end_job(h, "cb-2") - - assert h.microvms.suspended == [] - - -def test_acknowledges_a_resent_running_suspend_after_job_after_the_terminate_hook( - start: Any, -) -> None: - # A known job gets the duplicate 202 while terminating, also during a - # suspend refusal. That 202 carries the acknowledgement of the first - # request. - h = start() - h.run_hook() - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - assert eventually(lambda: h.started("cb-1")) - assert h.hook("suspend") == 200 - assert h.hook("terminate") == 200 - - assert post_job(h, "cb-1") == (202, "accepted") - h.jobs.release_all() - assert h.worker.idle(WAIT) - - assert h.jobs.started() == ["cb-1"] - assert h.microvms.suspended == [] - - -def test_acknowledges_a_resent_finished_suspend_after_job_during_its_own_suspend( - start: Any, -) -> None: - # The job has ended, and the worker suspends after it. A resend during - # that refusal gets the duplicate 202 with the acknowledgement. - h = start() - h.run_hook() - assert post_job(h, "cb-1", suspendAfter=True) == (202, "accepted") - end_job(h, "cb-1") - assert h.microvms.suspended == ["mvm-1"] - - assert post_job(h, "cb-1") == (202, "accepted") - assert post_job(h, "cb-2", suspendAfter=True) == (503, None) - assert h.jobs.started() == ["cb-1"] - - -def test_suspends_after_a_suspend_after_job_in_a_session_without_auto_suspend( - start: Any, logger: Any -) -> None: - h = start() - h.run_hook() - assert post_job(h, "cb-1", suspendAfter=True)[0] == 202 - h.advance(IDLE * 10) - assert h.microvms.suspended == [] - - end_job(h, "cb-1") - - assert h.microvms.suspended == ["mvm-1"] - assert info_lines(logger, JOB_SUSPEND_INFO) == [ - {"microvmId": "mvm-1", "callbackId": "cb-1"} - ] - # The refusal runs from the decision, as after an idle suspend. - assert h.job("cb-2") == 503 - h.hook("resume", LAMBDA_HEADERS) - assert h.job("cb-2") == 202 - - -def test_suspends_after_a_job_whose_handler_failed(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True, input={"fail": True}) - - end_job(h, "cb-1") - - assert h.lambda_client.recorded()[0]["kind"] == "failure" - assert h.microvms.suspended == ["mvm-1"] - - -def test_suspends_after_a_job_whose_report_failed(start: Any) -> None: - h = start() - h.run_hook() - h.lambda_client.completion_error = suspend_error("AccessDeniedException", 403) - post_job(h, "cb-1", suspendAfter=True) - - end_job(h, "cb-1") - - assert h.lambda_client.recorded() == [] - assert h.microvms.suspended == ["mvm-1"] - - -def test_suspends_only_when_the_report_attempts_have_ended(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - h.lambda_client.hold = True - - h.jobs.release("cb-1") - assert eventually(lambda: h.lambda_client.waiting() == 1) - assert h.microvms.suspended == [] - - h.lambda_client.finish() - end_job(h, "cb-1") - assert h.lambda_client.recorded() == [{"kind": "success", "callbackId": "cb-1"}] - assert h.microvms.suspended == ["mvm-1"] - - -def test_suspends_when_the_last_running_job_ends(start: Any, logger: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - post_job(h, "cb-2") - - end_job(h, "cb-1") - assert h.microvms.suspended == [] - - end_job(h, "cb-2") - assert h.microvms.suspended == ["mvm-1"] - assert info_lines(logger, JOB_SUSPEND_INFO) == [ - {"microvmId": "mvm-1", "callbackId": "cb-1"} - ] - - -def test_waits_for_a_background_failure_report(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - h.lambda_client.hold = True - # An invalid request names a callback, so the worker fails it in the - # background. - assert h.job("cb-bad", heartbeatTimeoutSeconds=0) == 400 - assert eventually(lambda: h.lambda_client.waiting() == 1) - h.lambda_client.hold = False - - end_job(h, "cb-1") - assert h.microvms.suspended == [] - - h.lambda_client.finish() - assert eventually(lambda: h.microvms.suspended == ["mvm-1"]) - - -def test_waits_for_a_run_hook_that_is_being_handled(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - later = slow_run_hook(h) - - end_job(h, "cb-1") - assert h.microvms.suspended == [] - - assert later.finish() == 200 - assert eventually(lambda: h.microvms.suspended == ["mvm-1"]) - - -def test_suspends_the_run_hook_microvm_not_the_one_of_the_job_request( - start: Any, -) -> None: - h = start() - # The job request arrives before the run hook, and names another MicroVM. - post_job(h, "cb-1", suspendAfter=True, microvmId="mvm-other") - h.run_hook() - - end_job(h, "cb-1") - - assert h.microvms.suspended == ["mvm-1"] - - -def test_warns_and_does_not_suspend_when_no_run_hook_named_a_microvm( - start: Any, logger: Any -) -> None: - h = start() - assert h.post(f"{HOOK_PATH_PREFIX}run", {"microvmId": "mvm-1"}) == 200 - post_job(h, "cb-1", suspendAfter=True, microvmId="mvm-other") - - end_job(h, "cb-1") - - assert h.microvms.suspended == [] - assert [line for line in logger.lines if line[1] == NO_TARGET_WARNING] == [ - ("warning", NO_TARGET_WARNING, {"callbackId": "cb-1"}) - ] - # The request was consumed. A later job end does not warn again. - post_job(h, "cb-2") - end_job(h, "cb-2") - assert logger.messages("warning").count(NO_TARGET_WARNING) == 1 - assert h.job("cb-3") == 202 - -def test_a_request_before_the_run_hook_fires_when_the_run_hook_names_a_microvm( - start: Any, logger: Any -) -> None: - # HTTP jobs can arrive before the run hook. The job ends before the run - # hook, so no target exists yet. The request waits for the run hook, - # and fires with the run hook's MicroVM ID. - h = start() - assert post_job(h, "cb-1", suspendAfter=True, microvmId="mvm-other") == ( - 202, - "accepted", - ) - end_job(h, "cb-1") - assert h.microvms.suspended == [] - assert h.listener._suspend_requested == "cb-1" # noqa: SLF001 - assert NO_TARGET_WARNING not in logger.messages("warning") - - assert h.run_hook() == 200 - - assert eventually(lambda: h.microvms.suspended == ["mvm-1"]) - h.settle() - assert info_lines(logger, JOB_SUSPEND_INFO) == [ - {"microvmId": "mvm-1", "callbackId": "cb-1"} - ] - assert NO_TARGET_WARNING not in logger.messages("warning") - - -def test_a_request_before_the_run_hook_is_dropped_when_the_run_hook_names_none( - start: Any, logger: Any -) -> None: - # The run hook without a payload names no MicroVM to suspend. So the - # request that waited for it is dropped with the warning. - h = start() - post_job(h, "cb-1", suspendAfter=True, microvmId="mvm-other") - end_job(h, "cb-1") - assert NO_TARGET_WARNING not in logger.messages("warning") - - assert h.post(f"{HOOK_PATH_PREFIX}run", {"microvmId": "mvm-1"}) == 200 - - assert eventually(lambda: NO_TARGET_WARNING in logger.messages("warning")) - assert [line for line in logger.lines if line[1] == NO_TARGET_WARNING] == [ - ("warning", NO_TARGET_WARNING, {"callbackId": "cb-1"}) - ] - assert h.listener._suspend_requested is None # noqa: SLF001 - post_job(h, "cb-2") - end_job(h, "cb-2") - assert h.microvms.suspended == [] - assert logger.messages("warning").count(NO_TARGET_WARNING) == 1 - - -def test_warns_once_when_requests_wait_for_the_run_hook( - start: Any, logger: Any -) -> None: - # An image without the run hook never ends the wait. The warning is the - # only sign of that. Two jobs that wait log it once. - h = start() - post_job(h, "cb-1", suspendAfter=True) - end_job(h, "cb-1") - post_job(h, "cb-2", suspendAfter=True) - end_job(h, "cb-2") - - assert [line for line in logger.lines if line[1] == RUN_HOOK_WAIT_WARNING] == [ - ("warning", RUN_HOOK_WAIT_WARNING, {"callbackId": "cb-1"}) - ] - assert h.microvms.suspended == [] - - -def test_does_not_warn_of_the_wait_when_the_run_hook_came_first( - start: Any, logger: Any -) -> None: - h = start() - # The job request arrives before the run hook, and ends after it. - post_job(h, "cb-1", suspendAfter=True) - h.run_hook() - - end_job(h, "cb-1") - - assert h.microvms.suspended == ["mvm-1"] - assert RUN_HOOK_WAIT_WARNING not in logger.messages("warning") - - -def test_close_drops_a_request_that_waits_for_the_run_hook(start: Any) -> None: - h = start() - post_job(h, "cb-1", suspendAfter=True) - end_job(h, "cb-1") - assert h.listener._suspend_requested == "cb-1" # noqa: SLF001 - - h.listener.close() - assert h.listener._suspend_requested is None # noqa: SLF001 - assert h.run_hook() == 200 - h.settle() - - assert h.microvms.suspended == [] - assert h.listener._suspend_requested is None # noqa: SLF001 - - -def test_a_resume_hook_does_not_repeat_the_suspend(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - end_job(h, "cb-1") - assert h.microvms.suspended == ["mvm-1"] - - h.hook("resume", LAMBDA_HEADERS) - h.advance(GRACE * 10) - - assert h.microvms.suspended == ["mvm-1"] - # A job that ends after the resume asks for nothing either. - post_job(h, "cb-2") - end_job(h, "cb-2") - assert h.microvms.suspended == ["mvm-1"] - - -def test_does_not_retry_a_refusal_for_now_without_auto_suspend( - start: Any, logger: Any -) -> None: - h = start() - h.microvms.failure = suspend_error("ExpiredTokenException", 400) - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - - end_job(h, "cb-1") - h.advance(GRACE * 10) - - assert h.microvms.suspended == ["mvm-1"] - assert_one_warning(logger, "refusedForNow", refusing_jobs=False, trigger="job") - assert h.job("cb-2") == 202 - - -def test_retries_a_refusal_for_now_with_auto_suspend(start: Any) -> None: - h = start() - h.microvms.failure = suspend_error("ExpiredTokenException", 400) - h.run_hook(IDLE) - post_job(h, "cb-1", suspendAfter=True) - - end_job(h, "cb-1") - assert h.microvms.suspended == ["mvm-1"] - h.advance(GRACE - 0.001) - assert h.microvms.suspended == ["mvm-1"] - - h.advance(0.001) - assert h.microvms.suspended == ["mvm-1", "mvm-1"] - - -def test_stops_the_idle_timer_that_the_job_end_armed(start: Any, logger: Any) -> None: - # The job end arms the idle time, and fires the suspend in the same - # block. A rejected call does not retry until a job ends. So the stale - # timer must not fire one idle time later. - h = start() - h.microvms.failure = suspend_error("AccessDeniedException", 403) - h.run_hook(IDLE) - post_job(h, "cb-1", suspendAfter=True) - - end_job(h, "cb-1") - h.advance(IDLE * 10) - - assert h.microvms.suspended == ["mvm-1"] - assert_one_warning(logger, "rejected", refusing_jobs=False, trigger="job") - - -def test_logs_an_unknown_outcome_of_the_job_suspend_with_its_trigger( - start: Any, logger: Any -) -> None: - h = start() - h.microvms.failure = suspend_error("InternalServerException", 500) - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - - end_job(h, "cb-1") - - assert_one_warning(logger, "unknown", refusing_jobs=True, trigger="job") - assert h.job("cb-2") == 503 - - -def test_does_not_suspend_after_the_terminate_hook(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - post_job(h, "cb-2") - end_job(h, "cb-1") - - assert h.hook("terminate", LAMBDA_HEADERS) == 200 - assert h.listener._suspend_requested is None # noqa: SLF001 - h.jobs.release_all() - assert h.worker.idle(WAIT) - - assert h.microvms.suspended == [] - assert h.listener._suspend_requested is None # noqa: SLF001 - - -def test_records_no_request_for_a_job_that_ends_after_the_terminate_hook( - start: Any, -) -> None: - # The terminate hook clears no request here, because none is pending - # yet. Only the job end's check of the terminate hook stops the - # suspend. - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - - assert h.hook("terminate", LAMBDA_HEADERS) == 200 - end_job(h, "cb-1") - assert h.worker.idle(WAIT) - - assert h.listener._suspend_requested is None # noqa: SLF001 - assert h.microvms.suspended == [] - - -def test_does_not_suspend_after_close(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - post_job(h, "cb-2") - end_job(h, "cb-1") - - h.listener.close() - end_job(h, "cb-2") - - assert h.microvms.suspended == [] - assert h.listener._suspend_requested is None # noqa: SLF001 - - -def test_a_job_that_ends_after_close_asks_for_no_suspend(start: Any) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - - h.listener.close() - end_job(h, "cb-1") - - assert h.microvms.suspended == [] - assert h.listener._suspend_requested is None # noqa: SLF001 - - -def test_fires_a_pending_suspend_when_the_running_call_ends(start: Any) -> None: - # 1. The idle call runs, and a resume hook ends its refusal. - # 2. A suspendAfter job runs and ends while the call still runs. - # 3. The request waits for the call, and fires when it ends. - h = start() - h.microvms.hold = True - h.run_hook(IDLE) - h.advance(IDLE) - assert eventually(lambda: h.microvms.suspended == ["mvm-1"]) - h.hook("resume", LAMBDA_HEADERS) - assert post_job(h, "cb-1", suspendAfter=True)[0] == 202 - h.finish_job("cb-1") - assert h.microvms.suspended == ["mvm-1"] - - h.microvms.finish() - - assert eventually(lambda: h.microvms.suspended == ["mvm-1", "mvm-1"]) - h.settle() - - -def test_an_idle_call_does_the_pending_suspend_too(start: Any) -> None: - # A background report holds the request. The idle time ends first, and - # its call suspends the MicroVM. So the request is done, and it does not - # fire again after the resume. - h = start() - h.run_hook(IDLE) - post_job(h, "cb-1", suspendAfter=True) - h.lambda_client.hold = True - assert h.job("cb-bad", heartbeatTimeoutSeconds=0) == 400 - assert eventually(lambda: h.lambda_client.waiting() == 1) - h.lambda_client.hold = False - end_job(h, "cb-1") - assert h.microvms.suspended == [] - - h.advance(IDLE) - assert h.microvms.suspended == ["mvm-1"] - h.lambda_client.finish() - assert h.worker.idle(WAIT) - h.hook("resume", LAMBDA_HEADERS) - post_job(h, "cb-2") - end_job(h, "cb-2") - - assert h.microvms.suspended == ["mvm-1"] - - -def test_idle_waits_for_the_suspend_after_a_job(start: Any) -> None: - h = start() - h.microvms.hold = True - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - - h.finish_job("cb-1") - - assert not h.worker.idle(0.1) - h.microvms.finish() - assert h.worker.idle(WAIT) - - -def test_accepts_jobs_when_the_thread_of_the_job_suspend_cannot_start( - start: Any, logger: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - h = start() - h.run_hook() - post_job(h, "cb-1", suspendAfter=True) - failing_thread_start(monkeypatch, "microvm-suspend") - - h.finish_job("cb-1") - - assert "could not start the suspend of the MicroVM after a job" in ( - logger.messages("error") - ) - assert h.microvms.suspended == [] - monkeypatch.undo() - assert h.job("cb-2") == 202 - - -def test_a_suspend_after_job_whose_thread_cannot_start_asks_for_nothing( - start: Any, monkeypatch: pytest.MonkeyPatch -) -> None: - h = start() - h.run_hook() - failing_thread_start(monkeypatch, "microvm-job") - - assert post_job(h, "cb-1", suspendAfter=True) == (500, None) - - monkeypatch.undo() - assert h.listener._suspend_after_jobs == set() # noqa: SLF001 - assert h.listener._suspend_requested is None # noqa: SLF001 - assert h.microvms.suspended == [] + assert response.status == 202 + assert response.getheader("X-Microvm-Suspend-After") is None diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py index dcd8dcf11..7a55a68e0 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py @@ -286,24 +286,12 @@ def test_an_unusable_callback_id_is_not_attached_to_errors(callback_id): # endregion dataclasses and callback IDs -@pytest.mark.parametrize( - ("overrides", "suspend_after"), - [ - ({"suspendAfter": True}, True), - ({"suspendAfter": False}, False), - ({"suspendAfter": "true"}, False), - ({"suspendAfter": 1}, False), - ({"suspendAfter": None}, False), - ({"suspendAfter": {}}, False), - ({}, False), - ], - ids=["true", "false", "string", "one", "null", "object", "absent"], -) -def test_job_request_reads_only_json_true_as_suspend_after(overrides, suspend_after): - # Any other value is ignored, not rejected, as in the JavaScript worker. - parsed = MicrovmJobRequest.from_dict(job_request(**overrides)) +def test_job_request_ignores_a_suspend_after_field(): + # The durable function suspends the MicroVM itself. An older durable + # function can still send the field, and the worker ignores it. + parsed = MicrovmJobRequest.from_dict(job_request(suspendAfter=True)) - assert parsed.suspend_after is suspend_after + assert not hasattr(parsed, "suspend_after") def test_run_hook_payload_ignores_suspend_after(): From b23cafd0527aa7dd0a6a6082e623a69d7d25e84c Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Tue, 3 Nov 2026 22:47:19 +0000 Subject: [PATCH 3/9] fix(extras): bound the suspend and move the flag SuspendMicrovm runs through call_before_deadline with the invocation reserve. An expired deadline logs the warning, and the job result is returned. The may-be-suspended flag is set after the job's child context returns, or fails with a job error. So the suspend_after job makes no state check before its own delivery. The warning carries the SuspendMicrovm request ID and says the MicroVM is billed until idle suspend or the session ends. Adds tests for a failed job without suspend_after, a retryable SDK error, and the replay of a failed job. --- .../README.md | 8 +- .../microvm/aws_errors.py | 1 + .../microvm/config.py | 7 +- .../microvm/operation.py | 28 ++- .../microvm/session.py | 35 +-- .../tests/microvm/session_test.py | 233 +++++++++++++++--- 6 files changed, 258 insertions(+), 54 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/README.md b/packages/aws-durable-execution-sdk-python-extras/README.md index 9b67f0c3a..3e5017759 100644 --- a/packages/aws-durable-execution-sdk-python-extras/README.md +++ b/packages/aws-durable-execution-sdk-python-extras/README.md @@ -303,7 +303,7 @@ What happens: 2. It adds no step and no checkpoint. The child context's own completion checkpoint covers it. On replay, a completed child context does not run again, so the session sends no second suspend. 3. A failed `SuspendMicrovm` call logs a warning and does not fail the job. The next job already checks the state and resumes the MicroVM when needed. 4. The job request carries no field for it, and the worker does nothing for it. The worker needs no change, and the MicroVM's role needs no suspend permission. -5. The session records that the MicroVM may be suspended. Each later job of the session handle checks the MicroVM state before delivery, and resumes a suspended MicroVM, as with `auto_suspend_on_idle`. The `suspend_after` job itself checks the state too, because the record is set before the job starts. +5. The session records that the MicroVM may be suspended. Each later job of the session handle checks the MicroVM state before delivery, and resumes a suspended MicroVM, as with `auto_suspend_on_idle`. The record is set after the `suspend_after` job's child context returns or raises, so that job does not check the state before its own delivery. The setting works with or without `auto_suspend_on_idle`. With both, the function suspends right after the job, and the worker's idle time applies after each resume. @@ -311,7 +311,7 @@ The call is not a durable operation, so these rules are yours to follow. The ses 1. Do not use it on a job that runs next to other jobs of the session (`map`, `parallel`, promise combinators). Those jobs freeze until the next job resumes the MicroVM, and can hit their heartbeat timeout. 2. The suspend runs inside the job's child context, so it follows the child context replay rules. A result larger than the child context checkpoint limit makes the child context run again on replay, and each run suspends again. -3. A retry after the Lambda invocation fails can send the suspend again. Suspending a suspended MicroVM succeeds. +3. A retry after the Lambda invocation fails can send the suspend again. The invocation can also end while the call runs, when other operations wait. The call is then sent again. Suspending a suspended MicroVM succeeds. 4. Adding or removing `suspend_after` changes what the session does on replay. Change it only through a new function version. The record that a job set `suspend_after` is kept in memory only. A replay of a completed job calls `vm.invoke` again, so it sets the record again. A replay of a completed `map` item, `parallel` branch, or child context does not run its function again. So the next job can skip the state check of a suspended MicroVM. For that case, every session job checks the state after a sign of a suspended MicroVM: @@ -337,7 +337,9 @@ It needs: 2. A `heartbeat_timeout` on the next job of at least its `retry_window` plus 30 seconds. The heartbeat timeout starts before delivery, so it includes the resume and the refusal. 3. `lambda:GetMicrovm`, `lambda:ResumeMicrovm`, and `lambda:SuspendMicrovm` on the image in the function's role. The MicroVM's role needs `lambda:SuspendMicrovm` only for `auto_suspend_on_idle`. -A failed suspend is not retried. The MicroVM then keeps running until the next job, the idle suspend, or the end of the session. The warning names `lambda:SuspendMicrovm` for a missing permission. +The suspend runs after a job timeout or a heartbeat timeout too, so a job that still runs in the MicroVM is frozen, not stopped. + +The call ends at the invocation reserve, like the other calls of the session. A failed suspend, or a call that does not end in time, is not retried. The MicroVM then keeps running until the next job, the idle suspend, or the end of the session. The warning names `lambda:SuspendMicrovm` for a missing permission. ### Session permissions diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/aws_errors.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/aws_errors.py index d18de11e6..74973bc49 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/aws_errors.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/aws_errors.py @@ -26,6 +26,7 @@ "ResumeMicrovm", "CreateMicrovmAuthToken", "TerminateMicrovm", + "SuspendMicrovm", ] """The Lambda MicroVMs API operations that the MicroVM operations call.""" diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py index 3fb29deda..d67708d52 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py @@ -382,7 +382,9 @@ class MicrovmInvokeOptions(MicrovmRequestConfig): checkpoint limit makes the child context run again on replay, and each run suspends again. 4. A retry after the Lambda invocation fails can send the suspend again. - Suspending a suspended MicroVM succeeds. + The invocation can also end while the call runs, when other + operations wait. The call is then sent again. Suspending a suspended + MicroVM succeeds. 5. The function's execution role needs ``lambda:SuspendMicrovm`` on the MicroVM image. 6. Adding or removing ``suspend_after`` changes what the session does on @@ -390,7 +392,8 @@ class MicrovmInvokeOptions(MicrovmRequestConfig): A failed ``SuspendMicrovm`` call logs a warning and does not fail the job. The MicroVM then keeps running until the next job or the end of - the session. + the session. The call ends at the invocation reserve, like the other + calls of the session, and that counts as a failed call. Set it on the job before a long wait, such as a durable wait or a callback. Do not set it on back-to-back jobs or on the last job of the diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py index 1fa7dc6f5..a5bd05b9e 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py @@ -77,7 +77,9 @@ DEFAULT_REQUEST_RETRY_WINDOW_SECONDS, MAX_REQUEST_RETRY_WINDOW_SECONDS, SendJobOptions, + call_before_deadline, require_time_for_call, + reserve_starts_at, send_job, ) from aws_durable_execution_sdk_python_extras.microvm.retry import ( @@ -758,15 +760,31 @@ def suspend(self, child: DurableContext, microvm_id: str) -> None: and resumes it when needed, and suspending a suspended MicroVM succeeds. """ + + def expired() -> Exception: + return TimeoutError( + "SuspendMicrovm did not end before the invocation reserve" + ) + try: - self._client().suspend_microvm(microvmIdentifier=microvm_id) + call_before_deadline( + lambda: self._client().suspend_microvm(microvmIdentifier=microvm_id), + reserve_starts_at(_remaining_seconds(child)), + expired, + ) except Exception as error: # noqa: BLE001 + detail = ( + str(add_request_id("SuspendMicrovm", error)) + if isinstance(error, ClientError) + else str(error) + ) child.logger.warning( f'MicroVM "{self.name}": SuspendMicrovm failed for {microvm_id}. ' - "The MicroVM keeps running, and the next job checks its state. " - "A missing permission needs lambda:SuspendMicrovm in the " - "function's execution role.", - extra={"microvmId": microvm_id, "error": str(error)}, + "The MicroVM keeps running and is billed until the idle suspend " + "or the end of the session. The next job checks its state. A " + "missing permission needs lambda:SuspendMicrovm in the " + f"function's execution role. Error: {detail}", + extra={"microvmId": microvm_id, "error": detail}, ) def _terminate_call(self, microvm_id: str) -> None: diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py index 9e54299b2..bd7f51028 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py @@ -218,11 +218,14 @@ def __init__( self._scope = scope self._launched = launched self._resume_before_delivery = resume_before_delivery - # Set by the first suspend_after job of this handle. Later jobs then - # check the state before delivery. The flag lives in memory only. A - # replay of a completed job, map item, or child context does not run - # its invoke() again, so it does not set the flag. The check after a - # sign of a suspended MicroVM covers that case. + # Set when a suspend_after job of this handle has returned or raised. + # Later jobs then check the state before delivery. The job itself + # does not, because its suspend has not run when it delivers. A + # replay of a completed job calls invoke() again, so it sets the flag + # too. The flag lives in memory only. A replay of a completed map + # item, parallel branch, or child context does not run its invoke() + # again, so it does not set the flag. The check after a sign of a + # suspended MicroVM covers that case. self._may_be_suspended = False @property @@ -319,12 +322,6 @@ def invoke( suspend_after = _validate_suspend_after(name, options.suspend_after) _validate_input(name, job_input) launched = self._launched - if suspend_after: - # The flag is set before the child context. A replay of a - # completed job returns its result without running the child - # context's function. So the flag is set on every replay of this - # call too. - self._may_be_suspended = True def run(job_context: DurableContext) -> Any: callback = create_job_callback( @@ -359,9 +356,19 @@ def run(job_context: DurableContext) -> Any: return result try: - return context.run_in_child_context(run, name=name) - except ChildContextError as error: - raise MicrovmError.from_child_context_error(name, error) from error + result = context.run_in_child_context(run, name=name) + except Exception as error: + # SuspendExecution is a BaseException, so a wait for the job + # does not set the flag. Only a failed job sets it: any other + # error ends the invocation, and the flag lives in memory only. + if isinstance(error, ChildContextError): + if suspend_after: + self._may_be_suspended = True + raise MicrovmError.from_child_context_error(name, error) from error + raise + if suspend_after: + self._may_be_suspended = True + return result @dataclass(frozen=True, kw_only=True) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index d92e150a7..5b6715fcd 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -9,6 +9,8 @@ import hashlib import json import logging +import threading +import time from collections.abc import Callable, Iterator from types import SimpleNamespace from typing import Any @@ -35,6 +37,7 @@ DurableFunctionTestResult, DurableFunctionTestRunner, ) +from botocore.exceptions import ClientError from botocore.stub import ANY, Stubber from aws_durable_execution_sdk_python_extras.microvm import ( @@ -49,7 +52,9 @@ microvm_session, operation, ) - +from aws_durable_execution_sdk_python_extras.microvm.request import ( + INVOCATION_RESERVE_SECONDS, +) WAIT = 30 INGRESS = ( @@ -1529,10 +1534,11 @@ def test_makes_no_state_check_for_a_launch_checkpoint_without_a_bool_resume_mode # suspend_after, and the check after a sign of a suspended MicroVM # --------------------------------------------------------------------------- -SUSPEND_WARNING = ( +SUSPEND_WARNING_START = ( 'MicroVM "pipeline": SuspendMicrovm failed for mvm-1. The MicroVM keeps ' - "running, and the next job checks its state. A missing permission needs " - "lambda:SuspendMicrovm in the function's execution role." + "running and is billed until the idle suspend or the end of the session. " + "The next job checks its state. A missing permission needs " + "lambda:SuspendMicrovm in the function's execution role. Error: " ) STATE_CHECK_WARNING = ( "GetMicrovm failed, so the operation could not check whether the MicroVM " @@ -1548,6 +1554,15 @@ def warnings(caplog: pytest.LogCaptureFixture, message: str) -> int: ) +def suspend_warnings(caplog: pytest.LogCaptureFixture) -> list[str]: + return [ + record.getMessage() + for record in caplog.records + if record.levelname == "WARNING" + and record.getMessage().startswith(SUSPEND_WARNING_START[:40]) + ] + + def suspend_then_wait_then_job( vm: MicrovmSession, ctx: DurableContext, event: Any ) -> Any: @@ -1633,15 +1648,15 @@ def test_logs_a_warning_and_returns_the_result_when_the_suspend_fails( run: Any, fake_client: Any, fake_endpoint: Any, - make_error: Any, caplog: pytest.LogCaptureFixture, ) -> None: endpoint = fake_endpoint() - client = fake_client( - endpoint=endpoint.url, - suspends=[make_error("AccessDeniedException", 403)], - states=["RUNNING", "RUNNING"], - ) + response: Any = { + "Error": {"Code": "AccessDeniedException", "Message": "not allowed"}, + "ResponseMetadata": {"HTTPStatusCode": 403, "RequestId": "req-suspend-1"}, + } + error = ClientError(response, "SuspendMicrovm") + client = fake_client(endpoint=endpoint.url, suspends=[error], states=["RUNNING"]) def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: build = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) @@ -1655,9 +1670,12 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: ) assert outcome(result) == {"result": [1, 2]} - assert warnings(caplog, SUSPEND_WARNING) == 1 + [warning] = suspend_warnings(caplog) + assert warning.startswith(SUSPEND_WARNING_START) + # The error carries the request ID of the failed call, as in the JS SDK. + assert "SuspendMicrovm request ID: req-suspend-1." in warning # The next job checks the state, as after a suspend that worked. - assert len(client.requests("get_microvm")) == 2 + assert len(client.requests("get_microvm")) == 1 assert len(endpoint.requests) == 2 @@ -1676,14 +1694,167 @@ def test_does_not_suspend_a_job_that_did_not_ask_for_it( assert client.requests("suspend_microvm") == [] -def test_checks_the_state_before_the_jobs_after_a_suspend_after_job( +def test_does_not_suspend_after_a_failed_job_that_did_not_ask_for_it( run: Any, fake_client: Any, fake_endpoint: Any ) -> None: - endpoint = fake_endpoint() + client = fake_client(endpoint=fake_endpoint().url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + try: + vm.invoke(ctx, "build", job_input=1, options=job()) + except MicrovmJobFailedError as error: + return str(error) + return None + + result = run( + session_handler(session_config(client), body), + None, + [("build", fail_with("BuildError", "no compiler"))], + ) + + assert "no compiler" in outcome(result)["result"] + assert client.requests("suspend_microvm") == [] + + +def test_does_not_suspend_when_the_wait_of_a_suspend_after_job_raises_an_sdk_error( + run: Any, + fake_client: Any, + fake_endpoint: Any, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A retryable InvocationError is not a job outcome. The first wait + # raises it, so the session sends no suspend for it. The next invocation + # waits for the same job, and that outcome sends the one suspend. + original = Callback.result + raised: list[bool] = [] + + def result_once(self: Any) -> Any: + if not raised: + raised.append(True) + suspends_when_raised.append(len(client.requests("suspend_microvm"))) + msg = "store throttled" + raise RetryableSerDesError(msg) + return original(self) + + suspends_when_raised: list[int] = [] + monkeypatch.setattr(Callback, "result", result_once) + client = fake_client(endpoint=fake_endpoint().url) + + result = run( + session_handler( + session_config(client), + lambda vm, ctx, event: vm.invoke( + ctx, "build", job_input=1, options=job(suspend_after=True) + ), + ), + None, + [("build", succeed_with("built"))], + ) + + assert outcome(result) == {"result": "built"} + assert raised == [True] + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + + +def test_sends_no_second_suspend_when_a_completed_failed_job_replays( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + client = fake_client(endpoint=fake_endpoint().url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + try: + vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + except MicrovmJobFailedError: + pass + # The wait ends the invocation. The next invocation replays the + # completed failed job. + ctx.wait(Duration.from_seconds(1), name="pause") + return "done" + + result = run( + session_handler(session_config(client), body), + None, + [("build", fail_with("BuildError", "no compiler"))], + ) + + assert outcome(result) == {"result": "done"} + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + + +def test_ends_the_suspend_at_the_invocation_reserve_and_returns_the_result( + run: Any, + fake_client: Any, + fake_endpoint: Any, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # 1.5 seconds remain before the reserve. The suspend call blocks longer. + monkeypatch.setattr( + operation, + "_remaining_seconds", + lambda context: lambda: INVOCATION_RESERVE_SECONDS + 1.5, + ) + release = threading.Event() client = fake_client( - endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] + endpoint=fake_endpoint().url, + suspends=[lambda request: release.wait(20)], + ) + elapsed: list[float] = [] + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + started = time.monotonic() + value = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + elapsed.append(time.monotonic() - started) + return value + + try: + with caplog.at_level(logging.WARNING): + result = run( + session_handler(session_config(client), body), + None, + [("build", succeed_with(1))], + ) + finally: + release.set() + + assert outcome(result) == {"result": 1} + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + assert elapsed[0] < 10 + [warning] = suspend_warnings(caplog) + assert warning.startswith(SUSPEND_WARNING_START) + assert "did not end before the invocation reserve" in warning + + +def test_checks_the_state_before_the_job_after_a_failed_suspend_after_job( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + endpoint = fake_endpoint() + client = fake_client(endpoint=endpoint.url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + try: + vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + except MicrovmJobFailedError: + pass + return vm.invoke(ctx, "test", job_input=2, options=job()) + + result = run( + session_handler(session_config(client), body), + None, + [("build", fail_with("BuildError", "no compiler")), ("test", succeed_with(2))], ) + assert outcome(result) == {"result": 2} + # The failed job set the flag, so only the second job checks the state. + assert len(client.requests("get_microvm")) == 1 + + +def test_checks_the_state_before_the_jobs_after_a_suspend_after_job( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + endpoint = fake_endpoint() + client = fake_client(endpoint=endpoint.url, states=["SUSPENDED", "RUNNING"]) + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: build = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) return [build, vm.invoke(ctx, "test", job_input=2, options=job())] @@ -1695,10 +1866,12 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: ) assert outcome(result) == {"result": [1, 2]} - # The flag is set before the first job's child context. So the first job - # checks the state too: RUNNING. The second job finds the MicroVM - # SUSPENDED, resumes it, and sends the job when it runs. - assert len(client.requests("get_microvm")) == 3 + # The flag is set after the first job's child context. So the first job + # makes no GetMicrovm before its own delivery. The second job finds the + # MicroVM SUSPENDED, resumes it, and sends the job when it runs. + names = [name for name, _ in client.calls] + assert names.index("get_microvm") > names.index("suspend_microvm") + assert len(client.requests("get_microvm")) == 2 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert len(endpoint.requests) == 2 @@ -1708,12 +1881,10 @@ def test_sets_the_flag_again_when_a_completed_suspend_after_job_replays( ) -> None: # The second job runs in a later invocation. The first job's child # context is complete then, so its function does not run again. invoke() - # sets the flag before the child context, so the second job still + # sets the flag after the child context returns, so the second job still # checks the state before delivery. endpoint = fake_endpoint() - client = fake_client( - endpoint=endpoint.url, states=["RUNNING", "SUSPENDED", "RUNNING"] - ) + client = fake_client(endpoint=endpoint.url, states=["SUSPENDED", "RUNNING"]) handler, invocations = handler_across_deploy( client, {}, {}, suspend_then_wait_then_job ) @@ -1723,7 +1894,7 @@ def test_sets_the_flag_again_when_a_completed_suspend_after_job_replays( assert outcome(result) == {"result": [1, 2]} assert sent_in[1] > sent_in[0] + 1 - assert len(client.requests("get_microvm")) == 3 + assert len(client.requests("get_microvm")) == 2 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert len(endpoint.requests) == 2 # The completed job does not run again on replay. So the session does @@ -1747,7 +1918,7 @@ def test_resumes_through_the_fallback_after_a_map_item_lost_the_flag( endpoint = fake_endpoint([202, 202, connect_failed]) client = fake_client( endpoint=endpoint.url, - states=["RUNNING", "RUNNING", "SUSPENDED", "RUNNING"], + states=["RUNNING", "SUSPENDED", "RUNNING"], ) def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: @@ -1776,9 +1947,10 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: ) assert outcome(result) == {"result": [["A", "B"], 2]} - # Item a and item b check the state before delivery. The job after the - # map checks it after the 502, and once more after the resume. - assert len(client.requests("get_microvm")) == 4 + # Item a runs before the flag is set, so it checks nothing. Item b checks + # the state before delivery. The job after the map checks it after the + # 502, and once more after the resume. + assert len(client.requests("get_microvm")) == 3 assert client.requests("resume_microvm") == [{"microvmIdentifier": "mvm-1"}] assert [request["body"]["input"] for request in endpoint.requests] == [ "a", @@ -1961,11 +2133,12 @@ def test_a_flagged_session_keeps_the_strict_state_check( run: Any, fake_client: Any, fake_endpoint: Any, make_error: Any ) -> None: # After a suspend_after job, the state check before delivery fails the - # job for a denied GetMicrovm, as in a session with auto-suspend. + # job for a denied GetMicrovm, as in a session with auto-suspend. The + # suspend_after job itself makes no state check. endpoint = fake_endpoint() client = fake_client( endpoint=endpoint.url, - states=["RUNNING", make_error("AccessDeniedException", 403)], + states=[make_error("AccessDeniedException", 403)], ) def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: From cf7eca52d2a88a94a8d4a13c3165c80b7c337550 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Tue, 3 Nov 2026 23:42:31 +0000 Subject: [PATCH 4/9] test(extras): pin suspend_after flag and timeout A failed job without suspend_after must not set the flag. The test now runs a second job and asserts it makes no GetMicrovm. A suspend_after job that gets no result raises MicrovmTimeoutError and sends exactly one SuspendMicrovm. This is the README promise. Also note in session.py that a handler can catch the error that ends the invocation. --- .../microvm/session.py | 3 +- .../tests/microvm/session_test.py | 37 +++++++++++++++++-- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py index bd7f51028..9d4925ec6 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/session.py @@ -360,7 +360,8 @@ def run(job_context: DurableContext) -> Any: except Exception as error: # SuspendExecution is a BaseException, so a wait for the job # does not set the flag. Only a failed job sets it: any other - # error ends the invocation, and the flag lives in memory only. + # error ends the invocation unless the handler catches it, and + # the flag lives in memory only. if isinstance(error, ChildContextError): if suspend_after: self._may_be_suspended = True diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 5b6715fcd..fc5a8170a 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -49,6 +49,7 @@ MicrovmNotRunningError, MicrovmSession, MicrovmSessionConfig, + MicrovmTimeoutError, microvm_session, operation, ) @@ -1703,17 +1704,20 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: try: vm.invoke(ctx, "build", job_input=1, options=job()) except MicrovmJobFailedError as error: - return str(error) - return None + message = str(error) + return [message, vm.invoke(ctx, "test", job_input=2, options=job())] result = run( session_handler(session_config(client), body), None, - [("build", fail_with("BuildError", "no compiler"))], + [("build", fail_with("BuildError", "no compiler")), ("test", succeed_with(2))], ) - assert "no compiler" in outcome(result)["result"] + assert "no compiler" in outcome(result)["result"][0] assert client.requests("suspend_microvm") == [] + # The failed job did not ask for a suspend, so it set no flag. The + # second job then makes no state check. + assert client.requests("get_microvm") == [] def test_does_not_suspend_when_the_wait_of_a_suspend_after_job_raises_an_sdk_error( @@ -1756,6 +1760,31 @@ def result_once(self: Any) -> Any: assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] +def test_suspends_and_fails_with_a_timeout_when_a_suspend_after_job_gets_no_result( + run: Any, fake_client: Any, fake_endpoint: Any +) -> None: + client = fake_client(endpoint=fake_endpoint().url) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + try: + vm.invoke( + ctx, + "build", + job_input=1, + options=MicrovmInvokeOptions( + timeout=Duration.from_seconds(1), suspend_after=True + ), + ) + except MicrovmTimeoutError as error: + return type(error).__name__ + return None + + result = run(session_handler(session_config(client), body), None, []) + + assert outcome(result) == {"result": "MicrovmTimeoutError"} + assert client.requests("suspend_microvm") == [{"microvmIdentifier": "mvm-1"}] + + def test_sends_no_second_suspend_when_a_completed_failed_job_replays( run: Any, fake_client: Any, fake_endpoint: Any ) -> None: From f4d9fb8dff3634caaf12f192d7ec52441ea376dd Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 4 Nov 2026 17:43:13 +0000 Subject: [PATCH 5/9] fix(extras): send no suspend after the reserve The session stops waiting for SuspendMicrovm at the invocation reserve, and the call goes on in its thread. Lambda freezes that thread after the invocation returns. A later invocation can then find the MicroVM RUNNING and deliver the next job. When the old thread resumes, a first attempt or a botocore retry would suspend the MicroVM with that job inside. So a before-send handler, as in the worker's self_suspend.py, stops every attempt after the first one and any attempt after the reserve. It acts only on the call's own thread, so other calls of a shared client go on. The handler is removed when the call ends. An attempt that was sent before the reserve can still reach the service after it. The docstring states this limit. The failed-suspend warning now says that the MicroVM may still be running, in the code, the README, and the config docstring. --- .../README.md | 2 +- .../microvm/config.py | 4 +- .../microvm/operation.py | 95 ++++++- .../tests/microvm/session_test.py | 251 +++++++++++++++++- 4 files changed, 338 insertions(+), 14 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/README.md b/packages/aws-durable-execution-sdk-python-extras/README.md index 3e5017759..a8514af41 100644 --- a/packages/aws-durable-execution-sdk-python-extras/README.md +++ b/packages/aws-durable-execution-sdk-python-extras/README.md @@ -339,7 +339,7 @@ It needs: The suspend runs after a job timeout or a heartbeat timeout too, so a job that still runs in the MicroVM is frozen, not stopped. -The call ends at the invocation reserve, like the other calls of the session. A failed suspend, or a call that does not end in time, is not retried. The MicroVM then keeps running until the next job, the idle suspend, or the end of the session. The warning names `lambda:SuspendMicrovm` for a missing permission. +The call ends at the invocation reserve, like the other calls of the session. A failed suspend, or a call that does not end in time, is not retried. The MicroVM may then still be running until the next job, the idle suspend, or the end of the session. The warning names `lambda:SuspendMicrovm` for a missing permission. ### Session permissions diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py index d67708d52..bf40dbe5d 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/config.py @@ -391,8 +391,8 @@ class MicrovmInvokeOptions(MicrovmRequestConfig): replay. Change it only through a new function version. A failed ``SuspendMicrovm`` call logs a warning and does not fail the - job. The MicroVM then keeps running until the next job or the end of - the session. The call ends at the invocation reserve, like the other + job. The MicroVM may then still be running until the next job or the + end of the session. The call ends at the invocation reserve, like the other calls of the session, and that counts as a failed call. Set it on the job before a long wait, such as a durable wait or a diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py index a5bd05b9e..456b377a6 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py @@ -10,6 +10,7 @@ import re import threading import time +import uuid from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -115,6 +116,20 @@ so an attempt that ran out of time is repeated, not doubled. """ +_SUSPEND_BEFORE_SEND = "before-send.lambda-microvms.SuspendMicrovm" +"""botocore emits this event before each attempt of a ``SuspendMicrovm`` +call.""" + + +class _SuspendStoppedError(Exception): + """The session's ``SuspendMicrovm`` call stopped an attempt before it was + sent. + + botocore retries only its own connection and HTTP client errors. So it + does not retry this one. + """ + + RUN_HOOK_DELIVERY = "run-hook" """The delivery in which the job goes in the ``runHookPayload`` of the launch.""" @@ -759,19 +774,83 @@ def suspend(self, child: DurableContext, microvm_id: str) -> None: fail the job. The next job of the session checks the MicroVM state and resumes it when needed, and suspending a suspended MicroVM succeeds. + + The call makes one attempt, and it sends no attempt after the + invocation reserve: + + 1. The wait for the call ends at the reserve, and the call goes on + in its thread. Lambda freezes that thread after the invocation + returns. + 2. A later invocation can find the MicroVM ``RUNNING`` and deliver + the next job. + 3. The old thread then resumes. An attempt that it sent then, or a + botocore retry of an attempt that lost its answer, would suspend + the MicroVM with the next job inside. + + So a ``before-send`` handler on the client stops each attempt after + the first one, and any attempt after the reserve. botocore does not + retry that error. The handler also covers a caller's client that + retries. It acts only on attempts of the thread that runs this call, + so other calls of a shared client are not affected. The handler is + removed when the call ends. A client without ``meta.events``, such + as a test double, gets no handler. + + botocore cannot cancel an attempt in flight. So an attempt sent + before the reserve can still reach the service after it, and after + the next job started. This method cannot prevent that. Such an + attempt can then freeze the next job, which waits until a later job + resumes the MicroVM or its heartbeat timeout ends it. The client's + connect and read timeouts bound how long the attempt can take: 5 + and 30 seconds for the default client. """ + deadline = reserve_starts_at(_remaining_seconds(child)) + abandoned = threading.Event() def expired() -> Exception: + abandoned.set() return TimeoutError( "SuspendMicrovm did not end before the invocation reserve" ) - try: - call_before_deadline( - lambda: self._client().suspend_microvm(microvmIdentifier=microvm_id), - reserve_starts_at(_remaining_seconds(child)), - expired, + def call() -> None: + client = self._client() + caller = threading.get_ident() + attempts = 0 + + def stop_attempt(**_: object) -> None: + nonlocal attempts + if threading.get_ident() != caller: + return + attempts += 1 + if attempts > 1: + msg = ( + "SuspendMicrovm makes one attempt. A retry could " + "suspend the MicroVM after the next job started." + ) + raise _SuspendStoppedError(msg) + if abandoned.is_set() or ( + deadline is not None and time.monotonic() >= deadline + ): + msg = "SuspendMicrovm did not start before the invocation reserve" + raise _SuspendStoppedError(msg) + + events = getattr(getattr(client, "meta", None), "events", None) + if events is None: + client.suspend_microvm(microvmIdentifier=microvm_id) + return + # A fresh ID per call. botocore ignores a second registration + # with an ID it already has. + unique_id = f"aws-durable-execution-microvm-suspend-{uuid.uuid4().hex}" + events.register_first( + _SUSPEND_BEFORE_SEND, stop_attempt, unique_id=unique_id ) + try: + client.suspend_microvm(microvmIdentifier=microvm_id) + finally: + events.unregister(_SUSPEND_BEFORE_SEND, unique_id=unique_id) + + try: + call_before_deadline(call, deadline, expired) except Exception as error: # noqa: BLE001 detail = ( str(add_request_id("SuspendMicrovm", error)) @@ -780,9 +859,9 @@ def expired() -> Exception: ) child.logger.warning( f'MicroVM "{self.name}": SuspendMicrovm failed for {microvm_id}. ' - "The MicroVM keeps running and is billed until the idle suspend " - "or the end of the session. The next job checks its state. A " - "missing permission needs lambda:SuspendMicrovm in the " + "The MicroVM may still be running and billed until the idle " + "suspend or the end of the session. The next job checks its " + "state. A missing permission needs lambda:SuspendMicrovm in the " f"function's execution role. Error: {detail}", extra={"microvmId": microvm_id, "error": detail}, ) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index fc5a8170a..4b339006c 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -37,7 +37,10 @@ DurableFunctionTestResult, DurableFunctionTestRunner, ) -from botocore.exceptions import ClientError +from botocore.awsrequest import AWSResponse, HTTPHeaders +from botocore.config import Config +from botocore.exceptions import ClientError, ConnectionClosedError +from botocore.retries.standard import ExponentialBackoff from botocore.stub import ANY, Stubber from aws_durable_execution_sdk_python_extras.microvm import ( @@ -1536,8 +1539,8 @@ def test_makes_no_state_check_for_a_launch_checkpoint_without_a_bool_resume_mode # --------------------------------------------------------------------------- SUSPEND_WARNING_START = ( - 'MicroVM "pipeline": SuspendMicrovm failed for mvm-1. The MicroVM keeps ' - "running and is billed until the idle suspend or the end of the session. " + 'MicroVM "pipeline": SuspendMicrovm failed for mvm-1. The MicroVM may still ' + "be running and billed until the idle suspend or the end of the session. " "The next job checks its state. A missing permission needs " "lambda:SuspendMicrovm in the function's execution role. Error: " ) @@ -1854,6 +1857,248 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: assert "did not end before the invocation reserve" in warning +class _Body: + def __init__(self, data: bytes) -> None: + self._data = data + + def stream(self, **_: Any) -> Any: + yield self._data + + +def _botocore_suspends( + fake_client: Any, monkeypatch: pytest.MonkeyPatch, **fields: Any +) -> tuple[Any, Any]: + """A fake client whose ``suspend_microvm`` is a real botocore call. + + The botocore client has the default client's three attempts and no + retry delay. Returns the fake client and the botocore client. The fake + client's ``suspend_done`` is set when a call ends. + """ + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False) + monkeypatch.setattr(ExponentialBackoff, "delay_amount", lambda self, context: 0) + botocore_client = boto3.session.Session().client( + "lambda-microvms", + region_name="us-west-2", + config=Config(retries={"mode": "standard", "total_max_attempts": 3}), + ) + + class Client(fake_client): # type: ignore[misc, valid-type] + meta = botocore_client.meta + suspend_done = threading.Event() + + def suspend_microvm(self, **request: Any) -> Any: + try: + return botocore_client.suspend_microvm(**request) + finally: + self.suspend_done.set() + + return Client(**fields), botocore_client + + +def _ok_answer() -> AWSResponse: + return AWSResponse( + "https://lambda.us-west-2.amazonaws.com", 200, HTTPHeaders(), _Body(b"{}") + ) + + +def _wait_until(condition: Callable[[], bool]) -> None: + deadline = time.monotonic() + WAIT + while not condition(): + assert time.monotonic() < deadline + time.sleep(0.01) + + +@pytest.mark.parametrize( + "answer_lost", [False, True], ids=["held-before-send", "answer-lost"] +) +def test_sends_no_suspend_attempt_after_the_reserve_when_the_next_job_runs( + run: Any, + fake_client: Any, + fake_endpoint: Any, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, + answer_lost: bool, +) -> None: + # 1. The suspend call is still running at the invocation reserve. The + # session stops waiting for it, and the call goes on in its thread. + # In Lambda, that thread freezes after the invocation returns. + # 2. The next job finds the MicroVM RUNNING and is delivered. + # 3. The old call then goes on. Held before its first attempt, it would + # send the attempt now. After a first attempt whose answer was lost, + # botocore would retry it now. Either attempt would suspend the + # MicroVM with the next job inside. + # So no attempt is sent after the reserve. + monkeypatch.setattr( + operation, + "_remaining_seconds", + lambda context: lambda: INVOCATION_RESERVE_SECONDS + 1.5, + ) + endpoint = fake_endpoint() + client, botocore_client = _botocore_suspends( + fake_client, monkeypatch, endpoint=endpoint.url + ) + release = threading.Event() + sent: list[float] = [] + + def hold_before_send(**_: Any) -> None: + if not answer_lost: + release.wait(WAIT) + + def service(**_: Any) -> AWSResponse: + sent.append(time.monotonic()) + if answer_lost and len(sent) == 1: + release.wait(WAIT) + raise ConnectionClosedError(endpoint_url="https://lambda.example") + return _ok_answer() + + events = botocore_client.meta.events + events.register("before-sign.lambda-microvms.SuspendMicrovm", hold_before_send) + events.register("before-send.lambda-microvms.SuspendMicrovm", service) + reserve_passed: list[float] = [] + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + build = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + reserve_passed.append(time.monotonic()) + return [build, vm.invoke(ctx, "test", job_input=2, options=job())] + + def release_the_old_call( + runner: DurableFunctionTestRunner, callback_id: str + ) -> None: + # The next job is in the MicroVM. Now the old call goes on. + _wait_until(lambda: len(endpoint.requests) == 2) + release.set() + assert client.suspend_done.wait(WAIT) + succeed_with(2)(runner, callback_id) + + try: + with caplog.at_level(logging.WARNING): + result = run( + session_handler(session_config(client), body), + None, + [("build", succeed_with(1)), ("test", release_the_old_call)], + ) + finally: + release.set() + botocore_client.close() + + assert outcome(result) == {"result": [1, 2]} + # Only an attempt sent before the reserve reached the service, and + # botocore made no retry of it. + assert len(sent) == (1 if answer_lost else 0) + assert all(at < reserve_passed[0] for at in sent) + [warning] = suspend_warnings(caplog) + assert "did not end before the invocation reserve" in warning + + +def _suspend_handlers(botocore_client: Any) -> list[str]: + handlers = botocore_client.meta.events._emitter._unique_id_handlers # noqa: SLF001 + return [key for key in handlers if "microvm-suspend" in key] + + +def test_makes_one_suspend_attempt_before_the_reserve_too( + run: Any, + fake_client: Any, + fake_endpoint: Any, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The first attempt can suspend the MicroVM and lose its answer. A retry + # can then reach the service after the next job started. So the call + # makes no retry, also while time remains before the reserve. + client, botocore_client = _botocore_suspends( + fake_client, monkeypatch, endpoint=fake_endpoint().url + ) + sent: list[int] = [] + + def lose_the_answer(**_: Any) -> None: + sent.append(1) + raise ConnectionClosedError(endpoint_url="https://lambda.example") + + botocore_client.meta.events.register( + "before-send.lambda-microvms.SuspendMicrovm", lose_the_answer + ) + + try: + with caplog.at_level(logging.WARNING): + result = run( + session_handler( + session_config(client), + lambda vm, ctx, event: vm.invoke( + ctx, "build", job_input=1, options=job(suspend_after=True) + ), + ), + None, + [("build", succeed_with(1))], + ) + # The handler is removed when the call ends, also when it raises. + assert _suspend_handlers(botocore_client) == [] + finally: + botocore_client.close() + + assert outcome(result) == {"result": 1} + assert sent == [1] + [warning] = suspend_warnings(caplog) + assert "SuspendMicrovm makes one attempt" in warning + + +def test_stops_only_the_attempts_of_its_own_call_on_a_shared_client( + run: Any, + fake_client: Any, + fake_endpoint: Any, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Another call of the same client, such as the suspend of another + # session, sends its attempt while the session's call runs. The session's + # handler leaves it alone, and the session's own first attempt is sent. + client, botocore_client = _botocore_suspends( + fake_client, monkeypatch, endpoint=fake_endpoint().url + ) + sent: list[str] = [] + other: list[threading.Thread] = [] + + def other_call_first(**_: Any) -> None: + if not other: + other.append( + threading.Thread( + target=lambda: botocore_client.suspend_microvm( + microvmIdentifier="mvm-other" + ) + ) + ) + other[0].start() + other[0].join(WAIT) + + def service(request: Any, **_: Any) -> AWSResponse: + sent.append(request.url.rsplit("/microvms/", 1)[1]) + return _ok_answer() + + events = botocore_client.meta.events + events.register("before-sign.lambda-microvms.SuspendMicrovm", other_call_first) + events.register("before-send.lambda-microvms.SuspendMicrovm", service) + + try: + with caplog.at_level(logging.WARNING): + result = run( + session_handler( + session_config(client), + lambda vm, ctx, event: vm.invoke( + ctx, "build", job_input=1, options=job(suspend_after=True) + ), + ), + None, + [("build", succeed_with(1))], + ) + finally: + botocore_client.close() + + assert outcome(result) == {"result": 1} + assert sent == ["mvm-other/suspend", "mvm-1/suspend"] + assert suspend_warnings(caplog) == [] + + def test_checks_the_state_before_the_job_after_a_failed_suspend_after_job( run: Any, fake_client: Any, fake_endpoint: Any ) -> None: From eeab99467883be632e67edad8b85f6956c9bfc44 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 4 Nov 2026 17:43:16 +0000 Subject: [PATCH 6/9] test(worker): drop suspendAfter compatibility The package is not released yet, so no older durable function sends suspendAfter. Remove the README text about such a request, and the three tests that sent the field to the worker. The extras test that checks that the SDK sends no suspendAfter field stays. --- .../README.md | 4 +- .../tests/lifecycle_hooks_test.py | 48 ------------------- .../tests/payload_test.py | 8 ---- 3 files changed, 2 insertions(+), 58 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/README.md b/packages/aws-durable-execution-sdk-python-microvm-worker/README.md index cfa5bd992..57033023e 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/README.md +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/README.md @@ -126,7 +126,7 @@ The worker creates a `lambda-microvms` client for each suspend, and closes it af ## Suspending after a job -The durable function suspends a session's MicroVM after a job. `vm.invoke` with `suspend_after=True` calls `SuspendMicrovm` itself, from the durable function, after the job's result arrives. The job request carries no field for it, and the worker does nothing for it. A request from an older durable function can still carry `"suspendAfter": true`. The worker ignores the field, sends no `X-Microvm-Suspend-After` header, and does not suspend. +The durable function suspends a session's MicroVM after a job. `vm.invoke` with `suspend_after=True` calls `SuspendMicrovm` itself, from the durable function, after the job's result arrives. The job request carries no field for it, and the worker does nothing for it. So the MicroVM's execution role needs no `lambda:SuspendMicrovm` for this case. The durable function's role needs it. See the [extras README](../aws-durable-execution-sdk-python-extras/README.md#suspending-after-a-job). @@ -181,7 +181,7 @@ The `runHookPayload` string holds: } ``` -`job` is absent when the job arrives over HTTP. An HTTP job request body has the job fields at the top level, with `version`, `region`, and an optional `microvmId`. `MicrovmJobRequest` holds those job fields in its `job` attribute. The worker ignores a `suspendAfter` field. +`job` is absent when the job arrives over HTTP. An HTTP job request body has the job fields at the top level, with `version`, `region`, and an optional `microvmId`. `MicrovmJobRequest` holds those job fields in its `job` attribute. A document that does not match raises `InvalidRunHookPayloadError`. When the document named a non-empty callback ID, the error carries `callback_id` and `region`, so the worker can fail that callback at once. diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py index b66b3ebf1..d6b95d0c2 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/lifecycle_hooks_test.py @@ -3489,51 +3489,3 @@ def create(region: str) -> Any: assert h.job("cb-1") == 503 h.hook("resume") assert h.job("cb-1") == 202 - - -# --------------------------------------------------------------------------- -# A job request no longer asks the worker to suspend -# --------------------------------------------------------------------------- - - -def test_ignores_a_suspend_after_field_of_a_job_request(start: Any) -> None: - # The durable function suspends the MicroVM after a session job. A - # request from an older durable function can still carry the field. - h = start() - h.run_hook() - - assert h.job("cb-1", suspendAfter=True) == 202 - h.finish_job("cb-1") - h.settle() - - assert h.microvms.suspended == [] - assert h.job("cb-2", suspendAfter=True) == 202 - - -def test_answers_a_job_request_with_no_suspend_after_header(start: Any) -> None: - h = start() - h.run_hook() - connection = http.client.HTTPConnection( - host_of(h.worker), h.worker.port, timeout=WAIT - ) - body = { - "version": 1, - "region": "us-east-1", - "callbackId": "cb-1", - "input": {}, - "suspendAfter": True, - } - try: - connection.request( - "POST", - "/job", - body=json.dumps(body).encode(), - headers={"Content-Type": "application/json"}, - ) - response = connection.getresponse() - response.read() - finally: - connection.close() - - assert response.status == 202 - assert response.getheader("X-Microvm-Suspend-After") is None diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py index 7a55a68e0..f60f07c15 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py @@ -286,14 +286,6 @@ def test_an_unusable_callback_id_is_not_attached_to_errors(callback_id): # endregion dataclasses and callback IDs -def test_job_request_ignores_a_suspend_after_field(): - # The durable function suspends the MicroVM itself. An older durable - # function can still send the field, and the worker ignores it. - parsed = MicrovmJobRequest.from_dict(job_request(suspendAfter=True)) - - assert not hasattr(parsed, "suspend_after") - - def test_run_hook_payload_ignores_suspend_after(): parsed = RunHookRequest.from_dict( run_hook({"version": 1, "region": "us-east-1", "suspendAfter": True}) From 3e535d6180c483c02dbcd10757dcc9a55c50943c Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 4 Nov 2026 19:03:56 +0000 Subject: [PATCH 7/9] fix(extras): stop suspend only after wait gives up The before-send handler stopped an attempt when the wait had given up or when the clock had passed the reserve. The wait gives up only after the clock passes the reserve, and before the invocation returns. So each half alone stopped the same attempts, and a test could not tell them apart. The handler now checks only the flag that the wait sets when it gives up. Without a deadline the call runs inline, so no deadline guard is needed. A new test holds an old suspend call past the reserve on one shared client. The next job also has suspend_after, and its call loses its answer. The next call sends exactly one attempt, and the old handler does not stop it. The test fails with a fixed handler ID and without the thread check. The docstring now gives the reason for the thread check: the default client is cached per Lambda environment, so an abandoned call's handler stays on it. It describes the one-attempt limit as narrowing the window, and drops the claim that the timeouts bound the window. --- .../microvm/operation.py | 41 ++++++----- .../tests/microvm/session_test.py | 71 ++++++++++++++++++- 2 files changed, 91 insertions(+), 21 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py index 456b377a6..22695136d 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py @@ -787,21 +787,28 @@ def suspend(self, child: DurableContext, microvm_id: str) -> None: botocore retry of an attempt that lost its answer, would suspend the MicroVM with the next job inside. - So a ``before-send`` handler on the client stops each attempt after - the first one, and any attempt after the reserve. botocore does not - retry that error. The handler also covers a caller's client that - retries. It acts only on attempts of the thread that runs this call, - so other calls of a shared client are not affected. The handler is - removed when the call ends. A client without ``meta.events``, such - as a test double, gets no handler. - - botocore cannot cancel an attempt in flight. So an attempt sent - before the reserve can still reach the service after it, and after - the next job started. This method cannot prevent that. Such an - attempt can then freeze the next job, which waits until a later job - resumes the MicroVM or its heartbeat timeout ends it. The client's - connect and read timeouts bound how long the attempt can take: 5 - and 30 seconds for the default client. + So a ``before-send`` handler on the client stops any attempt after + the wait gave up at the reserve. botocore does not retry that error. + The handler also stops each attempt after the first one. An attempt + sent before the reserve can still be in flight at the reserve, and + a retry adds one more such attempt. So one attempt narrows that + window, as in the worker. The handler also covers a caller's client + that retries. + + The handler acts only on attempts of the thread that runs this call. + The default client is cached per Lambda environment. An abandoned + call's handler stays registered on it until the call's thread ends. + Without the thread check, that handler would stop the call of a + later invocation. The check also leaves other calls of a shared + client alone. The handler is removed when the call ends. A client + without ``meta.events``, such as a test double, gets no handler. + + botocore cannot cancel an attempt in flight. An attempt that passed + the check before the reserve can still reach the service after it. + It can reach the service after the next job started. This method + cannot prevent that. Such an attempt can then freeze the next job, + which waits until a later job resumes the MicroVM or its heartbeat + timeout ends it. """ deadline = reserve_starts_at(_remaining_seconds(child)) abandoned = threading.Event() @@ -828,9 +835,7 @@ def stop_attempt(**_: object) -> None: "suspend the MicroVM after the next job started." ) raise _SuspendStoppedError(msg) - if abandoned.is_set() or ( - deadline is not None and time.monotonic() >= deadline - ): + if abandoned.is_set(): msg = "SuspendMicrovm did not start before the invocation reserve" raise _SuspendStoppedError(msg) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 4b339006c..4bebba796 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -2004,9 +2004,11 @@ def test_makes_one_suspend_attempt_before_the_reserve_too( caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, ) -> None: - # The first attempt can suspend the MicroVM and lose its answer. A retry - # can then reach the service after the next job started. So the call - # makes no retry, also while time remains before the reserve. + # Any attempt sent before the reserve can still be in flight at the + # reserve, and reach the service after the next job started. A retry + # adds one more such attempt. So the call makes no retry, also while + # time remains before the reserve. This narrows the window. It does not + # close it. client, botocore_client = _botocore_suspends( fake_client, monkeypatch, endpoint=fake_endpoint().url ) @@ -2043,6 +2045,69 @@ def lose_the_answer(**_: Any) -> None: assert "SuspendMicrovm makes one attempt" in warning +def test_an_abandoned_suspend_does_not_stop_the_next_suspend_on_the_client( + run: Any, + fake_client: Any, + fake_endpoint: Any, + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # The default client is cached per Lambda environment. The old suspend + # call is held past the reserve, so its handler stays on the client. + # The next job also has suspend_after, and its call loses its answer. + # The next call makes exactly its one attempt. The old handler does not + # stop it, and its own handler stops the retry. + monkeypatch.setattr( + operation, + "_remaining_seconds", + lambda context: lambda: INVOCATION_RESERVE_SECONDS + 1.5, + ) + client, botocore_client = _botocore_suspends( + fake_client, monkeypatch, endpoint=fake_endpoint().url + ) + release = threading.Event() + held: list[int] = [] + sent: list[int] = [] + + def hold_the_first_call(**_: Any) -> None: + if not held: + held.append(threading.get_ident()) + release.wait(WAIT) + + def lose_the_answer(**_: Any) -> None: + sent.append(threading.get_ident()) + raise ConnectionClosedError(endpoint_url="https://lambda.example") + + events = botocore_client.meta.events + events.register("before-sign.lambda-microvms.SuspendMicrovm", hold_the_first_call) + events.register("before-send.lambda-microvms.SuspendMicrovm", lose_the_answer) + + def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: + build = vm.invoke(ctx, "build", job_input=1, options=job(suspend_after=True)) + test = vm.invoke(ctx, "test", job_input=2, options=job(suspend_after=True)) + return [build, test] + + try: + with caplog.at_level(logging.WARNING): + result = run( + session_handler(session_config(client), body), + None, + [("build", succeed_with(1)), ("test", succeed_with(2))], + ) + assert len(_suspend_handlers(botocore_client)) == 1 + finally: + release.set() + _wait_until(lambda: _suspend_handlers(botocore_client) == []) + botocore_client.close() + + assert outcome(result) == {"result": [1, 2]} + assert len(sent) == 1 + assert sent[0] != held[0] + old, new = suspend_warnings(caplog) + assert "did not end before the invocation reserve" in old + assert "SuspendMicrovm makes one attempt" in new + + def test_stops_only_the_attempts_of_its_own_call_on_a_shared_client( run: Any, fake_client: Any, From 4e3d9127713f60456c7cb3045284888659953c14 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Wed, 4 Nov 2026 19:03:56 +0000 Subject: [PATCH 8/9] test(worker): drop suspendAfter payload test No code reads the suspendAfter field of a run-hook payload. So the test that checked that the worker ignores it has nothing to check. --- .../tests/payload_test.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py index f60f07c15..2192fd220 100644 --- a/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py +++ b/packages/aws-durable-execution-sdk-python-microvm-worker/tests/payload_test.py @@ -284,12 +284,3 @@ def test_an_unusable_callback_id_is_not_attached_to_errors(callback_id): # endregion dataclasses and callback IDs - - -def test_run_hook_payload_ignores_suspend_after(): - parsed = RunHookRequest.from_dict( - run_hook({"version": 1, "region": "us-east-1", "suspendAfter": True}) - ) - - assert parsed.payload is not None - assert not hasattr(parsed.payload, "suspend_after") From 61218a0cabb62fda6edc0772e2383f2095125a94 Mon Sep 17 00:00:00 2001 From: Pooya Paridel Date: Thu, 5 Nov 2026 06:01:13 +0000 Subject: [PATCH 9/9] fix(extras): hook suspend cutoff once per client Each SuspendMicrovm call added and removed its own before-send handler on the client. botocore's emitter stores the handler list of an event without a lock. So a registration while another thread emitted the event could leave a stored list without the new handler. Two sessions of a map share the default client. A suspend could then retry, or send after the invocation reserve. Now each client gets one handler, registered under a lock before its first suspend, and keeps it. Each call keeps its attempt count and its abandoned flag in a thread-local state. The handler does nothing on a thread without that state. This removes the per-call register and unregister, the unique ID, and the thread ID check. A new test holds one suspend inside botocore's emit and starts a second suspend. The second one makes one attempt, and no attempt after its wait gave up. --- .../microvm/operation.py | 111 ++++--- .../tests/microvm/session_test.py | 276 ++++++++++++++++-- 2 files changed, 330 insertions(+), 57 deletions(-) diff --git a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py index 22695136d..d5c3e51dc 100644 --- a/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py +++ b/packages/aws-durable-execution-sdk-python-extras/src/aws_durable_execution_sdk_python_extras/microvm/operation.py @@ -10,7 +10,7 @@ import re import threading import time -import uuid +import weakref from collections.abc import Callable, Mapping from dataclasses import dataclass from typing import TYPE_CHECKING, Any, cast @@ -130,6 +130,59 @@ class _SuspendStoppedError(Exception): """ +@dataclass +class _SuspendCall: + """The state of one session ``SuspendMicrovm`` call.""" + + attempts: int = 0 + abandoned: bool = False + """The wait for the call gave up at the invocation reserve.""" + + +_SUSPEND_CALL = threading.local() +"""The ``_SuspendCall`` of the call that runs on this thread, if any.""" + +_HOOKED_EVENTS: weakref.WeakSet[Any] = weakref.WeakSet() +"""The ``meta.events`` of each client that has ``_stop_suspend_attempt``. + +A weak set, so a client that the caller drops is not kept alive.""" + +_HOOK_LOCK = threading.Lock() + + +def _stop_suspend_attempt(**_: object) -> None: + """Stop an attempt of the session ``SuspendMicrovm`` call on this + thread: each attempt after the first one, and any attempt after the wait + for the call gave up. + + It does nothing on a thread without a session call. + """ + call: _SuspendCall | None = getattr(_SUSPEND_CALL, "call", None) + if call is None: + return + call.attempts += 1 + if call.attempts > 1: + msg = ( + "SuspendMicrovm makes one attempt. A retry could suspend the " + "MicroVM after the next job started." + ) + raise _SuspendStoppedError(msg) + if call.abandoned: + msg = "SuspendMicrovm did not start before the invocation reserve" + raise _SuspendStoppedError(msg) + + +def _hook_suspend(events: Any) -> None: + """Register ``_stop_suspend_attempt`` on a client's events, once. + + The lock makes two first suspends on one client register it once. + """ + with _HOOK_LOCK: + if events not in _HOOKED_EVENTS: + events.register_first(_SUSPEND_BEFORE_SEND, _stop_suspend_attempt) + _HOOKED_EVENTS.add(events) + + RUN_HOOK_DELIVERY = "run-hook" """The delivery in which the job goes in the ``runHookPayload`` of the launch.""" @@ -795,13 +848,19 @@ def suspend(self, child: DurableContext, microvm_id: str) -> None: window, as in the worker. The handler also covers a caller's client that retries. - The handler acts only on attempts of the thread that runs this call. - The default client is cached per Lambda environment. An abandoned - call's handler stays registered on it until the call's thread ends. - Without the thread check, that handler would stop the call of a - later invocation. The check also leaves other calls of a shared - client alone. The handler is removed when the call ends. A client - without ``meta.events``, such as a test double, gets no handler. + Each client gets the handler once, before its first suspend, and + keeps it. botocore's emitter stores the handler list of an event + without a lock. A handler that is added or removed while another + thread emits the event can be missing from the stored list. Two + sessions of a ``map`` share the default client. So no suspend + changes the handlers of a client that already has the handler. + + Each call keeps its attempt count and its abandoned flag in a + thread-local state, on the thread that runs the call. The handler + acts only on a thread with such a state. So it leaves the other + calls of a shared client alone, and an abandoned call does not stop + a later call. A client without ``meta.events``, such as a test + double, gets no handler. botocore cannot cancel an attempt in flight. An attempt that passed the check before the reserve can still reach the service after it. @@ -811,48 +870,24 @@ def suspend(self, child: DurableContext, microvm_id: str) -> None: timeout ends it. """ deadline = reserve_starts_at(_remaining_seconds(child)) - abandoned = threading.Event() + state = _SuspendCall() def expired() -> Exception: - abandoned.set() + state.abandoned = True return TimeoutError( "SuspendMicrovm did not end before the invocation reserve" ) def call() -> None: client = self._client() - caller = threading.get_ident() - attempts = 0 - - def stop_attempt(**_: object) -> None: - nonlocal attempts - if threading.get_ident() != caller: - return - attempts += 1 - if attempts > 1: - msg = ( - "SuspendMicrovm makes one attempt. A retry could " - "suspend the MicroVM after the next job started." - ) - raise _SuspendStoppedError(msg) - if abandoned.is_set(): - msg = "SuspendMicrovm did not start before the invocation reserve" - raise _SuspendStoppedError(msg) - events = getattr(getattr(client, "meta", None), "events", None) - if events is None: - client.suspend_microvm(microvmIdentifier=microvm_id) - return - # A fresh ID per call. botocore ignores a second registration - # with an ID it already has. - unique_id = f"aws-durable-execution-microvm-suspend-{uuid.uuid4().hex}" - events.register_first( - _SUSPEND_BEFORE_SEND, stop_attempt, unique_id=unique_id - ) + if events is not None: + _hook_suspend(events) + _SUSPEND_CALL.call = state try: client.suspend_microvm(microvmIdentifier=microvm_id) finally: - events.unregister(_SUSPEND_BEFORE_SEND, unique_id=unique_id) + _SUSPEND_CALL.call = None try: call_before_deadline(call, deadline, expired) diff --git a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py index 4bebba796..edd403edf 100644 --- a/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py +++ b/packages/aws-durable-execution-sdk-python-extras/tests/microvm/session_test.py @@ -1865,6 +1865,20 @@ def stream(self, **_: Any) -> Any: yield self._data +def _botocore_client(monkeypatch: pytest.MonkeyPatch) -> Any: + """A real botocore client with the default client's three attempts and + no retry delay.""" + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") + monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False) + monkeypatch.setattr(ExponentialBackoff, "delay_amount", lambda self, context: 0) + return boto3.session.Session().client( + "lambda-microvms", + region_name="us-west-2", + config=Config(retries={"mode": "standard", "total_max_attempts": 3}), + ) + + def _botocore_suspends( fake_client: Any, monkeypatch: pytest.MonkeyPatch, **fields: Any ) -> tuple[Any, Any]: @@ -1874,24 +1888,18 @@ def _botocore_suspends( retry delay. Returns the fake client and the botocore client. The fake client's ``suspend_done`` is set when a call ends. """ - monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test-key") - monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test-secret") - monkeypatch.delenv("AWS_SESSION_TOKEN", raising=False) - monkeypatch.setattr(ExponentialBackoff, "delay_amount", lambda self, context: 0) - botocore_client = boto3.session.Session().client( - "lambda-microvms", - region_name="us-west-2", - config=Config(retries={"mode": "standard", "total_max_attempts": 3}), - ) + botocore_client = _botocore_client(monkeypatch) class Client(fake_client): # type: ignore[misc, valid-type] meta = botocore_client.meta suspend_done = threading.Event() + suspends_ended = 0 def suspend_microvm(self, **request: Any) -> Any: try: return botocore_client.suspend_microvm(**request) finally: + type(self).suspends_ended += 1 self.suspend_done.set() return Client(**fields), botocore_client @@ -1992,9 +2000,11 @@ def release_the_old_call( assert "did not end before the invocation reserve" in warning -def _suspend_handlers(botocore_client: Any) -> list[str]: - handlers = botocore_client.meta.events._emitter._unique_id_handlers # noqa: SLF001 - return [key for key in handlers if "microvm-suspend" in key] +def _suspend_hooks(botocore_client: Any) -> int: + """Count the session's suspend handlers on the client.""" + emitter = botocore_client.meta.events._emitter # noqa: SLF001 + handlers = emitter._handlers.prefix_search(operation._SUSPEND_BEFORE_SEND) # noqa: SLF001 + return sum(handler is operation._stop_suspend_attempt for handler in handlers) # noqa: SLF001 def test_makes_one_suspend_attempt_before_the_reserve_too( @@ -2034,8 +2044,6 @@ def lose_the_answer(**_: Any) -> None: None, [("build", succeed_with(1))], ) - # The handler is removed when the call ends, also when it raises. - assert _suspend_handlers(botocore_client) == [] finally: botocore_client.close() @@ -2053,10 +2061,11 @@ def test_an_abandoned_suspend_does_not_stop_the_next_suspend_on_the_client( monkeypatch: pytest.MonkeyPatch, ) -> None: # The default client is cached per Lambda environment. The old suspend - # call is held past the reserve, so its handler stays on the client. + # call is held past the reserve, so it is abandoned and still runs. # The next job also has suspend_after, and its call loses its answer. - # The next call makes exactly its one attempt. The old handler does not - # stop it, and its own handler stops the retry. + # The next call makes exactly its one attempt. The old call's abandoned + # state does not stop it, and the handler stops its retry. The second + # suspend does not register the handler again. monkeypatch.setattr( operation, "_remaining_seconds", @@ -2094,10 +2103,10 @@ def body(vm: MicrovmSession, ctx: DurableContext, event: Any) -> Any: None, [("build", succeed_with(1)), ("test", succeed_with(2))], ) - assert len(_suspend_handlers(botocore_client)) == 1 + assert _suspend_hooks(botocore_client) == 1 finally: release.set() - _wait_until(lambda: _suspend_handlers(botocore_client) == []) + _wait_until(lambda: client.suspends_ended == 2) botocore_client.close() assert outcome(result) == {"result": [1, 2]} @@ -2164,6 +2173,235 @@ def service(request: Any, **_: Any) -> AWSResponse: assert suspend_warnings(caplog) == [] +def _scope(client: Any) -> operation._MicrovmScope: # noqa: SLF001 + return operation._MicrovmScope( # noqa: SLF001 + name="pipeline", + config=session_config(client), + partition="aws", + region="us-west-2", + timeout_seconds=3600, + retry_strategy=lambda error, attempt: RetryDecision.no_retry(), + ) + + +def _suspend(scope: Any, microvm_id: str, remaining: float | None) -> None: + """Run the session's suspend call, with ``remaining`` seconds left in + the invocation, or without a deadline.""" + lambda_context = ( + None + if remaining is None + else SimpleNamespace(get_remaining_time_in_millis=lambda: remaining * 1000) + ) + child = SimpleNamespace( + logger=logging.getLogger(__name__), lambda_context=lambda_context + ) + scope.suspend(child, microvm_id) + + +def _sent_to(sent: list[str]) -> Callable[..., AWSResponse]: + def service(request: Any, **_: Any) -> AWSResponse: + sent.append(request.url.rsplit("/microvms/", 1)[1]) + return _ok_answer() + + return service + + +class _EndedCalls: + """A client whose ``suspend_microvm`` is the botocore call. ``ended`` + is set for a MicroVM when its call ends.""" + + def __init__(self, botocore_client: Any) -> None: + self.meta = botocore_client.meta + self._client = botocore_client + self.ended = {"mvm-a": threading.Event(), "mvm-b": threading.Event()} + + def suspend_microvm(self, **request: Any) -> Any: + try: + return self._client.suspend_microvm(**request) + finally: + self.ended[request["microvmIdentifier"]].set() + + +@pytest.mark.parametrize( + "held", [False, True], ids=["answer-lost", "held-past-reserve"] +) +def test_keeps_the_cutoff_of_a_suspend_that_starts_while_another_emits( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch, held: bool +) -> None: + # botocore's emitter computes and stores the handler list of an event + # without a lock. + # 1. Suspend A computes the list of its before-send event. It stops + # before it stores the list. + # 2. Suspend B starts on the same client. + # 3. A stores its list. B's attempts then use the stored list. + # A change of the client's handlers in step 2 would leave B's attempts + # without the cutoff. Two sessions of a map share the default client. + # So B still makes one attempt, and no attempt after its wait gave up. + botocore_client = _botocore_client(monkeypatch) + client = _EndedCalls(botocore_client) + scope = _scope(client) + a_computed, a_go, a_sent, b_entered, b_gave_up = ( + threading.Event() for _ in range(5) + ) + b_sent: list[float] = [] + + def b_entered_botocore(params: Any, **_: Any) -> None: + if params["microvmIdentifier"] == "mvm-b": + b_entered.set() + + def hold_b(request: Any, **_: Any) -> None: + if "/mvm-b/" in request.url: + a_sent.wait(WAIT) + if held: + b_gave_up.wait(WAIT) + + def service(request: Any, **_: Any) -> AWSResponse: + if "/mvm-a/" in request.url: + # A stays in its call until B's call ended. + a_sent.set() + client.ended["mvm-b"].wait(WAIT) + return _ok_answer() + b_sent.append(time.monotonic()) + raise ConnectionClosedError(endpoint_url="https://lambda.example") + + events = botocore_client.meta.events + events.register( + "before-parameter-build.lambda-microvms.SuspendMicrovm", b_entered_botocore + ) + events.register("before-sign.lambda-microvms.SuspendMicrovm", hold_b) + events.register(operation._SUSPEND_BEFORE_SEND, service) # noqa: SLF001 + trie = events._emitter._handlers # noqa: SLF001 + search = trie.prefix_search + computed: list[Any] = [] + + def hold_the_first_list(key: str) -> Any: + handlers = search(key) + if key == operation._SUSPEND_BEFORE_SEND and not computed: # noqa: SLF001 + computed.append(handlers) + a_computed.set() + a_go.wait(WAIT) + return handlers + + monkeypatch.setattr(trie, "prefix_search", hold_the_first_list) + a = threading.Thread( + target=_suspend, args=(scope, "mvm-a", INVOCATION_RESERVE_SECONDS + WAIT) + ) + b = threading.Thread( + target=_suspend, + args=(scope, "mvm-b", INVOCATION_RESERVE_SECONDS + (1.5 if held else WAIT)), + ) + try: + with caplog.at_level(logging.WARNING): + a.start() + assert a_computed.wait(WAIT) + b.start() + assert b_entered.wait(WAIT) + a_go.set() + if held: + # B's wait gave up at the reserve. Its call goes on now. + b.join(WAIT) + b_gave_up.set() + assert client.ended["mvm-b"].wait(WAIT) + a.join(WAIT) + b.join(WAIT) + finally: + for event in (a_go, a_sent, b_gave_up, client.ended["mvm-b"]): + event.set() + a.join(WAIT) + botocore_client.close() + + assert len(b_sent) == (0 if held else 1) + [warning] = suspend_warnings(caplog) + assert "for mvm-b" in warning + assert ( + "did not end before the invocation reserve" + if held + else "SuspendMicrovm makes one attempt" + ) in warning + + +def test_two_first_suspends_on_a_client_register_the_handler_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Two sessions start their first suspend on a new client at the same + # time. The second one waits until the first one registered the + # handler, and does not register it again. A second handler would count + # each attempt twice and stop the first attempt. + botocore_client = _botocore_client(monkeypatch) + scope = _scope(botocore_client) + events = botocore_client.meta.events + sent: list[str] = [] + events.register(operation._SUSPEND_BEFORE_SEND, _sent_to(sent)) # noqa: SLF001 + registering, second, release = (threading.Event() for _ in range(3)) + registrations: list[Any] = [] + register_first = events.register_first + + def hold_the_first_registration(*args: Any, **kwargs: Any) -> None: + registrations.append(args) + if len(registrations) == 1: + registering.set() + release.wait(WAIT) + else: + second.set() + register_first(*args, **kwargs) + + class SignallingLock: + """A lock that signals when a second thread asks for it.""" + + def __init__(self) -> None: + self._lock = threading.Lock() + self._entries = 0 + + def __enter__(self) -> None: + self._entries += 1 + if self._entries == 2: + second.set() + self._lock.acquire() + + def __exit__(self, *_: object) -> None: + self._lock.release() + + monkeypatch.setattr(events, "register_first", hold_the_first_registration) + monkeypatch.setattr(operation, "_HOOK_LOCK", SignallingLock()) + a = threading.Thread(target=_suspend, args=(scope, "mvm-a", None)) + b = threading.Thread(target=_suspend, args=(scope, "mvm-b", None)) + try: + a.start() + assert registering.wait(WAIT) + b.start() + assert second.wait(WAIT) + release.set() + a.join(WAIT) + b.join(WAIT) + finally: + release.set() + botocore_client.close() + + assert len(registrations) == 1 + assert _suspend_hooks(botocore_client) == 1 + assert sorted(sent) == ["mvm-a/suspend", "mvm-b/suspend"] + + +def test_leaves_a_later_call_of_the_client_on_the_same_thread_alone( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Without a deadline, the suspend call runs on the session's thread. + # A later call of the client on that thread is not the session's call. + botocore_client = _botocore_client(monkeypatch) + sent: list[str] = [] + botocore_client.meta.events.register( + operation._SUSPEND_BEFORE_SEND, # noqa: SLF001 + _sent_to(sent), + ) + try: + _suspend(_scope(botocore_client), "mvm-1", None) + botocore_client.suspend_microvm(microvmIdentifier="mvm-2") + finally: + botocore_client.close() + + assert sent == ["mvm-1/suspend", "mvm-2/suspend"] + + def test_checks_the_state_before_the_job_after_a_failed_suspend_after_job( run: Any, fake_client: Any, fake_endpoint: Any ) -> None: