diff --git a/.github/scripts/lmi-ci.sh b/.github/scripts/lmi-ci.sh new file mode 100644 index 000000000..9c9a9a895 --- /dev/null +++ b/.github/scripts/lmi-ci.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# Python commands for the shared LMI reusable workflow's stage contract. +set -euo pipefail + +case "${1:?Expected a workflow stage}" in + setup) + uv pip install hatch==1.16.5 + hatch run lmi:shared-setup + ;; + harness) + hatch run lmi:lint + hatch run lmi:shared-tests --junitxml=lmi_tests/artifacts/unit.xml + hatch run examples:python - <<'PYTHON' +from pathlib import Path +import aws_durable_execution_sdk_python as sdk + +source = Path('packages/aws-durable-execution-sdk-python/src').resolve() +assert Path(sdk.__file__).resolve().is_relative_to(source), sdk.__file__ +PYTHON + ;; + build) hatch run lmi:build ;; + deploy) + hatch run lmi:python -m lmi_tests.deploy deploy \ + --run-id "$LMI_SHARED_RUN_ID" \ + --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" + ;; + run) hatch run lmi:shared ;; + collect) hatch run lmi:python -m lmi_tests.deploy collect ;; + cleanup) hatch run lmi:shared-cleanup ;; + summary) hatch run lmi:python -m lmi_tests.summary ;; + *) printf 'Unknown LMI workflow stage: %s\n' "$1" >&2; exit 2 ;; +esac diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml new file mode 100644 index 000000000..85ff5efda --- /dev/null +++ b/.github/workflows/lmi-e2e-tests.yml @@ -0,0 +1,35 @@ +name: LMI e2e tests + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: + inputs: + region: + description: Region containing the test LMI capacity provider + default: us-west-2 + type: string + +permissions: + contents: read + +jobs: + lifecycle: + permissions: + contents: read + id-token: write + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/lmi-lifecycle.yml@56e7e444457653775579c120469d931b71c8cfd8 + with: + conformance-ref: 56e7e444457653775579c120469d931b71c8cfd8 + sdk-script: .github/scripts/lmi-ci.sh + runtime-label: Python 3.14 + lambda-runtime: python3.14 + stack-name: python-lmi-e2e + region: ${{ inputs.region || 'us-west-2' }} + secrets: + TEST_ROLE_ARN: ${{ secrets.TEST_ROLE_ARN }} + TEST_ACCOUNT_ID: ${{ secrets.TEST_ACCOUNT_ID }} + TEST_LAMBDA_EXECUTION_ROLE_ARN: ${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} + CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} diff --git a/.gitignore b/.gitignore index 13ccb7ad3..516a14a52 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ dist/ **/template.generated.json .env + +/.conformance/ diff --git a/lmi_tests/.gitignore b/lmi_tests/.gitignore new file mode 100644 index 000000000..0d8880bee --- /dev/null +++ b/lmi_tests/.gitignore @@ -0,0 +1,3 @@ +artifacts/ +build/ +__pycache__/ diff --git a/lmi_tests/README.md b/lmi_tests/README.md new file mode 100644 index 000000000..d9859bec2 --- /dev/null +++ b/lmi_tests/README.md @@ -0,0 +1,141 @@ +# Python LMI lifecycle conformance + +This PR adapts the Python SDK to the seven shared lifecycle requirements in +[conformance #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186). +The exact dependency is pinned in `shared_contract.py` and the workflow. Each +shared requirement declares one scalar `scenario`; the Python adapter reads the +shared matrix directly rather than maintaining a second requirement catalog. + +| Requirement | Scenario | Target | Invocation roles | +| --- | --- | --- | --- | +| `lmi-lifecycle-1` | `native-overlap` | c2 | victim, peer | +| `lmi-lifecycle-2` | `normal-return` | c1 | victim | +| `lmi-lifecycle-3` | `deadline-during-step` | c1 | victim | +| `lmi-lifecycle-4` | `peer-recovery` | c2 | victim, peer, recovery | +| `lmi-lifecycle-5` | `suspension-retry` | c1 | victim, resume | +| `lmi-lifecycle-6` | `warm-reuse` | c1 | victim, warm | +| `lmi-lifecycle-7` | `normal-failure` | c1 | victim | + +The shared package owns assertions, AWS transport, persistent deployment, scenario +choreography, source normalization, strict reports, scheduling and retirement. +This repository owns the real Python SDK handlers, observation hooks, ZIP build +and thin runtime bindings. The workflow definition also lives in the shared +repository; `.github/workflows/lmi-e2e-tests.yml` calls its pinned +`lmi-lifecycle.yml`, using `.github/scripts/lmi-ci.sh` for Python stage commands. +OTel/exporter tests and the previous Python-specific matrix are deferred. +There is no local conformance profile or validator that replaces a shared rule. + +## Local validation + +Install Hatch and Python 3.14. Clone the shared repository into `.conformance` +and check out the exact `CONFORMANCE_REVISION` from `shared_contract.py`: + +```bash +git clone https://github.com/aws/aws-durable-execution-conformance-tests.git .conformance +git -C .conformance checkout --detach 56e7e444457653775579c120469d931b71c8cfd8 +hatch run lmi:shared-setup +hatch run lmi:lint +hatch run lmi:shared-tests +hatch run lmi:build +``` + +`shared-setup` verifies the commit and rejects edited shared package sources. +It installs only the shared core runner. The LMI environment contains the checked +out core SDK and testing library; the Lambda bundle contains the core SDK and +its dependencies plus fixture/observation modules. It does not include the OTel +SDK, an OTel plugin, local catalogs, validators or the shared runner itself. + +Python unit tests cover pinned workflow delegation, the Python artifact/manifest +binding, fixture observations and bootstrap. Shared driver regression tests now +live in the conformance repository and cover case ownership, cancellation, source +normalization, driver ordering and cleanup. Local e2e tests use the actual public +SDK and its in-process test service, replacing only AWS transport and evidence +storage. They verify checkpoint/replay and preserve genuine admission failures. +These controls do not establish real LMI placement or deployed acceptance. + +## Cloud execution + +Use a dedicated test account with the configured LMI capacity provider and Lambda +execution role. The workflow validates the account before deployment. The +persistent functions use Python 3.14, arm64, 2048 MiB and native concurrency 1 and +2, with one environment each. SDK artifact and configuration digests are retained +in every manifest and checked before invoking either target. + +Run cloud tests through the workflow's `workflow_dispatch` action or a trusted +same-repository PR. Its fixed `lmi-e2e-shared-capacity-provider` concurrency group +serializes cloud jobs across PRs and refs from deployment through final cleanup, +with `cancel-in-progress: false`. GitHub Actions is the only cross-run concurrency +control. Direct CLI calls do not acquire a lock and are not serialized with CI. + +`shared plan` writes the seven-case manifest without invoking functions. `shared +run` writes under `lmi_tests/artifacts/shared/standard/`: the manifest, coverage, +JSON/JUnit reports, launch journal and retained case evidence. Each invocation +uses a case-owned control/evidence prefix derived from `runId` and `caseId`. +The case hash uses only `caseId` within the run prefix. Plans, payloads and evidence +have no additional cohort identifier. Generate new run artifacts with this pinned +revision; inspect or clean up older artifacts with their original adapter revision. A service resume is journaled under +the same durable execution; it is not another driver Invoke. + +Trusted same-repository pull requests and manual workflow runs exercise the cloud +suite. Fork and Dependabot pull requests run only the unprivileged local harness. +There is no S3 lease, fence token or persistent quarantine record. Existing S3 +ownership objects are not read and cannot block a later workflow run. + +## Actual observations and failure handling + +`fixture.py` uses public durable APIs for the scenarios. Its instrumentation +observer captures actual operation IDs, parents, stored outcomes and user-task +lifetimes. The service-client wrapper forwards every SDK request and response; +it never supplies synthetic checkpoints or changes SDK cancellation flags. +The instrumentation observer is a measurement tool, not a plugin conformance test. + +For normal return/failure, the adapter records the actual durable wrapper boundary +and probes the original context after that boundary. The operation probe uses +`DurableContext.step`; the checkpoint probe calls the SDK's +`context.state.create_checkpoint(is_sync=False)` admission path without issuing a +raw Lambda request. SDK closure/revocation errors are recorded as rejection only when no user code +or new SDK request ran. Probes run after both wrapper return and wrapper raise. An accepted checkpoint, another error, or a still-pending operation +remains visible. Probe tasks are tracked separately; a timeout cannot make them +quiescent. Current SDK gaps are failures, not expected passes or exclusions. + +The deadline fixture holds real I/O across the runtime deadline and cooperatively +exits after it. A runtime clock alone never supplies a `platform_timeout` event: +that event requires the original request's service `InvocationCompleted` history. +The recovery victim first waits in a checkpointed preparation step. The driver +starts the peer halfway through the victim's native timeout, verifies the original +cohort and the peer's remaining recovery window, then releases preparation to +admit the fault. Preparation checkpoints the original native request ID, so +service retries keep the same durable sequence without reinjecting the victim's +fault or admission probes. Recovery overlap excludes expired original requests +and their retries. The original peer remains held until recovery completes. A still-running +durable execution is collected as such; terminal-state waiting and stops belong +to retirement after assertions. Releasing a victim fault cannot establish the +preceding cleanup or recovery assertion. + +The warm fixture creates invocation-owned late work inside a step. A later +invocation must find it in the same process; it observes the old callback using +the original request identity. Failure to reuse that process is a placement or +collection failure, not permission to select a replacement victim. + +`shared_evidence.py` supplies Python error expectations and process identity to +the shared source-evidence normalizer. +Service payload wrappers are decoded consistently with checkpoint snapshots; +truncated results are rejected. Replay compares a real resume's initial +checkpoints to the final service history and counts actual user-function entries. +Missing observations are never filled from expected answers. + +The shared behavior schema asks for one scenario-owned `task_exit`. Other user +and probe tasks remain in the same evidence as `observed_task_start/exit`, with +their original `sourceKind`. The independent retirement snapshot restores all +original task kinds, so a pending task cannot disappear from the audit. +Coordination barriers retain their complete `controlRecords` journal separately +from injected victim `faultsCreated`; releasing a healthy peer is not victim +fault recovery. + +Cleanup releases only the current run's controls and requests stops only for its +identified executions. It validates run, case and function identity before those +requests. It records `NO_CASES`, `CLEANUP_REQUESTED` or `FAILED` in `cleanup.json`; +completed cleanup requests do not claim that every task retired. Assertion and +retirement failures remain in the shared reports. The shared runner can prevent +reuse of an unretired target within the current run, but that state is not stored +as a lock for future workflow runs. diff --git a/lmi_tests/__init__.py b/lmi_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py new file mode 100644 index 000000000..ceb3c2830 --- /dev/null +++ b/lmi_tests/cloud.py @@ -0,0 +1,19 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python fixture binding to the shared AWS driver.""" + +from aws_durable_execution_conformance_tests.lmi.cloud import Cloud as SharedCloud +from lmi_tests.deploy import ARTIFACTS, manager + + +class Cloud(SharedCloud): + def __init__( + self, manifest, configuration=None, *, artifacts_dir=None, case_context=None + ): + super().__init__( + manifest, + configuration, + artifacts_dir=artifacts_dir or ARTIFACTS, + case_context=case_context, + verify_profile=manager().verify, + ) diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py new file mode 100644 index 000000000..95d166b79 --- /dev/null +++ b/lmi_tests/deploy.py @@ -0,0 +1,153 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build the Python fixture and delegate deployment to the shared LMI harness.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import zipfile + +from aws_durable_execution_conformance_tests.lmi.artifacts import ( + clear_run_artifacts, + save, +) +from aws_durable_execution_conformance_tests.lmi.deployment import DeploymentManager +from aws_durable_execution_conformance_tests.lmi.errors import ProvisioningError +from lmi_tests.runtime import RUNTIME + +ROOT = Path(__file__).resolve().parents[1] +ARTIFACTS = ROOT / "lmi_tests/artifacts" +DEFAULT_STACK = "python-lmi-e2e" + + +def manager(): + return DeploymentManager( + runtime=RUNTIME, + owner="python-sdk-lmi-e2e", + artifacts=ARTIFACTS, + bundle=ROOT / "lmi_tests/build/function.zip", + bucket_prefix="py-lmi-e2e", + ) + + +def build(): + clear_run_artifacts(ARTIFACTS) + directory = ROOT / "lmi_tests/build" + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir() + # The tested core SDK comes from this checkout; pip resolves its dependencies. + for package in ("aws-durable-execution-sdk-python",): + subprocess.run( + ["hatch", "build", "-t", "wheel", str(directory)], + cwd=ROOT / "packages" / package, + env={k: v for k, v in os.environ.items() if k != "HATCH_ENV_ACTIVE"}, + check=True, + ) + wheels = sorted(directory.glob("*.whl")) + if len(wheels) != 1: + raise ProvisioningError("Expected exactly the checked-out core SDK wheel") + wheel = next(directory.glob("aws_durable_execution_sdk_python-*.whl")) + target = directory / "package" + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--only-binary=:all:", + "--platform", + "manylinux2014_aarch64", + "--platform", + "manylinux_2_28_aarch64", + "--python-version", + "3.14", + "--implementation", + "cp", + "--abi", + "cp314", + "--no-compile", + "--target", + str(target), + "--report", + str(directory / "dependencies.json"), + *map(str, wheels), + ], + check=True, + ) + (target / "lmi_tests").mkdir() + for name in ( + "__init__.py", + "fixture.py", + "shared_observations.py", + ): + shutil.copy(ROOT / "lmi_tests" / name, target / "lmi_tests" / name) + artifact = directory / "function.zip" + resolved = json.loads((directory / "dependencies.json").read_text()) + with zipfile.ZipFile(artifact, "w", zipfile.ZIP_DEFLATED) as archive: + for path in sorted(target.rglob("*")): + if path.is_file(): + info = zipfile.ZipInfo(path.relative_to(target).as_posix()) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, path.read_bytes()) + save( + ARTIFACTS / "build.json", + { + "commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "wheel": wheel.name, + "wheels": { + item.name: hashlib.sha256(item.read_bytes()).hexdigest() + for item in wheels + }, + "dependencies": [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "hashes": item.get("download_info", {}) + .get("archive_info", {}) + .get("hashes", {}), + } + for item in resolved["install"] + ], + "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), + }, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["build", "deploy", "collect", "cleanup"]) + parser.add_argument("--run-id") + parser.add_argument("--stack-name", default=DEFAULT_STACK) + parser.add_argument("--runtime", choices=["python3.14"], default="python3.14") + args = parser.parse_args() + try: + if args.command == "build": + build() + elif args.command == "deploy": + manager().deploy(args) + else: + manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) + if args.command == "cleanup": + manager().cleanup(manifest) + else: + from lmi_tests.cloud import Cloud + + Cloud(manifest).collect() + except Exception as error: + save( + ARTIFACTS / f"{args.command}-error.json", + {"type": type(error).__name__, "message": str(error)}, + ) + raise + + +if __name__ == "__main__": + main() diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py new file mode 100644 index 000000000..0fc103c71 --- /dev/null +++ b/lmi_tests/fixture.py @@ -0,0 +1,825 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fault-injection handlers for real LMI. Never install these as SDK examples.""" + +from contextlib import nullcontext +from dataclasses import dataclass, field +import fcntl +import json +import os +import threading +import time +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from aws_durable_execution_sdk_python import StepError, durable_execution +from aws_durable_execution_sdk_python.config import ( + Duration, + MapConfig, + StepConfig, +) +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInput, + DurableExecutionInvocationInputWithClient, +) +from aws_durable_execution_sdk_python.lambda_service import LambdaClient +from aws_durable_execution_sdk_python.retries import RetryDecision +from aws_durable_execution_sdk_python.exceptions import ( + BackgroundThreadError, + CheckpointError, + OrphanedChildException, + SuspendExecution, +) +from lmi_tests.shared_observations import ( + GateRelease, + LifecycleObservationFactory, + LifecycleObservations, +) + + +PROCESS_ID = uuid.uuid4().hex +IO_CONFIG = Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}) +NO_RETRY = StepConfig(retry_strategy=lambda _error, _attempt: RetryDecision.no_retry()) + + +def environment_id(path="/tmp/python-lmi-environment"): + """/tmp is shared by the LMI Python workers; lock initialization across processes.""" + with open(path, "a+") as marker: + fcntl.flock(marker, fcntl.LOCK_EX) + marker.seek(0) + value = marker.read().strip() + if not value: + value = uuid.uuid4().hex + marker.write(value) + marker.flush() + os.fsync(marker.fileno()) + return value + + +def resources(): + return {"threads": [{"id": t.ident, "name": t.name} for t in threading.enumerate()]} + + +def observe(trace, kind, **fields): + """Preserve the legacy ledger when this invocation is not a shared case.""" + if trace.payload.get("coverageProfile"): + return trace.observe(kind, **fields) + return None + + +def configuration(trace, name, max_concurrency): + if trace.payload.get("coverageProfile"): + return trace.observer.configuration(name, max_concurrency) + return nullcontext() + + +class Trace: + def __init__(self, payload, context, execution_arn): + self.payload = dict(payload) + if "runId" in self.payload: + self.payload.setdefault("run", self.payload["runId"]) + self.shared = bool(self.payload.get("coverageProfile")) + if self.shared: + for key in ( + "runId", + "caseId", + "role", + "sharedScenario", + "controlPrefix", + ): + if not isinstance(self.payload.get(key), str) or not self.payload[key]: + raise ValueError(f"Shared invocation requires {key}") + if not self.payload["controlPrefix"].startswith( + f"runs/{self.payload['run']}/cases/" + ) or not self.payload["controlPrefix"].endswith("/"): + raise ValueError("Shared controls must use a case-owned run prefix") + self.context = context + self.s3 = boto3.client("s3", config=IO_CONFIG) + self.bucket = os.environ["LMI_BUCKET"] + self.lock = threading.Lock() + # Never make observation capture wait for the S3 publication lock. + self.capture_lock = threading.Lock() + self.capture_sequence = 0 + self.sequence = 0 + self.call_sequence = 0 + self.sdk_request_count = 0 + self.sdk_completed_at = None + self.sdk_observation = None + self.sdk_resources = None + self.identity = { + "run": os.environ["LMI_RUN_ID"], + "commit": os.environ["LMI_COMMIT"], + "marker": self.payload["marker"], + "scenario": self.payload["scenario"], + "execution": execution_arn, + "request": context.aws_request_id, + "environment": environment_id(), + "process": PROCESS_ID, + "pid": os.getpid(), + } + if self.shared: + self.identity.update( + { + key: self.payload[key] + for key in ( + "runId", + "caseId", + "role", + "coverageProfile", + "sharedScenario", + "behaviorBudgets", + ) + if key in self.payload + } + ) + self.control_prefix = self.payload.get( + "controlPrefix", f"runs/{self.identity['run']}/control/" + ) + self.deadline_observed_at_ms = time.time() * 1000 + self.remaining_ms = context.get_remaining_time_in_millis() + self.deadline = (self.deadline_observed_at_ms + self.remaining_ms) / 1000 + self.observer = LifecycleObservations(self) if self.shared else None + + def emit(self, phase, **fields): + # Keep original evidence order independently of retrospective boundaries. + with self.lock: + self.sequence += 1 + if phase == "CONFORMANCE": + fields["observation"] = { + **fields["observation"], + "sourceSequence": self.sequence, + } + event = { + **self.identity, + "phase": phase, + "sequence": self.sequence, + "time": time.time(), + "deadline": self.deadline, + **fields, + } + print("LMI_TEST " + json.dumps(event), flush=True) + self.s3.put_object( + Bucket=self.bucket, + Key=f"runs/{self.identity['run']}/events/{self.identity['marker']}/{self.identity['request']}/{self.sequence:06d}.json", + Body=json.dumps(event).encode(), + ContentType="application/json", + ) + return event + + def capture_observation(self, kind, **fields): + with self.capture_lock: + self.capture_sequence += 1 + at_ms = fields.pop("atMs", time.time_ns() / 1_000_000) + return { + "kind": kind, + "atMs": at_ms, + **fields, + "captureSequence": self.capture_sequence, + } + + def publish_observation(self, observation): + return self.emit("CONFORMANCE", observation=observation) + + def observe(self, kind, **fields): + return self.publish_observation(self.capture_observation(kind, **fields)) + + def next_call_id(self, kind): + with self.lock: + self.call_sequence += 1 + if kind in {"checkpoint", "poll"}: + self.sdk_request_count += 1 + return f"{self.identity['request']}:{kind}:{self.call_sequence}" + + def signal(self, name): + self.s3.put_object( + Bucket=self.bucket, Key=self.control_prefix + name, Body=b"release" + ) + + def released(self, name): + try: + response = self.s3.get_object( + Bucket=self.bucket, Key=self.control_prefix + name + ) + with response["Body"] as body: + state = body.read() + if state not in {b"hold", b"release"}: + raise ValueError(f"Invalid control state for {name}") + return state == b"release" + except (ClientError, ValueError) as error: + self.emit("CONTROL_ERROR", gate=name, error=str(error)) + observe(self, "control_error", gateId=name, error=str(error)) + raise + + def gate(self, name, *, effects=False, attempt=1, ready=None, preparation=False): + # This return value is checkpointed by existing controlled-io steps. + self.gate_release( + name, effects=effects, attempt=attempt, ready=ready, preparation=preparation + ) + + def gate_release( + self, + name, + *, + effects=False, + attempt=1, + ready=None, + on_hold=None, + on_release=None, + preparation=False, + ): + """Observe normal versus emergency release without changing step results.""" + end = time.monotonic() + 150 + entered = False + held_at_ms = None + task_id = self.next_call_id("io") if self.shared else None + observe(self, "io_start", taskId=task_id, gateId=name) + try: + while True: + # Shared cleanup releases the case abort control first. It must + # not masquerade as a normal measurement release of the target. + if self.shared and self.released("release-all"): + mode = "emergency" + release_observed_at_ms = time.time_ns() / 1_000_000 + else: + specific_release = self.released(name) + target_observed_at_ms = time.time_ns() / 1_000_000 + mode = ( + "normal" + if specific_release + else "emergency" + if not self.shared and self.released("release-all") + else None + ) + release_observed_at_ms = ( + target_observed_at_ms + if specific_release + else time.time_ns() / 1_000_000 + ) + if mode is not None: + released = GateRelease( + name, release_observed_at_ms, mode, held_at_ms + ) + if on_release is not None: + on_release(released) + if preparation and mode == "normal": + # The driver releases preparation only after the peer + # joined. This observes the cohort barrier within the + # original invocations' overlap, not at early warm-up. + observe( + self, + "barrier_arrived", + gateId=name, + atMs=released.observed_at_ms, + source="cohort-admission", + ) + observe( + self, + "fault_released", + gateId=name, + mode=mode, + atMs=released.observed_at_ms, + source="control-poll", + ) + return released + if ( + self.payload["scenario"] == "deadline" + and not preparation + and time.time() >= self.deadline + 0.2 + ): + observe(self, "cooperative_io_cancelled", source="runtime-deadline") + raise TimeoutError( + "Cooperative fixture I/O crossed the runtime deadline" + ) + if not entered: + held_at_ms = target_observed_at_ms + if on_hold is not None: + on_hold(held_at_ms) + self.emit("BLOCKED", gate=name) + observe(self, "blocked", gateId=name, atMs=held_at_ms) + if not preparation and not self.payload.get("preparation_gate"): + observe(self, "barrier_arrived", gateId=name, atMs=held_at_ms) + if self.payload["scenario"] == "deadline" and not preparation: + observe(self, "work_in_flight", gateId=name, atMs=held_at_ms) + if self.payload.get("role") == "recovery": + observe(self, "worker_recovered", atMs=held_at_ms) + if time.monotonic() >= end: + self.emit("ESCAPE", gate=name) + observe(self, "escape", gateId=name) + raise TimeoutError( + "Test emergency release; never a passing regression" + ) + if effects: + self.emit( + "EFFECT", operation="blocked-io", gate=name, attempt=attempt + ) + observe( + self, + "effect", + operation="blocked-io", + gateId=name, + attempt=attempt, + ) + else: + self.emit("ALIVE", gate=name) + observe(self, "alive", gateId=name) + if self.payload.get("role") == "peer": + observe(self, "peer_progress", gateId=name) + if not entered and ready: + self.signal(ready) + entered = True + time.sleep(0.2) # fault-injected blocking I/O, not a durable delay + except BaseException as error: + self.emit("IO_INTERRUPTED", gate=name, error=type(error).__name__) + observe( + self, + "io_interrupted", + taskId=task_id, + gateId=name, + error=type(error).__name__, + ) + raise + finally: + exited_at = time.time() * 1000 + self.emit("IO_EXIT", gate=name) + observe(self, "io_exit", taskId=task_id, gateId=name, atMs=exited_at) + + def body(self, name, value, step=None): + attempt = getattr(step, "attempt", 1) + self.emit("BODY", operation=name, value=value, attempt=attempt) + if self.shared: + self.observer.body(name, value, attempt) + return value + + +class ObservedClient: + """Forward real SDK calls; never fabricate checkpoint acknowledgments.""" + + def __init__(self, client, trace): + self.client, self.trace = client, trace + + def get_execution_state(self, **kwargs): + call_id = ( + self.trace.next_call_id("poll") + if self.trace.payload.get("coverageProfile") + else None + ) + self.trace.emit("POLL_CALL") + observe(self.trace, "poll_call", callId=call_id) + try: + return self.client.get_execution_state(**kwargs) + finally: + exited_at = time.time() * 1000 + self.trace.emit("POLL_EXIT") + observe(self.trace, "poll_exit", callId=call_id, atMs=exited_at) + + def checkpoint(self, **kwargs): + updates = [ + { + "id": u.operation_id, + "name": u.name, + "action": u.action.value, + "type": u.operation_type.value, + } + for u in kwargs["updates"] + ] + call_id = ( + self.trace.next_call_id("checkpoint") + if self.trace.payload.get("coverageProfile") + else None + ) + self.trace.emit("CHECKPOINT_CALL", updates=updates) + observe(self.trace, "checkpoint_call", callId=call_id, updates=updates) + try: + response = self.client.checkpoint(**kwargs) + acknowledged_at = time.time() * 1000 + self.trace.emit("CHECKPOINT_ACK", updates=updates) + observe( + self.trace, + "checkpoint_ack", + callId=call_id, + operationIds=[item["id"] for item in updates], + source="service", + atMs=acknowledged_at, + ) + return response + except BaseException as error: + self.trace.emit("CHECKPOINT_ERROR", error=type(error).__name__) + observe( + self.trace, + "checkpoint_error", + callId=call_id, + error=type(error).__name__, + ) + raise + finally: + exited_at = time.time() * 1000 + self.trace.emit("CHECKPOINT_EXIT") + observe(self.trace, "checkpoint_exit", callId=call_id, atMs=exited_at) + + +def workflow(event, context, trace): + trace.durable_context = context + scope = trace.observer.root_scope() if trace.shared else nullcontext() + with scope: + try: + result = run_workflow(event, context, trace) + except Exception: + observe(trace, "root_completed", outcome="FAILED") + raise + else: + observe(trace, "root_completed", outcome="SUCCEEDED") + return result + + +def run_workflow(event, context, trace): + marker, scenario = event["marker"], event["scenario"] + if scenario not in { + "success", + "failure", + "barrier", + "deadline", + "replay", + "warm-owner", + "warm-probe", + }: + raise ValueError(f"Unsupported lifecycle fixture: {scenario}") + try: + if target := event.get("target_environment"): + # Record placement before any fault work. Replays consume the same + # decision even if Lambda later resumes in another environment. + accepted = context.step( + lambda _: trace.identity["environment"] == target, + name="placement-admission", + config=NO_RETRY, + ) + trace.emit( + "PLACEMENT_ACCEPTED" if accepted else "PLACEMENT_MISS", target=target + ) + if not accepted: + return marker + value = context.step(lambda s: trace.body("success", marker, s), name="success") + if scenario == "failure": + raise ValueError("expected:" + marker) + if scenario in {"barrier", "deadline"}: + original_request = None + if preparation_gate := event.get("preparation_gate"): + + def prepare(_): + trace.gate(preparation_gate, preparation=True) + return trace.identity["request"] + + original_request = context.step( + prepare, name="prepare-recovery", config=NO_RETRY + ) + trace.recovery_original_request = original_request + + def controlled_io(step): + # Fault injection belongs to the selected native invocation. + # This decision is inside the step; replay still consumes the + # same preparation checkpoint and durable operation sequence. + if ( + original_request is not None + and trace.identity["request"] != original_request + ): + observe( + trace, "fault_not_reapplied", originalRequestId=original_request + ) + return None + return trace.gate( + event["gate"], effects=scenario == "deadline", attempt=step.attempt + ) + + context.step(controlled_io, name="controlled-io", config=NO_RETRY) + context.step(lambda s: trace.body("after-io", marker, s), name="after-io") + elif scenario == "warm-owner": + context.step( + lambda _: start_warm_work(trace), + name="start-late-work", + config=NO_RETRY, + ) + elif scenario == "warm-probe": + context.step( + lambda _: release_warm_work(trace), + name="release-late-work", + config=NO_RETRY, + ) + elif scenario == "replay": + + def fail(step): + trace.body("failure", marker, step) + raise ValueError("expected:" + marker) + + try: + context.step(fail, name="failure", config=NO_RETRY) + except StepError as error: + trace.emit( + "STORED_FAILURE", message=error.message, error_type=error.error_type + ) + observe( + trace, + "stored_failure", + error={"type": error.error_type, "message": error.message}, + ) + + def retry(step): + trace.body("retry", marker, step) + if step.attempt == 1: + raise ValueError("retry once") + return marker + + context.step( + retry, + name="retry", + config=StepConfig( + retry_strategy=lambda _e, n: ( + RetryDecision.retry(Duration.from_seconds(1)) + if n == 1 + else RetryDecision.no_retry() + ) + ), + ) + context.run_in_child_context( + lambda c: c.map( + [1, 2, 3], + lambda inner, item, _index, _all: inner.step( + lambda _: item, name="item" + ), + name="items", + config=MapConfig(max_concurrency=3), + ), + name="nested", + ) + context.parallel( + [ + lambda c: c.step(lambda _: value, name="left"), + lambda c: c.step(lambda _: value, name="right"), + ], + name="pair", + ) + context.wait(Duration.from_seconds(2), name="pause") + callback = context.create_callback(name="completion") + assert callback.result() == marker + trace.emit("USER_RESULT", value=value) + return value + finally: + exited_at = time.time() * 1000 + trace.emit("USER_EXIT") + observe(trace, "user_exit", atMs=exited_at) + + +def admission_probes(trace): + trace.observe("task_start", taskId="admission-probes", source="fixture") + closed = False + body_executed = False + + def body(_): + nonlocal body_executed + body_executed = True + trace.observe("probe_body_executed") + + try: + for probe, operation in ( + ( + "checkpoint", + lambda: trace.durable_context.state.create_checkpoint(is_sync=False), + ), + ( + "operation", + lambda: trace.durable_context.step( + body, name="operation-admission-probe", config=NO_RETRY + ), + ), + ): + trace.observe("probe_enter", probe=probe) + calls_before = trace.sdk_request_count + error_record = None + try: + operation() + except ( + OrphanedChildException, + BackgroundThreadError, + CheckpointError, + SuspendExecution, + ) as error: + # Failed/revoked SDK state can reject by re-raising its stored + # error. It is a rejection only before user code or a new RPC. + outcome = ( + "REJECTED" + if not body_executed and trace.sdk_request_count == calls_before + else "ERROR" + ) + error_record = {"type": type(error).__name__, "message": str(error)} + except BaseException as error: + outcome = "ERROR" + error_record = {"type": type(error).__name__, "message": str(error)} + else: + outcome = "ACCEPTED" + observed_at = time.time_ns() / 1_000_000 + if outcome == "REJECTED" and not closed: + trace.observe( + "admission_closed", atMs=observed_at, source="sdk-rejection" + ) + closed = True + trace.observe( + "admission_probe", + probe=probe, + outcome=outcome, + error=error_record, + atMs=observed_at, + bodyExecuted=body_executed, + newSdkRequests=trace.sdk_request_count - calls_before, + ) + finally: + trace.observe("task_exit", taskId="admission-probes", source="fixture") + + +def start_admission_probes(trace): + # A broken SDK can leave a late operation waiting forever. Keep that real + # task visible to the retirement audit instead of declaring it settled. + thread = threading.Thread( + target=admission_probes, args=(trace,), daemon=True, name="lmi-admission-probes" + ) + thread.start() + thread.join(timeout=0.5) + + +@dataclass +class WarmWork: + trace: object + ready: threading.Event = field(default_factory=threading.Event) + observed_during: str | None = None + + +WARM_WORK = {} +WARM_LOCK = threading.Lock() + + +def start_warm_work(trace): + key = (trace.payload["runId"], trace.payload["caseId"]) + work = WarmWork(trace) + with WARM_LOCK: + if key in WARM_WORK: + raise ValueError("A warm case cannot replace its original work") + WARM_WORK[key] = work + + def late(): + trace.observe("task_start", taskId="late-callback", source="fixture") + try: + if not work.ready.wait(timeout=120): + trace.observe( + "observation_error", + message="Original late work never observed warm reuse", + ) + return + trace.observe("late_activity", observedDuringRequestId=work.observed_during) + admission_probes(trace) + finally: + trace.observe("task_exit", taskId="late-callback", source="fixture") + with WARM_LOCK: + WARM_WORK.pop(key, None) + + threading.Thread(target=late, daemon=True, name="lmi-late-callback").start() + return None + + +def release_warm_work(trace): + key = (trace.payload["runId"], trace.payload["caseId"]) + with WARM_LOCK: + work = WARM_WORK.get(key) + if work is None: + raise ValueError("Warm request did not reuse the original process") + work.observed_during = trace.identity["request"] + work.ready.set() + # Keep the warm invocation alive long enough to observe the old callback. + time.sleep(0.5) + return None + + +def handler(event, context): + invocation = ( + event + if isinstance(event, DurableExecutionInvocationInputWithClient) + else DurableExecutionInvocationInput.from_json_dict(event) + ) + payload = next( + json.loads(op.execution_details.input_payload) + for op in invocation.initial_execution_state.operations + if op.execution_details and op.execution_details.input_payload + ) + payload = dict(payload) + if "runId" in payload: + payload.setdefault("run", payload["runId"]) + if payload["run"] != os.environ["LMI_RUN_ID"]: + raise ValueError("Input belongs to a different deployment") + trace = Trace(payload, context, invocation.durable_execution_arn) + source_client = ( + invocation.service_client + if isinstance(invocation, DurableExecutionInvocationInputWithClient) + else LambdaClient.initialize_client() + ) + client = ObservedClient(source_client, trace) + injected = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( + invocation, client + ) + + def invoke(plugins=None): + configured_plugins = ( + [LifecycleObservationFactory(trace.observer), *(plugins or [])] + if payload.get("coverageProfile") + else plugins + ) + decorated = durable_execution( + lambda e, c: workflow(e, c, trace), plugins=configured_plugins + ) + try: + return decorated(injected, context) + finally: + # Preserve the actual SDK boundary before post-call evidence I/O. + completed_at_ns = time.time_ns() + trace.sdk_completed_at = completed_at_ns / 1_000_000_000 + # Both semantic records describe this same boundary. Converting + # seconds back to milliseconds can round to a different float. + trace.sdk_completed_at_ms = completed_at_ns / 1_000_000 + if payload.get("coverageProfile"): + trace.sdk_observation = trace.capture_observation( + "sdk_completion", atMs=trace.sdk_completed_at_ms + ) + trace.sdk_resources = resources() + + before = resources() + entered_at = time.time() * 1000 + trace.emit("WRAPPER_ENTER", resources=before) + observe(trace, "wrapper_enter", atMs=entered_at, resources=before) + if payload.get("coverageProfile"): + trace.observe( + "runtime_deadline", + atMs=trace.deadline_observed_at_ms, + remainingTimeMs=trace.remaining_ms, + deadlineAtMs=trace.deadline * 1000, + source="runtime", + ) + try: + result = invoke() + trace.emit( + "WRAPPER_RETURN", + status=result["Status"], + resources=trace.sdk_resources, + time=trace.sdk_completed_at, + ) + if payload.get("coverageProfile"): + trace.publish_observation( + { + **trace.sdk_observation, + "kind": "wrapper_return", + "outcome": result["Status"], + "resources": trace.sdk_resources, + } + ) + if result["Status"] == "PENDING": + observe( + trace, + "pending", + atMs=trace.sdk_completed_at_ms, + source="sdk-wrapper", + ) + return result + except BaseException as error: + if payload.get("coverageProfile") and trace.sdk_observation is None: + trace.sdk_observation = trace.capture_observation("sdk_completion") + trace.emit( + "WRAPPER_RAISE", + error=type(error).__name__, + resources=getattr(trace, "sdk_resources", None) or resources(), + time=getattr(trace, "sdk_completed_at", None) or time.time(), + ) + if payload.get("coverageProfile"): + trace.publish_observation( + { + **trace.sdk_observation, + "kind": "wrapper_raise", + "outcome": "THREW", + "error": {"type": type(error).__name__, "message": str(error)}, + "resources": getattr(trace, "sdk_resources", None) or resources(), + } + ) + raise + finally: + if ( + payload.get("sharedScenario") + in { + "normal-return", + "normal-failure", + "deadline-during-step", + "peer-recovery", + } + and payload.get("role") == "victim" + and ( + payload.get("sharedScenario") != "peer-recovery" + or trace.identity["request"] + == getattr(trace, "recovery_original_request", None) + ) + ): + start_admission_probes(trace) diff --git a/lmi_tests/runtime.py b/lmi_tests/runtime.py new file mode 100644 index 000000000..019d47b59 --- /dev/null +++ b/lmi_tests/runtime.py @@ -0,0 +1,7 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python fixture configuration for the shared LMI harness.""" + +from aws_durable_execution_conformance_tests.lmi.runtime import RuntimeDefinition + +RUNTIME = RuntimeDefinition("python", "lmi_tests.fixture.handler") diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md new file mode 100644 index 000000000..c8a7c9d6b --- /dev/null +++ b/lmi_tests/shared-suite-design.md @@ -0,0 +1,35 @@ +# Shared lifecycle suite ownership + +The Python adapter consumes the seven standard lifecycle requirements pinned by +`shared_contract.py`. Each YAML file in the shared repository has one `scenario`. +Normal return is requirement 2; normal failure is requirement 7. There is no +Python-owned replacement catalog, OTel profile or exporter matrix in this PR. + +The shared repository owns requirement descriptions, roles, minimum native +concurrency, scheduling, behavior validators, AWS transport, deployment, scenario +driving, evidence normalization and strict result aggregation. It also owns the +reusable `lmi-lifecycle.yml` workflow, called at the same pinned SHA as the shared +package. The SDK repository keeps `.github/scripts/lmi-ci.sh` for Python toolchain +and stage commands, and maps each scenario to its real public API fixture and retains +source observations. `make_manifest` reads the shared matrix, uses c1 for +single-worker cases and c2 for overlap/recovery, and preserves every required role. + +The same shared runner handles admission, original launch identity, immutable +case observations and independent retirement auditing. The adapter never replaces +a failed original with a retry, translates a runtime deadline into invented +service evidence, or clears a resource merely because cleanup was attempted. + +The deployed instrumentation uses the public SDK observation hooks plus a +transparent service-client wrapper. It records operations and user-code entries; +it does not change production SDK behavior. Admission probes expose actual SDK +rejections, acceptance and pending work. Local harness tests validate that these +outcomes reach the shared validator accurately; they do not claim real-LMI +acceptance or fix the underlying SDK lifecycle behavior. + +The core-only Lambda bundle excludes driver code and conformance packages. CI +retains the dedicated test-account boundary, trusted-PR condition, exact dependency +pin and run-scoped cleanup. GitHub Actions is the sole cross-run concurrency +guard: the fixed cloud-job group covers deployment, assertions and cleanup, with +`cancel-in-progress: false`. The adapter has no shared-lock module, S3 lease, +fence token or persisted quarantine. The shared runner still schedules individual +cases and reports retirement failures within each run. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py new file mode 100644 index 000000000..f717f0b1b --- /dev/null +++ b/lmi_tests/shared.py @@ -0,0 +1,142 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Bootstrap the pinned shared harness and run it with the Python adapter.""" + +from __future__ import annotations +import argparse +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Any, Mapping +from lmi_tests.shared_contract import CONFORMANCE_REVISION + +ROOT = Path(__file__).resolve().parents[1] +ARTIFACTS = ROOT / "lmi_tests/artifacts" +COVERAGE_PROFILES = ("standard",) + + +def verify_checkout(checkout: Path) -> None: + revision = subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True + ).strip() + if revision != CONFORMANCE_REVISION: + raise ValueError( + f"Conformance checkout must be pinned to {CONFORMANCE_REVISION}" + ) + changed = subprocess.check_output( + ["git", "-C", str(checkout), "status", "--porcelain", "--", "packages"], + text=True, + ) + if changed: + raise ValueError("Pinned conformance package sources have local changes") + + +def setup(checkout: Path) -> None: + verify_checkout(checkout) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--editable", + str(checkout / "packages/aws-durable-execution-conformance-tests"), + ], + check=True, + ) + + +def verify_install(checkout: Path) -> None: + verify_checkout(checkout) + import aws_durable_execution_conformance_tests as core + + for module in (core,): + if not Path(module.__file__).resolve().is_relative_to(checkout.resolve()): + raise ValueError( + "Installed conformance packages are not from the pinned checkout; run shared setup" + ) + + +def make_manifest( + deployment: Mapping[str, Any], coverage: str, directory: Path +) -> dict[str, Any]: + from aws_durable_execution_conformance_tests.lmi.runner import ( + make_manifest as shared_manifest, + ) + from lmi_tests.shared_contract import configuration_identity + + core_wheels = { + name: digest + for name, digest in deployment["wheels"].items() + if name.startswith("aws_durable_execution_sdk_python-") + } + if len(core_wheels) != 1: + raise ValueError("Deployment does not retain an exact SDK wheel identity") + return shared_manifest( + deployment, + coverage, + directory, + runtime="python", + artifact_id="sha256:" + next(iter(core_wheels.values())), + conformance_revision=CONFORMANCE_REVISION, + configuration_identity=configuration_identity, + ) + + +def run(deployment: Mapping[str, Any], directory: Path) -> int: + from aws_durable_execution_conformance_tests.lmi.runner import run as shared_run + + return shared_run( + deployment, + directory, + manifest_factory=make_manifest, + adapter_name="lmi_tests.shared_adapter:create_adapter", + ) + + +def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: + from aws_durable_execution_conformance_tests.lmi.runner import ( + cleanup as shared_cleanup, + ) + + return shared_cleanup(deployment, directory) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("setup", "plan", "run", "cleanup")) + parser.add_argument( + "--checkout", + type=Path, + default=Path( + os.environ.get("LMI_CONFORMANCE_CHECKOUT", str(ROOT / ".conformance")) + ), + ) + parser.add_argument("--manifest", type=Path, default=ARTIFACTS / "manifest.json") + parser.add_argument("--output", type=Path, default=ARTIFACTS / "shared") + args = parser.parse_args() + if args.command == "setup": + setup(args.checkout) + return + from aws_durable_execution_conformance_tests.lmi.artifacts import save + from aws_durable_execution_conformance_tests.lmi.errors import CollectionError + + verify_install(args.checkout) + deployment = json.loads(args.manifest.read_text()) + expected_run = os.environ.get("LMI_SHARED_RUN_ID") + if expected_run and deployment.get("run") != expected_run: + raise CollectionError("Deployment manifest belongs to a different CI run") + if args.command == "cleanup": + raise SystemExit(cleanup(deployment, args.output)) + if args.command == "plan": + for coverage in COVERAGE_PROFILES: + output = args.output / coverage + save(output / "manifest.json", make_manifest(deployment, coverage, output)) + return + raise SystemExit(run(deployment, args.output)) + + +if __name__ == "__main__": + main() diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py new file mode 100644 index 000000000..0da3220c9 --- /dev/null +++ b/lmi_tests/shared_adapter.py @@ -0,0 +1,19 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Compose the shared LMI adapter with Python fixture policies.""" + +from aws_durable_execution_conformance_tests.lmi.adapter import SharedAdapter +from lmi_tests.cloud import Cloud +from lmi_tests.shared_contract import configuration_identity +from lmi_tests.shared_evidence import build_evidence +from lmi_tests.shared_scenarios import ScenarioDriver + + +def create_adapter(manifest): + return SharedAdapter( + manifest, + cloud_factory=Cloud, + driver_factory=ScenarioDriver, + evidence_builder=build_evidence, + configuration_identity=configuration_identity, + ) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py new file mode 100644 index 000000000..5f0207c2c --- /dev/null +++ b/lmi_tests/shared_contract.py @@ -0,0 +1,20 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned runner and verified deployment identities for the SDK adapter.""" + +from __future__ import annotations + +from typing import Any, Mapping + +CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" +# Updated only with the matching reviewed shared evidence contract. +CONFORMANCE_REVISION = "56e7e444457653775579c120469d931b71c8cfd8" + + +def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: + from aws_durable_execution_conformance_tests.lmi.configuration import ( + configuration_identity as identity, + ) + from lmi_tests.runtime import RUNTIME + + return identity(deployment, key, RUNTIME) diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py new file mode 100644 index 000000000..4b9a2a675 --- /dev/null +++ b/lmi_tests/shared_evidence.py @@ -0,0 +1,47 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python expected outcomes and worker identity for shared evidence normalization.""" + +from aws_durable_execution_conformance_tests.lmi.source_evidence import ( + build_evidence as shared_build_evidence, +) + + +def expected_outcome(item, scenario): + fixture = item.get("payload", {}).get("scenario") + if fixture == "failure": + return { + "status": "FAILED", + "error": { + "type": "ValueError", + "message": "expected:" + item["marker"], + }, + } + elif fixture == "deadline" and scenario in { + "deadline-during-step", + "peer-recovery", + }: + # This fixture asserts native deadline/retry behavior. Its logical + # execution may still be active or may finish after normal release. + return { + "allowedStatuses": [ + "RUNNING", + "SUCCEEDED", + "FAILED", + "TIMED_OUT", + "STOPPED", + ] + } + else: + return {"status": "SUCCEEDED", "result": item["marker"]} + + +def build_evidence(context, cloud, *, phase="observations"): + return shared_build_evidence( + context, + cloud, + runtime="python", + expected_outcome=expected_outcome, + worker_identity=lambda raw: raw["process"], + phase=phase, + ) diff --git a/lmi_tests/shared_observations.py b/lmi_tests/shared_observations.py new file mode 100644 index 000000000..7cde20d9c --- /dev/null +++ b/lmi_tests/shared_observations.py @@ -0,0 +1,349 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Invocation-owned evidence from public SDK hooks, without conformance imports.""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +import threading +from typing import Any, Protocol + +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + InvocationEndInfo, + InvocationStartInfo, + OperationChangeInfo, + OperationEndInfo, + OperationInfo, + OperationStartInfo, + OperationType, + UserFunctionEndInfo, + UserFunctionStartInfo, +) + + +class ObservationSink(Protocol): + """Record SDK observations without importing fixture.Trace.""" + + def observe(self, kind: str, **fields: Any) -> Mapping[str, Any]: ... + + +@dataclass(frozen=True) +class GateRelease: + gate_id: str + observed_at_ms: float + mode: str + held_at_ms: float | None = None + + +@dataclass(frozen=True) +class OperationObservation: + operation_id: str + parent_id: str | None + name: str | None + operation_type: str + sub_type: str | None + status: str + replayed: bool + attempt: int | None + result: str | None + error_type: str | None + error_message: str | None + started_at_ms: float | None + ended_at_ms: float | None + + @classmethod + def from_info(cls, info: OperationInfo) -> OperationObservation: + return cls( + operation_id=info.operation_id, + parent_id=info.parent_id, + name=info.name, + operation_type=info.operation_type.value, + sub_type=info.sub_type, + status=info.status.value, + replayed=info.is_replayed, + attempt=info.attempt, + result=info.result, + error_type=info.error.type if info.error else None, + error_message=info.error.message if info.error else None, + started_at_ms=info.start_time.timestamp() * 1000 + if info.start_time + else None, + ended_at_ms=info.end_time.timestamp() * 1000 if info.end_time else None, + ) + + def to_record(self) -> dict[str, Any]: + return { + "operationId": self.operation_id, + "parentId": self.parent_id, + "name": self.name, + "operationType": self.operation_type, + "subType": self.sub_type, + "status": self.status, + "isReplayed": self.replayed, + "attempt": self.attempt, + "serializedResult": self.result, + "error": {"type": self.error_type, "message": self.error_message} + if self.error_type is not None or self.error_message is not None + else None, + "startedAtMs": self.started_at_ms, + "endedAtMs": self.ended_at_ms, + } + + +@dataclass(frozen=True) +class UserTask: + operation: OperationObservation + task_id: str + + +class _ThreadTasks(threading.local): + def __init__(self) -> None: + self.stack: list[UserTask] = [] + + +class LifecycleObservations(DurableInstrumentationPlugin): + """One native invocation's actual operations and active public user scopes.""" + + def __init__(self, sink: ObservationSink) -> None: + self._sink = sink + self._lock = threading.Lock() + self._operations: dict[str, OperationObservation] = {} + self._closed: set[str] = set() + self._configurations: dict[tuple[str, str | None], int] = {} + self._threads = _ThreadTasks() + self._task_sequence = 0 + + def error(self, message: str) -> None: + self._sink.observe("observation_error", message=message) + + def _remember(self, info: OperationInfo) -> OperationObservation: + operation = OperationObservation.from_info(info) + with self._lock: + self._operations[operation.operation_id] = operation + return operation + + def current(self) -> OperationObservation | None: + return self._threads.stack[-1].operation if self._threads.stack else None + + def parent_id(self) -> str | None: + current = self.current() + if current is None: + self.error("Public user-function scope is unavailable") + return None + return ( + None + if current.operation_type == OperationType.EXECUTION.value + else current.operation_id + ) + + @contextmanager + def root_scope(self) -> Iterator[None]: + """Observe the actual public handler body; root user hooks do not fire.""" + with self._lock: + roots = [ + item + for item in self._operations.values() + if item.operation_type == OperationType.EXECUTION.value + ] + if len(roots) != 1: + self.error("Handler entry has no unique public invocation root operation") + yield + return + task = UserTask(roots[0], "root") + self._threads.stack.append(task) + self._sink.observe( + "task_start", + taskId="root", + source="public-handler", + **task.operation.to_record(), + ) + try: + yield + finally: + self._threads.stack.pop() + self._sink.observe( + "task_exit", + taskId="root", + operationId=task.operation.operation_id, + source="public-handler", + ) + + def body(self, name: str, value: Any, attempt: int) -> None: + current = self.current() + if ( + current is None + or current.operation_type != OperationType.STEP.value + or current.name != name + ): + self.error(f"Body {name!r} has no matching public STEP hook") + return + if current.attempt != attempt: + self.error(f"Body {name!r} attempt differs from its public STEP hook") + return + self._sink.observe( + "body", + operation=name, + operationId=current.operation_id, + parentId=current.parent_id, + attempt=attempt, + value=value, + ) + + def closed_ancestor(self) -> str | None: + current = self.current() + if current is None: + self.error("Late operation has no active public user-function scope") + return None + with self._lock: + operation: OperationObservation | None = current + while operation is not None: + if operation.operation_id in self._closed: + return operation.operation_id + operation = self._operations.get(operation.parent_id or "") + self.error("Late operation has no observed completed ancestor") + return None + + def completed_context(self, name: str) -> str | None: + parent_id = self.parent_id() + with self._lock: + matches = [ + operation.operation_id + for operation in self._operations.values() + if operation.name == name + and operation.parent_id == parent_id + and operation.operation_type == OperationType.CONTEXT.value + and operation.operation_id in self._closed + ] + if len(matches) != 1: + self.error( + f"Returned context {name!r} has no unique public completion hook" + ) + return None + return matches[0] + + @contextmanager + def configuration(self, name: str, max_concurrency: int) -> Iterator[None]: + """Bind an actual public call's config to its operation-start hook.""" + key = (name, self.parent_id()) + with self._lock: + self._configurations[key] = max_concurrency + try: + yield + finally: + with self._lock: + missing = self._configurations.pop(key, None) + if missing is not None: + self.error( + f"Configured call {name!r} had no matching public operation hook" + ) + + def on_invocation_start(self, info: InvocationStartInfo) -> None: + operations = [ + self._remember(operation) for operation in info.operations.values() + ] + with self._lock: + self._closed.update( + operation.operation_id + for operation in operations + if operation.operation_type == OperationType.CONTEXT.value + and operation.status + in {"SUCCEEDED", "FAILED", "CANCELLED", "TIMED_OUT", "STOPPED"} + ) + self._sink.observe( + "invocation_start", + requestId=info.request_id, + executionArn=info.execution_arn, + isFirstInvocation=info.is_first_invocation, + operations=[operation.to_record() for operation in operations], + ) + + def on_invocation_end(self, info: InvocationEndInfo) -> None: + operations = [ + self._remember(operation) for operation in info.operations.values() + ] + self._sink.observe( + "invocation_end", + outcome=info.status.value, + operations=[operation.to_record() for operation in operations], + serializedResult=info.execution_result, + error={"type": info.error.type, "message": info.error.message} + if info.error + else None, + ) + + def on_operation_start(self, info: OperationStartInfo) -> None: + operation = self._remember(info) + self._sink.observe("operation_start", **operation.to_record()) + with self._lock: + configured = self._configurations.pop( + (operation.name or "", operation.parent_id), None + ) + if configured is not None: + self._sink.observe( + "operation_config", + operationId=operation.operation_id, + maxConcurrency=configured, + source="public-fixture", + ) + + def on_operation_end(self, info: OperationEndInfo) -> None: + operation = self._remember(info) + self._sink.observe("operation_end", **operation.to_record()) + if info.operation_type is OperationType.CONTEXT: + with self._lock: + self._closed.add(operation.operation_id) + self._sink.observe( + "scope_closed", + scopeId=operation.operation_id, + parentId=operation.parent_id, + ) + + def on_operation_change(self, info: OperationChangeInfo) -> None: + changed = [ + self._remember(operation) for operation in info.updated_operations.values() + ] + self._sink.observe( + "operation_change", + operations=[operation.to_record() for operation in changed], + ) + + def on_user_function_start(self, info: UserFunctionStartInfo) -> None: + operation = self._remember(info) + with self._lock: + self._task_sequence += 1 + ordinal = self._task_sequence + task_id = ( + "root" + if info.operation_type is OperationType.EXECUTION + else f"user-task-{ordinal}" + ) + self._threads.stack.append(UserTask(operation, task_id)) + self._sink.observe("task_start", taskId=task_id, **operation.to_record()) + + def on_user_function_end(self, info: UserFunctionEndInfo) -> None: + stack = self._threads.stack + if not stack or stack[-1].operation.operation_id != info.operation_id: + self.error("Public user-function end did not match its active start") + return + task = stack.pop() + self._sink.observe( + "task_exit", + taskId=task.task_id, + operationId=info.operation_id, + parentId=info.parent_id, + outcome=info.outcome.value, + attempt=task.operation.attempt, + ) + + +@dataclass(frozen=True) +class LifecycleObservationFactory: + """One factory/observer is constructed for each native fixture invocation.""" + + observer: LifecycleObservations + + def create_plugin(self, info: InvocationStartInfo) -> DurableInstrumentationPlugin: + return self.observer diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py new file mode 100644 index 000000000..797b64902 --- /dev/null +++ b/lmi_tests/shared_scenarios.py @@ -0,0 +1,33 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Bind shared lifecycle choreography to Python fixture entry points.""" + +from aws_durable_execution_conformance_tests.lmi.scenarios import ( + ScenarioDriver as SharedScenarioDriver, +) + +FIXTURES = { + "native-overlap": "barrier", + "normal-return": "success", + "deadline-during-step": "deadline", + "peer-recovery": "deadline", + "suspension-retry": "replay", + "warm-reuse": "warm-owner", + "normal-failure": "failure", +} + + +class ScenarioDriver(SharedScenarioDriver): + def __init__(self, context, cloud, **options): + super().__init__( + context, + cloud, + runtime="python", + fixtures={ + **FIXTURES, + "peer": "barrier", + "recovery": "barrier", + "warm": "warm-probe", + }, + **options, + ) diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py new file mode 100644 index 000000000..0a2e9fad2 --- /dev/null +++ b/lmi_tests/summary.py @@ -0,0 +1,9 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Render the shared report for the Python fixture's artifact directory.""" + +from pathlib import Path +from aws_durable_execution_conformance_tests.lmi.summary import summarize + +if __name__ == "__main__": + print(summarize(Path(__file__).parent / "artifacts")) diff --git a/lmi_tests/tests/e2e/test_peer_recovery_local.py b/lmi_tests/tests/e2e/test_peer_recovery_local.py new file mode 100644 index 000000000..8fcb3e132 --- /dev/null +++ b/lmi_tests/tests/e2e/test_peer_recovery_local.py @@ -0,0 +1,114 @@ +"""The real SDK checkpoints preparation before the controlled recovery fault.""" + +import json + +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def test_preparation_gate_precedes_fault_and_keeps_one_original_barrier(monkeypatch): + ledger = install_ledger(monkeypatch) + payload = shared_payload("deadline") + payload.update( + sharedScenario="peer-recovery", preparation_gate="prepare", gate="fault" + ) + prefix = payload["controlPrefix"] + for gate in ("prepare", "fault", "release-all"): + ledger.objects[prefix + gate] = b"hold" + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=15) + prepared = ledger.wait_for("blocked")["observation"] + assert prepared["gateId"] == "prepare" + assert not [ + e + for e in ledger.observations() + if e["observation"]["kind"] == "barrier_arrived" + ] + assert not [ + e + for e in ledger.observations() + if e["observation"]["kind"] == "work_in_flight" + ] + ledger.objects[prefix + "prepare"] = b"release" + fault = ledger.wait_for("work_in_flight")["observation"] + assert fault["gateId"] == "fault" + ledger.objects[prefix + "fault"] = b"release" + result = runner.wait_for_result(arn, timeout=15) + assert result.result == '"marker"' + observations = [event["observation"] for event in ledger.observations()] + barriers = [event for event in observations if event["kind"] == "barrier_arrived"] + assert len(barriers) == 1 and barriers[0]["gateId"] == "prepare" + released = next( + e + for e in observations + if e["kind"] == "fault_released" and e["gateId"] == "prepare" + ) + assert barriers[0]["atMs"] == released["atMs"] + assert prepared["atMs"] < barriers[0]["atMs"] <= fault["atMs"] + checkpoint = next( + event + for event in observations + if event["kind"] == "checkpoint_ack" + and result.get_step("prepare-recovery").operation_id in event["operationIds"] + ) + assert prepared["atMs"] < checkpoint["atMs"] <= fault["atMs"] + assert not [event for event in observations if event["kind"] == "observation_error"] + + +def test_service_retry_replays_preparation_without_reinjecting_the_original_fault( + monkeypatch, +): + from aws_durable_execution_sdk_python.exceptions import RetryableSerDesError + + ledger = install_ledger(monkeypatch) + payload = shared_payload("deadline") + payload.update( + sharedScenario="peer-recovery", preparation_gate="prepare", gate="fault" + ) + prefix = payload["controlPrefix"] + ledger.objects[prefix + "prepare"] = b"release" + ledger.objects[prefix + "fault"] = b"hold" + ledger.objects[prefix + "release-all"] = b"hold" + gate = fixture.Trace.gate + fault_requests = [] + + def interrupted(trace, name, **options): + if name == "fault": + fault_requests.append(trace.identity["request"]) + # Trigger the SDK's actual invocation-retry path after preparation + # checkpointed, leaving controlled-io incomplete for replay. + raise RetryableSerDesError("transient interruption") + return gate(trace, name, **options) + + monkeypatch.setattr(fixture.Trace, "gate", interrupted) + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(payload), execution_timeout=20) + assert result.result == '"marker"' + assert len(fault_requests) == 1 + observations = ledger.observations() + raised = next( + e["observation"] + for e in observations + if e["request"] == fault_requests[0] + and e["observation"]["kind"] == "wrapper_raise" + ) + original_probes = [ + e["observation"] + for e in observations + if e["request"] == fault_requests[0] + and e["observation"]["kind"] == "admission_probe" + ] + assert original_probes and all(e["atMs"] >= raised["atMs"] for e in original_probes) + skipped = [ + e for e in observations if e["observation"]["kind"] == "fault_not_reapplied" + ] + assert len(skipped) == 1 + assert skipped[0]["request"] != fault_requests[0] + assert skipped[0]["observation"]["originalRequestId"] == fault_requests[0] + assert not [ + e + for e in observations + if e["request"] == skipped[0]["request"] + and e["observation"]["kind"] == "admission_probe" + ] diff --git a/lmi_tests/tests/e2e/test_shared_driver_local.py b/lmi_tests/tests/e2e/test_shared_driver_local.py new file mode 100644 index 000000000..d5816900d --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_driver_local.py @@ -0,0 +1,142 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise shared phase routing against the real SDK's local durable runner. + +This checks public handler/callback/replay behavior, not real-LMI placement or +cloud conformance acceptance. +""" + +import json +from threading import Lock +import time + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests.fixture import workflow +from lmi_tests.shared_scenarios import ScenarioDriver + + +class LocalTrace: + def __init__(self): + self.events = [] + self.shared = False + self.payload = {} + self.lock = Lock() + + def emit(self, phase, **fields): + with self.lock: + self.events.append({"phase": phase, "time": time.time(), **fields}) + + def body(self, name, value, step): + self.emit("BODY", operation=name, value=value, attempt=step.attempt) + return value + + def snapshot(self): + with self.lock: + return list(self.events) + + +class LocalService: + """Replace only AWS transport, keeping the actual fixture and SDK runner.""" + + def __init__(self): + self.manifest = {"driverTimeout": 15} + self.trace = LocalTrace() + self.invocations = [] + self.callback_records = [] + self.result = None + decorated = durable_execution( + lambda event, context: workflow(event, context, self.trace) + ) + + def handler(event, context): + self.trace.emit("WRAPPER_ENTER", request=context.aws_request_id) + result = decorated(event, context) + self.trace.emit( + "WRAPPER_RETURN", + request=context.aws_request_id, + status=result["Status"], + ) + return result + + self.runner = DurableFunctionTestRunner(handler, poll_interval=0.01) + + def start(self, fixture, **options): + payload = {"marker": options["marker"], "scenario": fixture} + arn = self.runner.run_async(input=payload) + item = {"arn": arn, "marker": payload["marker"], "runner": self.runner} + self.invocations.append(item) + return item + + def poll(self, predicate, seconds=15, **options): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + observed = predicate() + if observed: + return observed + time.sleep(0.01) + raise AssertionError(options.get("message", "local observation timeout")) + + def phase(self, item, phase, **options): + return self.poll( + lambda: [ + event for event in self.trace.snapshot() if event["phase"] == phase + ], + **options, + ) + + def for_item(self, item): + return self.trace.snapshot() + + def check_active(self): + pass + + def finish(self, item, status="SUCCEEDED"): + self.result = self.runner.wait_for_result(item["arn"], timeout=15) + assert self.result.status.value == status + + +def test_shared_callback_driver_resumes_the_original_public_sdk_execution(): + service = LocalService() + profile = Profile("native-1", "local-function", 1, 1, "local-sdk", "local-config") + context = ScenarioContext( + "local-run", + Scenario( + "callback-case", + "lmi-lifecycle-5", + profile.profile_id, + "suspension-retry", + roles=("victim", "resume"), + ), + profile, + ) + driver = ScenarioDriver(context, service) + with service.runner: + driver.start() + driver.place() + driver.execute() + assert len(service.invocations) == 1 + assert len(service.callback_records) == 1 + assert service.callback_records[0]["executionArn"] == service.invocations[0]["arn"] + assert json.loads(service.result.result) == service.invocations[0]["marker"] + events = service.trace.snapshot() + assert any( + event["phase"] == "WRAPPER_RETURN" and event["status"] == "PENDING" + for event in events + ) + # The local runner deliberately reuses its fake AWS request ID. Actual + # wrapper calls, PENDING results and skipped bodies establish local replay; + # this is not a claim about native LMI request identity or placement. + assert sum(event["phase"] == "WRAPPER_ENTER" for event in events) > 1 + bodies = [event for event in events if event["phase"] == "BODY"] + assert sum(event["operation"] == "success" for event in bodies) == 1 + assert sum(event["operation"] == "failure" for event in bodies) == 1 + assert [event["attempt"] for event in bodies if event["operation"] == "retry"] == [ + 1, + 2, + ] diff --git a/lmi_tests/tests/e2e/test_shared_evidence_local.py b/lmi_tests/tests/e2e/test_shared_evidence_local.py new file mode 100644 index 000000000..795d8576e --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_evidence_local.py @@ -0,0 +1,162 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real SDK invocation/replay evidence reaches the shared validator unchanged.""" + +import json +import time + +import pytest +from pathlib import Path + +from aws_durable_execution_conformance_tests.lmi.app import validate +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.cloud import Cloud +from lmi_tests.shared_evidence import build_evidence +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( + monkeypatch, tmp_path +): + ledger = install_ledger(monkeypatch) + payload = shared_payload("replay") + payload["sharedScenario"] = "suspension-retry" + payload["behaviorBudgets"] = {"observationGraceMs": 1} + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + callback = runner.wait_for_callback(arn, name="completion", timeout=20) + runner.send_callback_success(callback, result=b"marker") + result = runner.wait_for_result(arn, timeout=20) + history = runner.get_execution_history( + arn, include_execution_data=True + ).to_dict()["Events"] + evidence, cloud = assemble( + ledger, + result, + history, + payload, + tmp_path, + "lmi-lifecycle-5", + "suspension-retry", + ) + assert evidence["collectionErrors"] == [] + assert validate(evidence) == [] + bodies = [event for event in evidence["events"] if event["kind"] == "body"] + assert len([event for event in bodies if event["operation"] == "success"]) == 1 + # Losing a real source object must not turn into an inferred complete stream. + cloud.events.pop(next(iter(cloud.events))) + incomplete = build_evidence(cloud.case_context, cloud, phase="observations") + assert incomplete["collectionErrors"] + assert incomplete["collection"]["watermarks"] == {} + + +def assemble(ledger, result, history, payload, tmp_path, requirement_id, scenario): + raw = ledger.records() + arn = next(e["execution"] for e in raw if e["phase"] == "WRAPPER_ENTER") + profile = Profile( + "c1", "local-function", 1, 1, "actual-local-sdk", "local-config", "reuse-worker" + ) + context = ScenarioContext( + "unit", + Scenario( + "case", + requirement_id, + "c1", + scenario, + roles=("victim", "resume") + if scenario == "suspension-retry" + else ("victim",), + coverage_profile="standard", + behavior_budgets={"observationGraceMs": 1}, + ), + profile, + ) + raw = ledger.records() + entries = sorted( + (event for event in raw if event["phase"] == "WRAPPER_ENTER"), + key=lambda event: event["time"], + ) + roles = {} + for index, event in enumerate(entries): + role = "resume" if index and scenario == "suspension-retry" else "victim" + roles[event["request"]] = role + context.record_attempt(event["request"], role=role, requestId=event["request"]) + cloud = object.__new__(Cloud) + cloud.case_context = context + cloud.manifest = { + "commit": "tested-commit", + "invocationTimeout": 30, + "conformanceRevision": "local-pinned-conformance", + } + cloud.events = {str(index): event for index, event in enumerate(raw)} + cloud.invocations = [ + { + "arn": arn, + "marker": "marker", + "role": "victim", + "payload": payload, + "native_requests": list(roles), + "trace_header": None, + } + ] + cloud.request_roles = roles + cloud.raw_histories = {arn: history} + cloud.execution_results = { + arn: {"Status": result.status.value, "Result": result.result} + } + cloud.control_records = [] + cloud.source_uris = {"memory://actual-sdk-ledger"} + time.sleep(0.002) + cloud.observation_ended_at = time.time() + cloud.retired_at = cloud.observation_ended_at + cloud.artifacts_dir = Path(tmp_path) + evidence = build_evidence(context, cloud, phase="final") + return evidence, cloud + + +@pytest.mark.parametrize( + "requirement_id,scenario,fixture_name,outcome", + [ + ("lmi-lifecycle-2", "normal-return", "success", "SUCCEEDED"), + ("lmi-lifecycle-7", "normal-failure", "failure", "FAILED"), + ], +) +def test_actual_disposal_outcome_and_late_admission_findings_reach_shared_validation( + monkeypatch, tmp_path, requirement_id, scenario, fixture_name, outcome +): + ledger = install_ledger(monkeypatch) + payload = shared_payload(fixture_name) + payload["sharedScenario"] = scenario + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(payload), execution_timeout=15) + arn = next( + e["execution"] for e in ledger.records() if e["phase"] == "WRAPPER_ENTER" + ) + history = runner.get_execution_history( + arn, include_execution_data=True + ).to_dict()["Events"] + assert result.status.value == outcome + evidence, _ = assemble( + ledger, result, history, payload, tmp_path, requirement_id, scenario + ) + victim = evidence["selectedRequests"]["victim"] + original = next( + item for item in evidence["attempts"] if item["requestId"] == victim + ) + assert original["outcome"] == outcome + probes = [e for e in evidence["events"] if e["kind"] == "admission_probe"] + assert any(e["probe"] == "checkpoint" for e in probes) + errors = validate(evidence) + if len(probes) == 2 and all(e["outcome"] == "REJECTED" for e in probes): + assert errors == [] + else: + # Test the harness, not an assumed SDK fix: actual acceptance or missing + # completion must remain a failed shared result, never fabricated rejection. + assert errors + assert any(e["outcome"] != "REJECTED" for e in probes) or len(probes) < 2 diff --git a/lmi_tests/tests/e2e/test_shared_observations_local.py b/lmi_tests/tests/e2e/test_shared_observations_local.py new file mode 100644 index 000000000..956aafc60 --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_observations_local.py @@ -0,0 +1,123 @@ +"""Run the actual Lambda fixture through the public in-process SDK service.""" + +from __future__ import annotations + +import json +from collections import Counter + +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def values(ledger, kind): + return [ + record["observation"] + for record in ledger.observations() + if record["observation"]["kind"] == kind + ] + + +def test_actual_fixture_public_hooks_bind_bodies_checkpoints_and_root_lifetime( + monkeypatch, +): + ledger = install_ledger(monkeypatch) + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(shared_payload()), execution_timeout=30) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + body = values(ledger, "body") + assert ( + len(body) == 1 + and body[0]["operationId"] == result.get_step("success").operation_id + ) + assert body[0]["parentId"] == result.get_step("success").parent_id + assert body[0]["operationId"] in { + operation + for ack in values(ledger, "checkpoint_ack") + for operation in ack["operationIds"] + } + assert {item["callId"] for item in values(ledger, "checkpoint_call")} == { + item["callId"] for item in values(ledger, "checkpoint_exit") + } + assert {item["taskId"] for item in values(ledger, "task_start")} == { + item["taskId"] for item in values(ledger, "task_exit") + } + assert ( + next(item for item in values(ledger, "task_exit") if item["taskId"] == "root")[ + "atMs" + ] + <= values(ledger, "wrapper_return")[0]["atMs"] + ) + + +def test_actual_service_replay_keeps_operation_identity_and_skips_completed_bodies( + monkeypatch, +): + ledger = install_ledger(monkeypatch) + payload = shared_payload("replay") + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + callback = runner.wait_for_callback(arn, name="completion", timeout=20) + runner.send_callback_success(callback, result=b"marker") + result = runner.wait_for_result(arn, timeout=20) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + bodies = values(ledger, "body") + assert Counter(item["operation"] for item in bodies) == { + "success": 1, + "failure": 1, + "retry": 2, + } + assert { + item["operationId"] for item in bodies if item["operation"] == "success" + } == {result.get_step("success").operation_id} + starts = values(ledger, "invocation_start") + assert len(starts) >= 2 and any(not item["isFirstInvocation"] for item in starts) + replay = next(item for item in starts if not item["isFirstInvocation"]) + assert any( + item["operationId"] == result.get_step("success").operation_id + and item["status"] == "SUCCEEDED" + for item in replay["operations"] + ) + assert "PENDING" in {item["outcome"] for item in values(ledger, "wrapper_return")} + observations = ledger.observations() + for record in observations: + if record["observation"]["kind"] != "pending": + continue + wrapper = next( + item["observation"] + for item in observations + if item["request"] == record["request"] + and item["observation"]["kind"] == "wrapper_return" + ) + assert record["observation"]["atMs"] == wrapper["atMs"] + + +def test_warm_invocation_observes_late_work_under_its_original_request(monkeypatch): + ledger = install_ledger(monkeypatch) + owner = shared_payload("warm-owner") + owner["sharedScenario"] = "warm-reuse" + warm = {**owner, "scenario": "warm-probe", "role": "warm", "marker": "warm-marker"} + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + first = runner.run(input=json.dumps(owner), execution_timeout=15) + second = runner.run(input=json.dumps(warm), execution_timeout=15) + assert first.result == '"marker"' + assert second.result == '"warm-marker"' + raw = ledger.records() + original = next( + e for e in raw if e["phase"] == "WRAPPER_RETURN" and e["role"] == "victim" + ) + entered = next( + e for e in raw if e["phase"] == "WRAPPER_ENTER" and e["role"] == "warm" + ) + ended = next( + e for e in raw if e["phase"] == "WRAPPER_RETURN" and e["role"] == "warm" + ) + late = next( + e for e in raw if e.get("observation", {}).get("kind") == "late_activity" + ) + assert late["request"] == original["request"] + assert late["execution"] == original["execution"] + assert late["observation"]["observedDuringRequestId"] == entered["request"] + assert original["time"] < entered["time"] <= late["time"] <= ended["time"] diff --git a/lmi_tests/tests/unit/support.py b/lmi_tests/tests/unit/support.py new file mode 100644 index 000000000..5640071e9 --- /dev/null +++ b/lmi_tests/tests/unit/support.py @@ -0,0 +1,80 @@ +"""Assert fixed behavior, not the bug. All blocking fixture work has a final release.""" + +import json +import time +from unittest.mock import Mock + + +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInputWithClient, + InitialExecutionState, +) +from aws_durable_execution_sdk_python.lambda_service import ( + CheckpointOutput, + CheckpointUpdatedExecutionState, + ContextDetails, + ExecutionDetails, + Operation, + OperationStatus, + OperationType, + StepDetails, +) + + +class MemoryClient: + def checkpoint(self, *, updates, **_kwargs): + changed = [ + Operation( + operation_id=u.operation_id, + operation_type=u.operation_type, + status={ + "START": OperationStatus.STARTED, + "SUCCEED": OperationStatus.SUCCEEDED, + "FAIL": OperationStatus.FAILED, + }[u.action.value], + name=u.name, + parent_id=u.parent_id, + sub_type=u.sub_type, + step_details=StepDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.STEP + else None, + context_details=ContextDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.CONTEXT + else None, + ) + for u in updates + ] + return CheckpointOutput( + "next-token", CheckpointUpdatedExecutionState(operations=changed) + ) + + +def invocation(): + return DurableExecutionInvocationInputWithClient( + durable_execution_arn="arn:test:execution/local", + checkpoint_token="token", + initial_execution_state=InitialExecutionState( + operations=[ + Operation( + operation_id="local", + operation_type=OperationType.EXECUTION, + status=OperationStatus.STARTED, + execution_details=ExecutionDetails(input_payload=json.dumps({})), + ) + ], + next_marker="", + ), + service_client=MemoryClient(), + ) + + +def lambda_context(remaining): + context = Mock() + context.aws_request_id = "local-request" + context.client_context = context.identity = context.tenant_id = None + context.invoked_function_arn = "test-function" + context._epoch_deadline_time_in_ms = int(time.time() * 1000) + remaining + context.get_remaining_time_in_millis.side_effect = lambda: ( + context._epoch_deadline_time_in_ms - int(time.time() * 1000) + ) + return context diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py new file mode 100644 index 000000000..1ba8259a4 --- /dev/null +++ b/lmi_tests/tests/unit/test_control.py @@ -0,0 +1,138 @@ +"""Regressions for the real-cloud 403 and premature-loser false positives.""" + +import io +import threading +import time +from types import SimpleNamespace +from unittest.mock import Mock + +import boto3 +from botocore.exceptions import ClientError +from botocore.stub import Stubber +import pytest + +from lmi_tests.fixture import Trace +from lmi_tests import fixture + + +@pytest.fixture +def trace(): + value = Trace.__new__(Trace) + value.capture_lock, value.capture_sequence = threading.Lock(), 0 + value.s3 = boto3.client( + "s3", + region_name="us-west-2", + aws_access_key_id="test", + aws_secret_access_key="test", + ) + value.bucket = "test-bucket" + value.identity = {"run": "unit"} + value.payload = {"scenario": "control"} + value.shared = False + value.control_prefix = "runs/unit/control/" + value.emit = Mock() + return value + + +@pytest.mark.parametrize("state,released", [(b"hold", False), (b"release", True)]) +def test_control_reads_explicit_state_and_closes_body(trace, state, released): + body = io.BytesIO(state) + with Stubber(trace.s3) as stub: + stub.add_response( + "get_object", + {"Body": body}, + {"Bucket": "test-bucket", "Key": "runs/unit/control/gate"}, + ) + assert trace.released("gate") is released + assert body.closed + trace.emit.assert_not_called() + + +@pytest.mark.parametrize("code,status", [("403", 403), ("NoSuchKey", 404)]) +def test_missing_or_forbidden_control_is_not_a_hold_or_release(trace, code, status): + with Stubber(trace.s3) as stub: + stub.add_client_error( + "get_object", service_error_code=code, http_status_code=status + ) + with pytest.raises(ClientError): + trace.gate("gate", effects=True, ready="ready") + phases = [call.args[0] for call in trace.emit.call_args_list] + assert "CONTROL_ERROR" in phases + assert not {"BLOCKED", "ALIVE", "EFFECT"} & set(phases) + + +def test_invalid_control_content_cannot_release_work(trace): + with Stubber(trace.s3) as stub: + stub.add_response("get_object", {"Body": io.BytesIO(b"unexpected")}) + with pytest.raises(ValueError, match="Invalid control state"): + trace.released("gate") + assert trace.emit.call_args.args[0] == "CONTROL_ERROR" + + +@pytest.mark.parametrize("elapsed,escaped", [(90, False), (180, True)]) +def test_fault_gate_outlives_invocation_deadline_but_remains_bounded( + trace, monkeypatch, elapsed, escaped +): + trace.deadline = 60 + times = iter([0, elapsed]) + trace.released = Mock(side_effect=[False, False, True]) + monkeypatch.setattr( + fixture, + "time", + SimpleNamespace( + monotonic=lambda: next(times), + sleep=lambda _: None, + time=time.time, + time_ns=time.time_ns, + ), + ) + if escaped: + with pytest.raises(TimeoutError, match="emergency release"): + trace.gate("fault", effects=True) + else: + trace.gate("fault", effects=True) + phases = [call.args[0] for call in trace.emit.call_args_list] + assert ("ESCAPE" in phases) is escaped + assert ("EFFECT" in phases) is not escaped + + +def test_winner_is_signalled_only_after_loser_actually_enters_held_io(trace): + controls = { + "runs/unit/control/loser": b"hold", + "runs/unit/control/ready": b"hold", + "runs/unit/control/release-all": b"hold", + } + trace.s3 = Mock() + trace.s3.get_object.side_effect = lambda **kw: { + "Body": io.BytesIO(controls[kw["Key"]]) + } + + def signal(**kwargs): + assert ( + kwargs["Key"] == "runs/unit/control/ready" and kwargs["Body"] == b"release" + ) + assert [c.args[0] for c in trace.emit.call_args_list] == ["BLOCKED", "EFFECT"] + controls[kwargs["Key"]] = kwargs["Body"] + controls["runs/unit/control/loser"] = b"release" + + trace.s3.put_object.side_effect = signal + trace.gate("loser", effects=True, ready="ready") + assert [c.args[0] for c in trace.emit.call_args_list] == [ + "BLOCKED", + "EFFECT", + "IO_EXIT", + ] + assert controls["runs/unit/control/release-all"] == b"hold" + + +def test_emitted_records_are_partitioned_by_case_and_request(trace): + trace.s3 = Mock() + trace.identity = {"run": "unit", "marker": "case", "request": "request"} + trace.sequence = 0 + trace.deadline = 10 + trace.lock = threading.Lock() + Trace.emit(trace, "BODY") + assert ( + trace.s3.put_object.call_args.kwargs["Key"] + == "runs/unit/events/case/request/000001.json" + ) diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py new file mode 100644 index 000000000..57d1e8f92 --- /dev/null +++ b/lmi_tests/tests/unit/test_deploy.py @@ -0,0 +1,21 @@ +from pathlib import Path +from unittest.mock import Mock +import pytest +from lmi_tests import deploy + + +def test_failed_build_clears_prior_run_before_subprocess( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + artifacts = tmp_path / "lmi_tests/artifacts" + deploy.save(artifacts / "manifest.json", {"run": "previous", "owned": True}) + deploy.save(artifacts / "cleanup.json", {"run": "previous"}) + monkeypatch.setattr(deploy, "ROOT", tmp_path) + monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) + monkeypatch.setattr( + deploy.subprocess, "run", Mock(side_effect=RuntimeError("build failed")) + ) + with pytest.raises(RuntimeError, match="build failed"): + deploy.build() + assert not (artifacts / "manifest.json").exists() + assert not (artifacts / "cleanup.json").exists() diff --git a/lmi_tests/tests/unit/test_fixture.py b/lmi_tests/tests/unit/test_fixture.py new file mode 100644 index 000000000..fda57fd63 --- /dev/null +++ b/lmi_tests/tests/unit/test_fixture.py @@ -0,0 +1,143 @@ +import json +import subprocess +import sys +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.lambda_service import ( + OperationAction, + OperationType, + OperationUpdate, +) +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests.fixture import ObservedClient, workflow +from lmi_tests import fixture +from lmi_tests.tests.unit.support import ( + MemoryClient, + invocation, + lambda_context, +) + + +class LocalTrace: + def __init__(self): + self.events = [] + self.shared = False + self.payload = {"scenario": "checkpoint", "marker": "local"} + + def emit(self, phase, **fields): + self.events.append({"phase": phase, **fields}) + + def body(self, name, value, step): + self.emit("BODY", operation=name, value=value, attempt=step.attempt) + return value + + def gate(self, name): + self.emit("GATE", name=name) + + +def test_environment_marker_is_shared_across_processes(tmp_path): + path = str(tmp_path / "marker") + workers = [ + subprocess.Popen( + [ + sys.executable, + "-c", + "import sys; from lmi_tests.fixture import environment_id; print(environment_id(sys.argv[1]))", + path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + for _ in range(4) + ] + try: + values = [] + for worker in workers: + stdout, stderr = worker.communicate(timeout=15) + assert worker.returncode == 0, stderr + values.append(stdout.strip()) + finally: + for worker in workers: + if worker.poll() is None: + worker.kill() + worker.wait(timeout=5) + assert len(set(values)) == 1 + assert len(values[0]) == 32 + + +def test_checkpoint_proxy_preserves_real_result_and_errors(): + trace, backend = LocalTrace(), Mock() + observed = ObservedClient(backend, trace) + update = OperationUpdate( + "id", + OperationType.STEP, + OperationAction.SUCCEED, + name="checkpoint-loser", + payload="result", + ) + result = observed.checkpoint( + updates=[update], checkpoint_token="secret", durable_execution_arn="arn" + ) + assert result is backend.checkpoint.return_value + backend.checkpoint.assert_called_once_with( + updates=[update], checkpoint_token="secret", durable_execution_arn="arn" + ) + assert [e["phase"] for e in trace.events] == [ + "CHECKPOINT_CALL", + "CHECKPOINT_ACK", + "CHECKPOINT_EXIT", + ] + assert "secret" not in json.dumps(trace.events) + trace.events.clear() + backend.checkpoint.side_effect = RuntimeError("network failure") + with pytest.raises(RuntimeError, match="network failure"): + observed.checkpoint(updates=[update]) + assert [e["phase"] for e in trace.events] == [ + "CHECKPOINT_CALL", + "CHECKPOINT_ERROR", + "CHECKPOINT_EXIT", + ] + + +def test_replay_fixture_uses_real_public_apis_and_preserves_outcomes(): + trace = LocalTrace() + handler = durable_execution(lambda e, c: workflow(e, c, trace)) + with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: + arn = runner.run_async(input={"marker": "local", "scenario": "replay"}) + callback = runner.wait_for_callback(arn, name="completion", timeout=10) + runner.send_callback_success(callback, result=b"local") + result = runner.wait_for_result(arn, timeout=10) + assert result.status.value == "SUCCEEDED" + assert json.loads(result.result) == "local" + bodies = [e for e in trace.events if e["phase"] == "BODY"] + assert sum(e["operation"] == "success" for e in bodies) == 1 + assert sum(e["operation"] == "failure" for e in bodies) == 1 + assert [e["attempt"] for e in bodies if e["operation"] == "retry"] == [1, 2] + failures = [e for e in trace.events if e["phase"] == "STORED_FAILURE"] + assert len(failures) >= 2 + assert all(e["message"] == "expected:local" for e in failures) + + +def test_cloud_entrypoint_unwraps_runtime_event_and_observes_real_wrapper(monkeypatch): + trace = LocalTrace() + trace.payload["scenario"] = "success" + monkeypatch.setenv("LMI_RUN_ID", "unit") + monkeypatch.setattr(fixture, "Trace", lambda *_: trace) + monkeypatch.setattr( + fixture.LambdaClient, "initialize_client", lambda: MemoryClient() + ) + event = invocation().to_json_dict() + event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps({"run": "unit", "marker": "local", "scenario": "success"}) + result = fixture.handler(event, lambda_context(30000)) + assert result["Status"] == "SUCCEEDED" + assert json.loads(result["Result"]) == "local" + phases = [e["phase"] for e in trace.events] + assert phases[0] == "WRAPPER_ENTER" and phases[-1] == "WRAPPER_RETURN" + assert phases.index("USER_EXIT") < phases.index("WRAPPER_RETURN") + assert "CHECKPOINT_ACK" in phases diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py new file mode 100644 index 000000000..a9b562670 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -0,0 +1,133 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Plans pin the shared matrix and immutable SDK/deployment inputs.""" + +import json +from unittest.mock import patch + +import pytest + +from aws_durable_execution_conformance_tests.lmi.app import load_manifest +from lmi_tests.shared import make_manifest, verify_checkout +from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity + + +def deployment(): + return { + "run": "run-1", + "stack": "stack", + "region": "us-west-2", + "account": "123456789012", + "runtime": "python3.14", + "role": "arn:role", + "bucket": "private", + "commit": "sdk-commit", + "provider": "arn:aws:lambda:us-west-2:123456789012:capacity-provider:provider", + "codeSha256": "bundle-sha", + "invocationTimeout": 60, + "executionTimeout": 240, + "driverTimeout": 120, + "concurrencies": {"c1": 1, "c2": 2}, + "functions": { + "c1": "arn:function:c1:$LATEST.PUBLISHED", + "c2": "arn:function:c2:$LATEST.PUBLISHED", + }, + "wheels": { + "aws_durable_execution_sdk_python-3.0.0.whl": "core-sha", + }, + } + + +def test_generated_manifest_covers_the_seven_shared_scenarios(tmp_path): + from aws_durable_execution_conformance_tests.lmi.app import coverage_inventory + + source = deployment() + plan = make_manifest(source, "standard", tmp_path) + assert len(plan["scenarios"]) == 7 + assert "lease" not in plan["sdk"] + assert "sharedLease" not in plan["sdk"]["deployment"] + assert {c["requirement_id"] for c in plan["scenarios"]} == { + f"lmi-lifecycle-{i}" for i in range(1, 8) + } + assert {c["profile_id"] for c in plan["scenarios"]} == {"c1", "c2"} + assert all( + "otel_view" not in c and "export_mode" not in c for c in plan["scenarios"] + ) + assert plan["sdk"]["deployment"]["conformanceRevision"] == CONFORMANCE_REVISION + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + assert load_manifest(path) == plan + assert coverage_inventory(plan)["missing"] == [] + assert ( + next(c for c in plan["scenarios"] if c["scenario"] == "normal-failure")[ + "requirement_id" + ] + == "lmi-lifecycle-7" + ) + plan["sdk"]["deployment"]["commit"] = "mutated" + assert source["commit"] == "sdk-commit" + + +def test_no_unidentified_artifact_can_be_planned(tmp_path): + source = deployment() + source["wheels"] = {} + with pytest.raises(ValueError, match="wheel"): + make_manifest(source, "standard", tmp_path) + + +def test_configuration_identity_changes_with_the_tested_settings(): + source = deployment() + original = configuration_identity(source, "c1") + assert original != configuration_identity(source, "c2") + source["codeSha256"] = "other-bundle" + assert original != configuration_identity(source, "c1") + + +def test_only_the_exact_clean_shared_revision_is_accepted(tmp_path): + with patch("lmi_tests.shared.subprocess.check_output", side_effect=["wrong\n"]): + with pytest.raises(ValueError, match="pinned"): + verify_checkout(tmp_path) + with patch( + "lmi_tests.shared.subprocess.check_output", + side_effect=[CONFORMANCE_REVISION + "\n", " M packages/validator.py\n"], + ): + with pytest.raises(ValueError, match="local changes"): + verify_checkout(tmp_path) + with patch( + "lmi_tests.shared.subprocess.check_output", + side_effect=[CONFORMANCE_REVISION + "\n", ""], + ): + verify_checkout(tmp_path) + + +def test_shared_run_writes_reports_without_an_external_lock(monkeypatch, tmp_path): + from unittest.mock import Mock + from lmi_tests import shared + from aws_durable_execution_conformance_tests.lmi import app + + async def results(plan, adapter_name, *, audit_dir): + assert adapter_name == "lmi_tests.shared_adapter:create_adapter" + assert "lease" not in plan["sdk"] + return [ + { + "case_id": case["case_id"], + "requirement_id": case["requirement_id"], + "profile_id": case["profile_id"], + "status": "PASSED", + "errors": [], + } + for case in plan["scenarios"] + ] + + client = Mock( + side_effect=AssertionError("Run orchestration must not acquire an AWS lock") + ) + from aws_durable_execution_conformance_tests.lmi import runner + + monkeypatch.setattr(runner, "client", client) + monkeypatch.setattr(app, "live_results", results) + output = tmp_path / "run" + assert shared.run(deployment(), output) == 0 + report = json.loads((output / "standard/report.json").read_text()) + assert report["summary"]["passed"] == 7 + client.assert_not_called() diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py new file mode 100644 index 000000000..e1335c580 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -0,0 +1,464 @@ +"""Public hook records preserve actual IDs and never fill missing observations.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from dataclasses import asdict +from itertools import count +import io +import json +import threading +import time +from typing import Any +from types import SimpleNamespace + +import pytest + +from aws_durable_execution_sdk_python.lambda_service import ErrorObject, OperationStatus +from aws_durable_execution_sdk_python.plugin import ( + InvocationStartInfo, + OperationEndInfo, + OperationInfo, + OperationStartInfo, + OperationType, + UserFunctionEndInfo, + UserFunctionStartInfo, +) +from lmi_tests import fixture +from aws_durable_execution_sdk_python_testing import invoker as local_invoker +from lmi_tests.shared_observations import ( + LifecycleObservationFactory, + LifecycleObservations, + OperationObservation, +) +from lmi_tests.tests.unit.support import invocation, lambda_context + + +class MemoryLedger: + """Replace only evidence/control storage; the SDK local service stays real.""" + + def __init__(self) -> None: + self.objects: dict[str, bytes] = {} + self.lock = threading.Lock() + + def put_object(self, *, Bucket, Key, Body, **_kwargs): + with self.lock: + self.objects[Key] = Body + return {} + + def get_object(self, *, Bucket, Key): + with self.lock: + return {"Body": io.BytesIO(self.objects[Key])} + + def records(self) -> list[dict[str, Any]]: + with self.lock: + records = [ + json.loads(body) + for key, body in self.objects.items() + if "/events/" in key + ] + return sorted( + records, key=lambda record: (record["request"], record["sequence"]) + ) + + def observations(self) -> list[dict[str, Any]]: + return [record for record in self.records() if record["phase"] == "CONFORMANCE"] + + def wait_for(self, kind: str, timeout: float = 10) -> dict[str, Any]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + found = [ + record + for record in self.observations() + if record["observation"]["kind"] == kind + ] + if found: + return found[-1] + time.sleep(0.01) + raise AssertionError(f"No actual {kind} observation") + + +class MemorySink: + def __init__(self) -> None: + self.records: list[dict[str, Any]] = [] + self.lock = threading.Lock() + + def observe(self, kind: str, **fields: Any) -> dict[str, Any]: + value = {"kind": kind, **fields} + with self.lock: + self.records.append(value) + return value + + +def shared_payload(scenario: str = "success") -> dict[str, Any]: + return { + "runId": "unit", + "caseId": "case", + "role": "victim", + "coverageProfile": "standard", + "sharedScenario": scenario, + "scenario": scenario, + "marker": "marker", + "controlPrefix": "runs/unit/cases/owner/control/", + "behaviorBudgets": {"observationGraceMs": 1}, + } + + +def install_ledger(monkeypatch) -> MemoryLedger: + ledger = MemoryLedger() + monkeypatch.setenv("LMI_RUN_ID", "unit") + monkeypatch.setenv("LMI_COMMIT", "tested-commit") + monkeypatch.setenv("LMI_BUCKET", "private-evidence") + monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) + monkeypatch.setattr(fixture.boto3, "client", lambda *_args, **_kwargs: ledger) + monkeypatch.setattr(fixture, "environment_id", lambda: "local-environment") + original_context = local_invoker.create_test_lambda_context + requests = count(1) + + def distinct_local_context(**kwargs): + # The stock local service reuses one mock request ID. Supply its runtime + # context before invoking the real fixture, never alter captured evidence. + context = original_context(**kwargs) + return type(context)( + **{**asdict(context), "aws_request_id": f"local-request-{next(requests)}"} + ) + + monkeypatch.setattr( + local_invoker, "create_test_lambda_context", distinct_local_context + ) + return ledger + + +def user_info( + operation_id: str, + *, + parent_id: str | None = None, + name: str = "work", + kind: OperationType = OperationType.STEP, +) -> UserFunctionStartInfo: + return UserFunctionStartInfo( + operation_id=operation_id, + operation_type=kind, + sub_type=None, + name=name, + parent_id=parent_id, + start_time=datetime.now(UTC), + is_replayed=False, + status=OperationStatus.STARTED, + attempt=1, + ) + + +def test_missing_body_identity_stays_missing_instead_of_being_guessed() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.body("work", "marker", 1) + assert [item["kind"] for item in sink.records] == ["observation_error"] + assert "operationId" not in sink.records[0] + + +@pytest.mark.parametrize("wrong", ["name", "attempt"]) +def test_body_requires_matching_public_step_hook(wrong: str) -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.on_user_function_start(user_info("actual-id")) + observer.body( + "other" if wrong == "name" else "work", "marker", 2 if wrong == "attempt" else 1 + ) + assert sink.records[-1]["kind"] == "observation_error" + assert not any(item["kind"] == "body" for item in sink.records) + + +def test_concurrent_public_user_scopes_keep_their_actual_operation_ids() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + barrier = threading.Barrier(2) + + def body(identity: str) -> None: + info = user_info(identity, parent_id="actual-parent") + observer.on_user_function_start(info) + barrier.wait(timeout=3) + observer.body("work", identity, 1) + observer.on_user_function_end(UserFunctionEndInfo.from_start_info(info, None)) + assert observer.current() is None + + with ThreadPoolExecutor(2) as executor: + list(executor.map(body, ["first-id", "second-id"])) + bodies = [item for item in sink.records if item["kind"] == "body"] + assert {(item["operationId"], item["value"]) for item in bodies} == { + ("first-id", "first-id"), + ("second-id", "second-id"), + } + assert { + item["taskId"] for item in sink.records if item["kind"] == "task_start" + } == {item["taskId"] for item in sink.records if item["kind"] == "task_exit"} + + +def test_closed_ancestor_comes_from_the_real_parent_end_hook() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + parent = OperationEndInfo( + operation_id="parent-id", + operation_type=OperationType.CONTEXT, + sub_type="Parallel", + name="race", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.SUCCEEDED, + ) + observer.on_operation_end(parent) + observer.on_user_function_start( + user_info("child-id", parent_id="parent-id", kind=OperationType.CONTEXT) + ) + assert observer.closed_ancestor() == "parent-id" + assert sink.records[1] == { + "kind": "scope_closed", + "scopeId": "parent-id", + "parentId": None, + } + + +def test_missing_parent_completion_and_unmatched_task_end_are_visible() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + assert observer.closed_ancestor() is None + info = user_info("child-id", parent_id="not-observed", kind=OperationType.CONTEXT) + observer.on_user_function_start(info) + assert observer.closed_ancestor() is None + observer.on_user_function_end( + UserFunctionEndInfo.from_start_info(user_info("wrong-id"), None) + ) + assert sum(item["kind"] == "observation_error" for item in sink.records) == 3 + + +def test_configuration_uses_hook_identity_and_reports_a_missing_hook() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.on_user_function_start(user_info("root", kind=OperationType.EXECUTION)) + with observer.configuration("map", 1): + observer.on_operation_start( + OperationStartInfo( + operation_id="actual-map", + operation_type=OperationType.CONTEXT, + sub_type="Map", + name="map", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.STARTED, + ) + ) + assert ( + next(item for item in sink.records if item["kind"] == "operation_config")[ + "operationId" + ] + == "actual-map" + ) + with observer.configuration("missing-map", 1): + pass + assert sink.records[-1]["kind"] == "observation_error" + + +def test_checkpoint_snapshot_does_not_claim_reexecution_or_drop_stored_errors() -> None: + original = OperationInfo( + operation_id="stored-id", + operation_type=OperationType.STEP, + sub_type="Step", + name="failure", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.FAILED, + error=ErrorObject( + type="ValueError", message="stored failure", data=None, stack_trace=None + ), + ) + record = OperationObservation.from_info(original).to_record() + assert record["error"] == {"type": "ValueError", "message": "stored failure"} + assert record["startedAtMs"] is None and record["isReplayed"] is False + sink = MemorySink() + observer = LifecycleObservations(sink) + info = InvocationStartInfo( + request_id="actual-request", + execution_arn="actual-execution", + is_first_invocation=False, + operations={"stored-id": original}, + ) + assert LifecycleObservationFactory(observer).create_plugin(info) is observer + observer.on_invocation_start(info) + assert sink.records[0]["operations"] == [record] + assert not any(item["kind"] == "body" for item in sink.records) + + +def test_trace_records_immutable_case_identity_and_actual_source_sequence( + monkeypatch, +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + trace = fixture.Trace(payload, lambda_context(30_000), "actual-execution") + observed_at = time.time() * 1000 - 10 + record = trace.observe("boundary", atMs=observed_at) + assert record["observation"] == { + "kind": "boundary", + "atMs": observed_at, + "sourceSequence": record["sequence"], + "captureSequence": 1, + } + assert record["caseId"] == "case" + assert trace.payload["run"] == "unit" + assert ledger.observations() == [record] + + +@pytest.mark.parametrize("field", ["caseId", "role", "controlPrefix"]) +def test_shared_case_cannot_silently_drop_ownership_metadata( + monkeypatch, field: str +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + del payload[field] + with pytest.raises(ValueError, match=field): + fixture.Trace(payload, lambda_context(30_000), "execution") + assert ledger.records() == [] + + +def test_shared_case_cannot_fall_back_to_global_controls(monkeypatch) -> None: + install_ledger(monkeypatch) + payload = shared_payload() + payload["controlPrefix"] = "runs/unit/control/" + with pytest.raises(ValueError, match="case-owned"): + fixture.Trace(payload, lambda_context(30_000), "execution") + + +@pytest.mark.parametrize("mode", ["normal", "emergency"]) +def test_gate_release_reports_observed_provenance_and_keeps_controls_case_owned( + monkeypatch, mode: str +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + trace = fixture.Trace(payload, lambda_context(30_000), "execution") + prefix = payload["controlPrefix"] + ledger.objects[prefix + "gate"] = b"release" if mode == "normal" else b"hold" + ledger.objects[prefix + "release-all"] = b"hold" if mode == "normal" else b"release" + ledger.objects["runs/unit/cases/other/control/gate"] = b"hold" + released = trace.gate_release("gate") + assert released.mode == mode + assert ledger.objects["runs/unit/cases/other/control/gate"] == b"hold" + observations = [item["observation"] for item in ledger.observations()] + assert [item["kind"] for item in observations] == [ + "io_start", + "fault_released", + "io_exit", + ] + assert observations[0]["taskId"] == observations[-1]["taskId"] + assert observations[1]["atMs"] == released.observed_at_ms + + +def test_root_scope_requires_the_public_invocation_operation() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + with observer.root_scope(): + pass + assert [item["kind"] for item in sink.records] == ["observation_error"] + assert observer.completed_context("unknown") is None + assert sink.records[-1]["kind"] == "observation_error" + + +def test_gate_preserves_the_existing_checkpointed_none_result(monkeypatch) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + ledger.objects[payload["controlPrefix"] + "gate"] = b"release" + ledger.objects[payload["controlPrefix"] + "release-all"] = b"hold" + trace = fixture.Trace(payload, lambda_context(30_000), "execution") + assert trace.gate("gate") is None + + +@pytest.mark.parametrize("method", ["checkpoint", "get_execution_state"]) +def test_failed_real_client_call_keeps_its_dispatch_and_exit_identity( + monkeypatch, method: str +) -> None: + from unittest.mock import Mock + + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30_000), "execution") + backend = Mock() + getattr(backend, method).side_effect = OSError("service unavailable") + client = fixture.ObservedClient(backend, trace) + with pytest.raises(OSError, match="service unavailable"): + getattr(client, method)(**({"updates": []} if method == "checkpoint" else {})) + observations = [record["observation"] for record in ledger.observations()] + prefix = "checkpoint" if method == "checkpoint" else "poll" + entered = next(item for item in observations if item["kind"] == prefix + "_call") + exited = next(item for item in observations if item["kind"] == prefix + "_exit") + assert entered["callId"] == exited["callId"] and entered["atMs"] <= exited["atMs"] + assert not any(item["kind"] == "checkpoint_ack" for item in observations) + + +@pytest.mark.parametrize("completed_ns", [1794607984811568141, 1794607984811569459]) +def test_pending_and_wrapper_return_share_one_captured_timestamp( + monkeypatch, completed_ns +): + ledger = install_ledger(monkeypatch) + original = invocation() + payload = original.to_json_dict() + payload["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps(shared_payload()) + injected = fixture.DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( + fixture.DurableExecutionInvocationInput.from_json_dict(payload), + original.service_client, + ) + monkeypatch.setattr( + fixture, "time", SimpleNamespace(time=time.time, time_ns=lambda: completed_ns) + ) + monkeypatch.setattr( + fixture, + "durable_execution", + lambda *args, **kwargs: lambda *args: {"Status": "PENDING"}, + ) + assert fixture.handler(injected, lambda_context(30000)) == {"Status": "PENDING"} + observations = [record["observation"] for record in ledger.observations()] + wrapper = next(item for item in observations if item["kind"] == "wrapper_return") + pending = next(item for item in observations if item["kind"] == "pending") + assert pending["atMs"] == wrapper["atMs"] == completed_ns / 1_000_000 + + +@pytest.mark.parametrize("kind", ["checkpoint", "background", "suspend"]) +@pytest.mark.parametrize("effect", ["none", "body", "rpc"]) +def test_stored_sdk_errors_are_rejections_only_before_new_work( + monkeypatch, kind, effect +): + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") + errors = { + "checkpoint": lambda: fixture.CheckpointError("Invalid checkpoint token"), + "background": lambda: fixture.BackgroundThreadError( + "failed", ValueError("source") + ), + "suspend": lambda: fixture.SuspendExecution("revoked"), + } + + def checkpoint(**options): + if effect == "rpc": + trace.next_call_id("checkpoint") + raise errors[kind]() + + def step(body, **options): + if effect == "body": + body(None) + if effect == "rpc": + trace.next_call_id("checkpoint") + raise errors[kind]() + + trace.durable_context = SimpleNamespace( + state=SimpleNamespace(create_checkpoint=checkpoint), step=step + ) + fixture.admission_probes(trace) + probes = [ + r["observation"] + for r in ledger.observations() + if r["observation"]["kind"] == "admission_probe" + ] + assert probes[0]["outcome"] == ("ERROR" if effect == "rpc" else "REJECTED") + assert probes[1]["outcome"] == ("REJECTED" if effect == "none" else "ERROR") diff --git a/lmi_tests/tests/unit/test_shared_workflow.py b/lmi_tests/tests/unit/test_shared_workflow.py new file mode 100644 index 000000000..221d51701 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_workflow.py @@ -0,0 +1,106 @@ +from pathlib import Path +import yaml +from lmi_tests.shared_contract import CONFORMANCE_REVISION + +ROOT = Path(__file__).resolve().parents[3] + + +def test_shared_setup_bootstraps_before_conformance_is_installed(): + import subprocess + import sys + import textwrap + + script = textwrap.dedent(""" + import importlib.abc + import sys + from pathlib import Path + from unittest.mock import patch + + class NoConformance(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith('aws_durable_execution_conformance_tests'): + raise AssertionError('bootstrap tried to import uninstalled conformance') + return None + + sys.meta_path.insert(0, NoConformance()) + from lmi_tests import shared + with patch.object(shared, 'verify_checkout') as verify, patch.object(shared.subprocess, 'run') as install: + shared.setup(Path('.conformance')) + verify.assert_called_once_with(Path('.conformance')) + assert install.call_args.args[0][1:4] == ['-m', 'pip', 'install'] + """) + completed = subprocess.run( + [sys.executable, "-c", script], + cwd=ROOT, + capture_output=True, + text=True, + timeout=20, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_workflow_calls_the_pinned_shared_definition(): + workflow = yaml.safe_load( + (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() + ) + caller = workflow["jobs"]["lifecycle"] + assert list(workflow["jobs"]) == ["lifecycle"] + assert ( + caller["uses"] + == f"aws/aws-durable-execution-conformance-tests/.github/workflows/lmi-lifecycle.yml@{CONFORMANCE_REVISION}" + ) + assert caller["with"]["conformance-ref"] == CONFORMANCE_REVISION + assert (ROOT / caller["with"]["sdk-script"]).is_file() + assert set(caller["secrets"]) == { + "TEST_ROLE_ARN", + "TEST_ACCOUNT_ID", + "TEST_LAMBDA_EXECUTION_ROLE_ARN", + "CAPACITY_PROVIDER_ARN", + } + triggers = workflow.get("on", workflow.get(True)) + assert set(triggers["pull_request"]["types"]) == { + "opened", + "synchronize", + "reopened", + } + assert "workflow_dispatch" in triggers + + +def test_stage_dispatch_propagates_failure_and_preserves_deployment_identity(tmp_path): + import os + import subprocess + + fake = tmp_path / "hatch" + fake.write_text( + '#!/bin/bash\nprintf "%s\\n" "$@" > "$CALL_LOG"\nexit "${HATCH_RESULT:-0}"\n' + ) + fake.chmod(0o755) + calls = tmp_path / "calls.txt" + env = { + **os.environ, + "PATH": str(tmp_path) + ":" + os.environ["PATH"], + "CALL_LOG": str(calls), + "LMI_SHARED_RUN_ID": "123-2", + "LMI_RUNTIME": "python3.14", + "LMI_STACK_NAME": "python-lmi-e2e", + } + script = ROOT / ".github/scripts/lmi-ci.sh" + subprocess.run(["bash", str(script), "deploy"], env=env, check=True) + assert calls.read_text().splitlines() == [ + "run", + "lmi:python", + "-m", + "lmi_tests.deploy", + "deploy", + "--run-id", + "123-2", + "--runtime", + "python3.14", + "--stack-name", + "python-lmi-e2e", + ] + failed = subprocess.run( + ["bash", str(script), "run"], env={**env, "HATCH_RESULT": "17"}, check=False + ) + assert failed.returncode == 17 + assert calls.read_text().splitlines() == ["run", "lmi:shared"] diff --git a/pyproject.toml b/pyproject.toml index 2011407cb..a9ce7947b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,8 @@ version = "0.0.0" requires-python = ">=3.11" [tool.hatch.envs.default] +# The root coordinates workspace packages; it is not an installable distribution. +skip-install = true workspace.members = ["packages/*"] [tool.hatch.envs.test] @@ -25,6 +27,24 @@ all = "pytest {args}" cov = "pytest --cov-report=term-missing --cov {args}" examples-integration = "pytest --runner-mode=cloud -m example packages/aws-durable-execution-sdk-python-examples/test/ -v {args}" +[tool.hatch.envs.lmi] +detached = true +python = "3.14" +workspace.members = [ + "packages/aws-durable-execution-sdk-python", + "packages/aws-durable-execution-sdk-python-testing", +] +dependencies = ["boto3>=1.43.0", "pytest", "ruff", "build", "mypy"] + +[tool.hatch.envs.lmi.scripts] +unit = "pytest lmi_tests/tests/unit {args}" +lint = "ruff check lmi_tests && ruff format --check lmi_tests" +build = "python -m lmi_tests.deploy build {args}" +shared-setup = "python -m lmi_tests.shared setup {args}" +shared = "python -m lmi_tests.shared run {args}" +shared-cleanup = "python -m lmi_tests.shared cleanup {args}" +shared-tests = "python -m pytest lmi_tests/tests/unit lmi_tests/tests/e2e {args}" + [tool.pytest.ini_options] # Declare custom markers to avoid warnings with --strict-markers addopts = "-v --strict-markers --import-mode=importlib"