From 384529ae62b893f04ae98a058c56cf3c1951c654 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:23:10 +0000 Subject: [PATCH 01/32] test: add LMI cloud lifecycle regressions Add opt-in cloud deployment and lifecycle coverage for Python LMI. Assert invocation deadlines, early branch cleanup, replay, and worker recovery; keep issue 741 regressions red until the SDK fix. --- .github/workflows/lmi-e2e-tests.yml | 121 ++++ lmi_tests/.gitignore | 3 + lmi_tests/README.md | 179 ++++++ lmi_tests/__init__.py | 0 lmi_tests/cloud.py | 220 ++++++++ lmi_tests/deploy.py | 559 +++++++++++++++++++ lmi_tests/evidence.py | 192 +++++++ lmi_tests/fixture.py | 348 ++++++++++++ lmi_tests/summary.py | 58 ++ lmi_tests/tests/e2e/conftest.py | 53 ++ lmi_tests/tests/e2e/test_lifecycle.py | 173 ++++++ lmi_tests/tests/regressions/conftest.py | 11 + lmi_tests/tests/regressions/test_issue741.py | 168 ++++++ lmi_tests/tests/unit/test_cloud.py | 108 ++++ lmi_tests/tests/unit/test_deploy.py | 172 ++++++ lmi_tests/tests/unit/test_evidence.py | 191 +++++++ lmi_tests/tests/unit/test_fixture.py | 155 +++++ pyproject.toml | 15 + 18 files changed, 2726 insertions(+) create mode 100644 .github/workflows/lmi-e2e-tests.yml create mode 100644 lmi_tests/.gitignore create mode 100644 lmi_tests/README.md create mode 100644 lmi_tests/__init__.py create mode 100644 lmi_tests/cloud.py create mode 100644 lmi_tests/deploy.py create mode 100644 lmi_tests/evidence.py create mode 100644 lmi_tests/fixture.py create mode 100644 lmi_tests/summary.py create mode 100644 lmi_tests/tests/e2e/conftest.py create mode 100644 lmi_tests/tests/e2e/test_lifecycle.py create mode 100644 lmi_tests/tests/regressions/conftest.py create mode 100644 lmi_tests/tests/regressions/test_issue741.py create mode 100644 lmi_tests/tests/unit/test_cloud.py create mode 100644 lmi_tests/tests/unit/test_deploy.py create mode 100644 lmi_tests/tests/unit/test_evidence.py create mode 100644 lmi_tests/tests/unit/test_fixture.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml new file mode 100644 index 000000000..50d4ee1ca --- /dev/null +++ b/.github/workflows/lmi-e2e-tests.yml @@ -0,0 +1,121 @@ +name: LMI e2e tests + +on: + push: + branches: [main] + pull_request: + types: [opened, synchronize, reopened, labeled] + workflow_dispatch: + inputs: + region: + description: Region containing the test LMI capacity provider + default: us-west-2 + type: string + +permissions: + contents: read + +jobs: + harness: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.13' + - run: pip install hatch==1.16.5 + - run: hatch run lmi:lint + - run: hatch run lmi:unit --junitxml=lmi_tests/artifacts/unit.xml + - uses: actions/upload-artifact@v7 + if: always() + with: + name: lmi-harness-tests + path: lmi_tests/artifacts/ + + cloud: + needs: harness + # Start with manual runs and explicit trusted PR opt-in; no scheduled cloud job. + if: >- + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && + github.actor != 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository && + contains(github.event.pull_request.labels.*.name, 'run-lmi-e2e')) + runs-on: ubuntu-latest + timeout-minutes: 55 + permissions: + contents: read + id-token: write + strategy: + fail-fast: false + max-parallel: 1 + matrix: + python: ['3.13', '3.14'] + concurrency: [1, 2] + concurrency: + group: python-lmi-e2e-${{ matrix.python }}-${{ matrix.concurrency }} + cancel-in-progress: false + env: + AWS_REGION: ${{ inputs.region || 'us-west-2' }} + CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} + TEST_ACCOUNT_ID: ${{ secrets.TEST_ACCOUNT_ID }} + TEST_LAMBDA_EXECUTION_ROLE_ARN: ${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python }} + - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ secrets.TEST_ROLE_ARN }} + role-session-name: python-lmi-e2e + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} + - name: Verify test account and prerequisites + run: | + test -n "$CAPACITY_PROVIDER_ARN" + test -n "$TEST_LAMBDA_EXECUTION_ROLE_ARN" + test "$(aws sts get-caller-identity --query Account --output text)" = "$TEST_ACCOUNT_ID" + - run: pip install hatch==1.16.5 + - name: Build SDK checkout and reconcile expired test stacks + timeout-minutes: 5 + run: | + hatch run lmi:build + hatch run lmi:python -m lmi_tests.deploy reconcile + - name: Deploy verified LMI functions + timeout-minutes: 28 + env: + LMI_RUNTIME: python${{ matrix.python }} + LMI_CONCURRENCY: ${{ matrix.concurrency }} + run: | + hatch run lmi:python -m lmi_tests.deploy deploy \ + --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ + --runtime "$LMI_RUNTIME" --concurrency "$LMI_CONCURRENCY" + - name: Run cloud regressions (red until issue 741 is fixed) + timeout-minutes: 12 + run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml + - name: Collect service history and runtime evidence + if: always() + timeout-minutes: 3 + run: | + if test -f lmi_tests/artifacts/manifest.json; then + hatch run lmi:python -m lmi_tests.deploy collect + fi + - name: Release blocked work and retire run-owned resources + if: always() + timeout-minutes: 5 + run: | + if test -f lmi_tests/artifacts/manifest.json; then + hatch run lmi:python -m lmi_tests.deploy cleanup + fi + - name: Retain diagnostics + if: always() + uses: actions/upload-artifact@v7 + with: + name: lmi-${{ matrix.python }}-c${{ matrix.concurrency }}-${{ github.run_id }}-${{ github.run_attempt }} + retention-days: 7 + path: lmi_tests/artifacts/ + - name: Summarize outcomes + if: always() + run: python -m lmi_tests.summary >> "$GITHUB_STEP_SUMMARY" diff --git a/lmi_tests/.gitignore b/lmi_tests/.gitignore new file mode 100644 index 000000000..0d8880bee --- /dev/null +++ b/lmi_tests/.gitignore @@ -0,0 +1,3 @@ +artifacts/ +build/ +__pycache__/ diff --git a/lmi_tests/README.md b/lmi_tests/README.md new file mode 100644 index 000000000..b072ac9c3 --- /dev/null +++ b/lmi_tests/README.md @@ -0,0 +1,179 @@ +# Python LMI end-to-end tests + +This suite implements [#742](https://github.com/aws/aws-durable-execution-sdk-python/issues/742) +and asserts the invocation-lifetime behavior requested by +[#741](https://github.com/aws/aws-durable-execution-sdk-python/issues/741). +It follows the separate cloud-suite approach in +[Java PR #728](https://github.com/aws/aws-durable-execution-sdk-java/pull/728). +It changes no production SDK code. The deadline and early-completion regressions +are **expected to fail on the current SDK**. They are ordinary failing assertions, +with no `xfail`, swallowed failure, or expected-failure success status. Cloud tests +and local red regressions require explicit opt-in; the ordinary test suite remains +usable while the fix is designed. + +## Run locally + +Use Python 3.13 or 3.14 and Hatch from the repository root: + +```sh +hatch run lmi:unit +hatch run lmi:lint +hatch run lmi:regressions --junitxml=lmi_tests/artifacts/regressions.xml +``` + +The last command asserts four fixed behaviors against the real decorator/step/map/ +parallel APIs with an in-memory checkpoint service: an expired invocation starts no +step, and an early result does not pin a parallel, map, or nested-map wrapper past +its invocation budget. These local tests have final releases and bounded fixture +waits. They establish the SDK defects, not real LMI worker recovery. + +The green harness tests exercise negative evidence controls, process-shared marker +initialization, real public-API replay/callback/retry with the local runner, +checkpoint response/error forwarding, deployment readback, and ownership/retirement. + +## Cloud prerequisites and configuration + +Use a **test account** and region supporting durable functions on LMI with Python +3.13/3.14 and arm64. Provision an existing test-owned capacity provider with an +explicit 2–128 vCPU maximum. Its `VpcConfig` supplies subnet/security-group +configuration; change networking on that provider before the run. Workers need +access to Lambda's checkpoint API, S3, and CloudWatch Logs through NAT or endpoints. +The suite saves the provider configuration, but never creates, updates, or deletes +the provider. Its owner remains responsible for its capacity costs and lifecycle. + +Set: + +| Variable | Purpose | +| --- | --- | +| `AWS_REGION` | Region matching the provider | +| `TEST_ACCOUNT_ID` | Required authenticated account check | +| `CAPACITY_PROVIDER_ARN` | Existing test provider with bounded capacity | +| `TEST_LAMBDA_EXECUTION_ROLE_ARN` | Lambda role with LMI/durable execution and logging permissions | + +The deployment identity needs CloudFormation stack operations, S3 bucket/policy/ +object operations, Lambda deployment/configuration/invoke/history/stop/delete, +`iam:PassRole` for the supplied role, CloudWatch Logs read/create/delete, and +read access to the provider. The stack's private bucket policy grants that function +role access **only** to this run's event and control objects. The SDK and dependencies +come from a wheel built from the checked-out core package; no published SDK wheel +is substituted. The artifact hash, SDK wheel version, and commit are retained. + +```sh +hatch run lmi:build +hatch run lmi:python -m lmi_tests.deploy reconcile +hatch run lmi:python -m lmi_tests.deploy deploy \ + --run-id local-20260923-a --runtime python3.13 --concurrency 2 +hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml +# Run both commands below even when a scenario fails: +hatch run lmi:python -m lmi_tests.deploy collect +hatch run lmi:python -m lmi_tests.deploy cleanup +python -m lmi_tests.summary +``` + +For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls +`collect` followed by `cleanup`; the workflow already has unconditional steps. +Use a fresh run ID for each deployment. Run all four combinations of runtime +`python3.13`/`python3.14` and environment concurrency `1`/`2`, as CI does. +The independent SDK branch-concurrency settings include 2 and 3. + +Each combination creates a unique stack, private bucket, log groups, and two +functions (`normal`, `deadline`). Both functions declare native scaling limits of +exactly one environment and use 2 GiB / 1 vCPU. LMI automatically publishes +`$LATEST.PUBLISHED`; every invocation uses that qualified target. Creating an extra +numbered version would allocate another set of environments. Before assertions, +the suite checks Active state, runtime/architecture, durability, timeout, provider, +process concurrency, applied scaling limits, commit/run identity, and code hash. +Unsupported configurations, insufficient permissions, and unavailable capacity fail +provisioning. There is no fallback to standard Lambda and no passing cloud skip. + +The normal handler mapping is saved in `artifacts/function-name-map.json`, using +the existing `PYTEST_FUNCTION_NAME_MAP` convention. The test driver reuses +`DurableFunctionCloudTestRunner` for async invocation, callback completion, and +waiting for results, with finite boto transport timeouts. + +## Evidence and regression contracts + +| Scenario | Evidence required | +| --- | --- | +| Replay/callback/retry | Real service histories; one body execution for checkpointed success and failure; stable failure type/message; retry attempts 1 and 2; nested child/map, parallel, wait, callback completion; PENDING and resumed requests | +| Environment concurrency | Held external barriers overlap in one shared `/tmp` environment marker, with distinct request IDs, execution ARNs, process UUIDs, and PIDs; c1 is the baseline, c2 must prove two processes | +| Warm cleanup | Repeated success, failure, and suspension/resume; explicit reuse of the same environment/process; no residual SDK threads after wrapper return; bounded current RSS and FD growth after warming | +| Early completion (#741) | `first_successful` parallel, `min_successful=1` map, and nested pools; loser blocked inside a step; separate winner selection, user result, user exit, wrapper exit; no late effects after grace | +| Checkpoint settlement | Real synchronous service checkpoint is acknowledged but response delivery to the SDK waiter is externally held; wrapper cannot return before settlement; releasing the response must unblock branch cleanup | +| Invocation deadline (#741) | A real short function timeout, service invocation-completed error for the exact request, external side effects, full original-environment worker capacity recovery, and healthy concurrent work on another process for c2 | +| Timeout/retry | A second service invocation of the **same** durable execution, completed step skipped, original attempt effects cease; repeated interrupted at-least-once work is recorded separately and is not called exactly-once | + +The current SDK has no public cooperative cancellation API. The fault fixture +therefore deliberately remains blocked until interrupted/retired by the eventual +SDK/runtime policy or released externally. Its 75-second emergency limit always +invalidates a regression pass. It does not implement deadline cancellation in test +code, use `shutdown(wait=False)` as a fix, kill the process itself, or manufacture +successful checkpoints. Selecting a bounded retirement strategy belongs to #741. + +The initial **test acceptance budget** is five seconds after invocation deadline +or early winner selection. This is not a published SDK cancellation contract; +review/update it together with the #741 fix. A platform-supported replacement of +an affected Python worker in the same environment is allowed. Whole-environment +replacement is reported as a placement failure until a supported policy and +corresponding evidence are agreed; scaling elsewhere cannot hide a pinned worker. +The local regressions use a one-second invocation budget and two-second observation. + +All trace records include run, commit, marker, execution ARN, request, environment, +process initialization UUID, PID, sequence, timestamp, and phase. Body/effect records +include operation/attempt where relevant. A file lock safely initializes the shared +environment UUID across Python processes. Test-only wrapper/client diagnostics do +not control durable branches. All barrier and side-effect I/O in user code lives +inside steps; operation names stay static. Infrastructure diagnostics can repeat on +replay and are explicitly separate from the body/effect ledger. + +S3 records remain readable when a wrapper is stuck. CloudWatch logs and real history +provide independent runtime/service evidence. Recovery probes are queued at the +deadline, before waiting for eventually consistent history. A client timeout or a +logical execution's `TIMED_OUT` status alone cannot satisfy invocation-timeout +assertions. Missing service retry evidence is a collection/precondition failure, +never a passing retry test. No explicit second logical execution is mislabeled as +a service retry. + +## Independent budgets and results + +| Budget | Default | +| --- | --- | +| Lambda invocation | Normal 60 s; deadline fixture 10 s | +| Durable logical execution | 240 s | +| Driver result polling | 120 s | +| Cleanup acceptance grace | 5 s | +| Fault-fixture emergency I/O release | 75 s | +| Workflow cloud assertions | 12 min | +| Matrix job including provisioning/retirement | 55 min | + +Artifacts contain JUnit, configuration/provider readbacks, qualified targets, +commit/wheel/code hash, invocation inputs/ARNs, all lifecycle and side-effect records, +execution histories/results, logs, setup/collection errors, and cleanup confirmation. +Tokens are omitted from client tracing and redacted from structured histories. +JUnit `lmi_outcome` properties and the Actions summary distinguish +`ProvisioningError`, `PlacementError`, `CollectionError`, and `RegressionAssertion`. +An unexecuted scenario is never summarized as passing. + +## CI and resource ownership + +The dedicated workflow runs the green harness checks on PRs and main pushes. +Cloud tests initially run on `workflow_dispatch` or same-repository PRs labeled +`run-lmi-e2e`, excluding Dependabot and forks. It reuses `TEST_ROLE_ARN`, +`TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. Cloud checks are +intentionally not required green until #741 is implemented and cloud placement / +timing has been validated. Scheduled and larger stress runs can be added after +this suite is stable. + +The workflow collects evidence before teardown, even after test failure or ordinary +cancellation. Cleanup sends external releases, stops running durable executions, +deletes the run's functions to retire their environments, empties the bucket, and +waits for stack deletion. Stopping an execution alone is not evidence that Python +code stopped. Only suite/run-tagged resources are eligible for deletion. A failed +create/name collision cannot cause another run's stack to be deleted. + +Forced runner termination may prevent an `always()` step. Each stack has a four-hour +expiry tag; `reconcile` runs before the next cloud deployment and can also be invoked +manually. It retires only expired stacks owned by this suite. S3 objects additionally +expire after two days; object expiry does not retire compute. Inspect cleanup errors +and run reconciliation if `cleanup.json` does not confirm deletion. The shared +capacity provider remains under its owner's explicit lifecycle policy. diff --git a/lmi_tests/__init__.py b/lmi_tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py new file mode 100644 index 000000000..f64f90b3c --- /dev/null +++ b/lmi_tests/cloud.py @@ -0,0 +1,220 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Bounded real-service driver using the repository's cloud runner.""" + +import json +import time +import uuid + +from botocore.exceptions import ClientError + +from aws_durable_execution_sdk_python_testing import DurableFunctionCloudTestRunner +from lmi_tests.deploy import ARTIFACTS, QUALIFIER, client, save, verify +from lmi_tests.evidence import CollectionError, select + + +class Cloud: + def __init__(self, manifest): + self.manifest = manifest + self.lam, self.s3, self.logs = client("lambda"), client("s3"), client("logs") + self.events = {} + self.invocations = [] + self.gates = set() + + def verify(self): + for key, arn in self.manifest["functions"].items(): + config = self.lam.get_function_configuration(FunctionName=arn) + scaling = self.lam.get_function_scaling_config( + FunctionName=arn.rsplit(":", 1)[0], Qualifier=QUALIFIER + ) + verify(config, scaling, self.manifest, key) + + def start(self, scenario, fixture="normal", gate=None): + marker = scenario + "-" + uuid.uuid4().hex[:12] + payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} + if gate: + payload["gate"] = gate + self.gates.add(gate) + self.gates.update({marker + "-loser", marker + "-checkpoint"}) + runner = DurableFunctionCloudTestRunner( + self.manifest["functions"][fixture], region=self.manifest["region"] + ) + runner.lambda_client = self.lam # bounded transport, including async start + try: + arn = runner.run_async(input=payload) + if not arn: + raise CollectionError("Invoke did not return a durable execution ARN") + except Exception as error: + save( + ARTIFACTS / f"invocations/{marker}.json", + {"payload": payload, "error": str(error)}, + ) + raise CollectionError(f"{marker}: invocation API failed") from error + item = { + "marker": marker, + "arn": arn, + "fixture": fixture, + "runner": runner, + "started": time.time(), + } + self.invocations.append(item) + print(f"LMI {scenario}: marker={marker}, execution={arn}", flush=True) + save( + ARTIFACTS / f"invocations/{marker}.json", + { + "payload": payload, + "arn": arn, + "function": runner.function_name, + "started": item["started"], + }, + ) + return item + + def release(self, gate): + self.s3.put_object( + Bucket=self.manifest["bucket"], Key="control/" + gate, Body=b"release" + ) + + def refresh(self): + try: + for page in self.s3.get_paginator("list_objects_v2").paginate( + Bucket=self.manifest["bucket"], Prefix="events/" + ): + for obj in page.get("Contents", []): + if obj["Key"] in self.events: + continue + response = self.s3.get_object( + Bucket=self.manifest["bucket"], Key=obj["Key"] + ) + with response["Body"] as stream: + event = json.loads(stream.read()) + if ( + event["run"] != self.manifest["run"] + or event["commit"] != self.manifest["commit"] + ): + raise CollectionError("Event came from another build/run") + self.events[obj["Key"]] = event + except ClientError as error: + raise CollectionError( + "Could not retrieve external lifecycle/side-effect ledger" + ) from error + events = sorted( + self.events.values(), key=lambda e: (e["time"], e["request"], e["sequence"]) + ) + save(ARTIFACTS / "events.json", events) + return events + + def for_item(self, item): + return [e for e in self.refresh() if e["marker"] == item["marker"]] + + def poll( + self, + predicate, + seconds=30, + category=CollectionError, + message="Evidence deadline exceeded", + ): + end = time.monotonic() + seconds + while True: + value = predicate() + if value: + return value + if time.monotonic() >= end: + raise category(message) + time.sleep(0.5) + + def phase(self, item, phase, **kwargs): + return self.poll(lambda: select(self.for_item(item), phase), **kwargs) + + def finish(self, item, status="SUCCEEDED"): + try: + item["runner"].wait_for_result( + item["arn"], timeout=self.manifest["driverTimeout"] + ) + except TimeoutError as error: + raise CollectionError( + "Driver timeout, not Lambda invocation timeout evidence" + ) from error + result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) + save(ARTIFACTS / f"executions/{item['marker']}.json", result) + assert result["Status"] == status, ( + f"{item['marker']}: {result['Status']} (expected {status})" + ) + if status == "SUCCEEDED": + assert json.loads(result["Result"]) == item["marker"], ( + "Result crossed execution boundaries" + ) + self.phase(item, "WRAPPER_RETURN") + return self.history(item) + + def history(self, item): + events, marker = [], None + while True: + request = {"DurableExecutionArn": item["arn"], "IncludeExecutionData": True} + if marker: + request["Marker"] = marker + result = self.lam.get_durable_execution_history(**request) + events.extend(result.get("Events", [])) + marker = result.get("NextMarker") + if not marker: + break + save(ARTIFACTS / f"histories/{item['marker']}.json", events) + return events + + def platform_timeout(self, item, request): + # A TIMED_OUT logical execution or a client poll limit is insufficient. + # Require a real invocation-completed service event for this request. + def observed(): + for event in self.history(item): + details = event.get("InvocationCompletedDetails", {}) + error = json.dumps(details.get("Error", {})).lower().replace(" ", "") + if details.get("RequestId") == request and any( + label in error for label in ("timeout", "timedout") + ): + return event + return None + + return self.poll( + observed, + seconds=40, + message="No service invocation timeout evidence for " + request, + ) + + def collect(self): + errors = [] + try: + events = self.refresh() + # Independent collect steps can recover execution ARNs without the + # pytest process or successful invoke response. + items = { + e["execution"]: {"arn": e["execution"], "marker": e["marker"]} + for e in events + } + for item in items.values(): + self.history(item) + save( + ARTIFACTS / f"executions/{item['marker']}.json", + self.lam.get_durable_execution(DurableExecutionArn=item["arn"]), + ) + except Exception as error: + errors.append(str(error)) + for key, arn in self.manifest["functions"].items(): + try: + config = self.lam.get_function_configuration(FunctionName=arn) + logs = [] + for page in self.logs.get_paginator("filter_log_events").paginate( + logGroupName=config["LoggingConfig"]["LogGroup"], + startTime=int(self.manifest["created"] * 1000), + ): + logs.extend(page.get("events", [])) + save(ARTIFACTS / f"logs/{key}.json", logs) + except Exception as error: + errors.append(str(error)) + if errors: + save(ARTIFACTS / "collection-errors.json", errors) + raise CollectionError("; ".join(errors)) + + def release_all(self): + for gate in self.gates: + self.release(gate) + self.gates.clear() diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py new file mode 100644 index 000000000..289c994bb --- /dev/null +++ b/lmi_tests/deploy.py @@ -0,0 +1,559 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Build, provision, verify, collect, and retire run-owned LMI resources.""" + +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +import time +import zipfile + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from lmi_tests.evidence import ProvisioningError + + +ROOT = Path(__file__).resolve().parents[1] +ARTIFACTS = ROOT / "lmi_tests/artifacts" +OWNER = "python-sdk-lmi-e2e" +SCALING = {"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1} +QUALIFIER = "$LATEST.PUBLISHED" + + +def client(service, region=None): + return boto3.client( + service, + region_name=region or os.environ["AWS_REGION"], + config=Config( + connect_timeout=3, read_timeout=10, retries={"total_max_attempts": 2} + ), + ) + + +def scrub(value): + if isinstance(value, dict): + return { + k: "" if k in {"CheckpointToken", "CallbackId"} else scrub(v) + for k, v in value.items() + } + if isinstance(value, list): + return [scrub(v) for v in value] + return value + + +def save(path, data): + path = Path(path) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(scrub(data), indent=2, default=str) + "\n") + + +def build(): + directory = ROOT / "lmi_tests/build" + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir() + # Hatch builds this checkout's wheel; pip resolves only its dependencies. + subprocess.run( + ["hatch", "build", "-t", "wheel", str(directory)], + cwd=ROOT / "packages/aws-durable-execution-sdk-python", + env={k: v for k, v in os.environ.items() if k != "HATCH_ENV_ACTIVE"}, + check=True, + ) + wheel = next(directory.glob("*.whl")) + target = directory / "package" + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--only-binary=:all:", + "--no-compile", + "--target", + str(target), + str(wheel), + ], + check=True, + ) + (target / "lmi_tests").mkdir() + for name in ("__init__.py", "fixture.py"): + shutil.copy(ROOT / "lmi_tests" / name, target / "lmi_tests" / name) + artifact = directory / "function.zip" + with zipfile.ZipFile(artifact, "w", zipfile.ZIP_DEFLATED) as archive: + for path in sorted(target.rglob("*")): + if path.is_file(): + info = zipfile.ZipInfo(path.relative_to(target).as_posix()) + info.compress_type = zipfile.ZIP_DEFLATED + archive.writestr(info, path.read_bytes()) + save( + ARTIFACTS / "build.json", + { + "commit": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True + ).strip(), + "wheel": wheel.name, + "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), + }, + ) + + +def template(manifest, functions=True): + bucket = manifest["bucket"] + resources = { + "Bucket": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketName": bucket, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": True, + "IgnorePublicAcls": True, + "BlockPublicPolicy": True, + "RestrictPublicBuckets": True, + }, + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + {"ServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}} + ] + }, + "LifecycleConfiguration": { + "Rules": [ + { + "Id": "expiry", + "Status": "Enabled", + "ExpirationInDays": 2, + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + }, + } + ] + }, + }, + }, + "BucketPolicy": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": {"Ref": "Bucket"}, + "PolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": {"AWS": manifest["role"]}, + "Action": ["s3:GetObject", "s3:PutObject"], + "Resource": [ + f"arn:aws:s3:::{bucket}/events/*", + f"arn:aws:s3:::{bucket}/control/*", + ], + }, + { + "Effect": "Deny", + "Principal": "*", + "Action": "s3:*", + "Resource": [ + f"arn:aws:s3:::{bucket}", + f"arn:aws:s3:::{bucket}/*", + ], + "Condition": {"Bool": {"aws:SecureTransport": "false"}}, + }, + ], + }, + }, + }, + } + outputs = {} + if functions: + for key, timeout in ( + ("normal", manifest["invocationTimeout"]), + ("deadline", manifest["deadlineTimeout"]), + ): + name = manifest["stack"] + "-" + key + resources[key + "Logs"] = { + "Type": "AWS::Logs::LogGroup", + "Properties": { + "LogGroupName": "/aws/lambda/" + name, + "RetentionInDays": 1, + }, + } + resources[key + "Function"] = { + "Type": "AWS::Lambda::Function", + "DependsOn": "BucketPolicy", + "Properties": { + "FunctionName": name, + "Runtime": manifest["runtime"], + "Architectures": ["arm64"], + "Handler": "lmi_tests.fixture.handler", + "Role": manifest["role"], + "MemorySize": 2048, + "Timeout": timeout, + "Code": {"S3Bucket": bucket, "S3Key": manifest["codeKey"]}, + "FunctionScalingConfig": SCALING, + "DurableConfig": { + "ExecutionTimeout": manifest["executionTimeout"], + "RetentionPeriodInDays": 1, + }, + "CapacityProviderConfig": { + "LambdaManagedInstancesCapacityProviderConfig": { + "CapacityProviderArn": manifest["provider"], + "PerExecutionEnvironmentMaxConcurrency": manifest[ + "concurrency" + ], + "ExecutionEnvironmentMemoryGiBPerVCpu": 2, + } + }, + "Environment": { + "Variables": { + "LMI_BUCKET": bucket, + "LMI_COMMIT": manifest["commit"], + "LMI_RUN_ID": manifest["run"], + "AWS_RETRY_MODE": "standard", + } + }, + "LoggingConfig": { + "LogFormat": "JSON", + "ApplicationLogLevel": "INFO", + "SystemLogLevel": "INFO", + "LogGroup": {"Ref": key + "Logs"}, + }, + }, + } + # LMI publishes this qualified version automatically, including its + # scaling settings. An extra numbered version would allocate more capacity. + outputs[key] = { + "Value": { + "Fn::Join": [ + "", + [{"Fn::GetAtt": [key + "Function", "Arn"]}, ":" + QUALIFIER], + ] + } + } + return { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": resources, + "Outputs": outputs, + } + + +def wait_stack(cfn, name, expected, seconds=1500): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + try: + result = cfn.describe_stacks(StackName=name)["Stacks"][0] + except ClientError as error: + if expected == "DELETE_COMPLETE" and "does not exist" in str(error): + return None + raise + status = result["StackStatus"] + if status == expected: + return result + if "FAILED" in status or "ROLLBACK" in status: + save( + ARTIFACTS / "stack-events.json", + cfn.describe_stack_events(StackName=name), + ) + raise ProvisioningError(f"{name}: {status}; no fallback to standard Lambda") + time.sleep(3) + raise ProvisioningError(f"{name}: provisioning/retirement deadline exceeded") + + +def verify(config, scaling, manifest, key): + actual = config.get("CapacityProviderConfig", {}).get( + "LambdaManagedInstancesCapacityProviderConfig", {} + ) + checks = [ + actual.get("CapacityProviderArn") == manifest["provider"], + actual.get("PerExecutionEnvironmentMaxConcurrency") == manifest["concurrency"], + config.get("Runtime") == manifest["runtime"], + config.get("Architectures") == ["arm64"], + config.get("Version") == QUALIFIER, + config.get("CodeSha256") == manifest["codeSha256"], + config.get("MemorySize") == 2048, + config.get("State") == "Active", + config.get("DurableConfig", {}).get("ExecutionTimeout") + == manifest["executionTimeout"], + config.get("Timeout") + == manifest["deadlineTimeout" if key == "deadline" else "invocationTimeout"], + scaling.get("AppliedFunctionScalingConfig") == SCALING, + config.get("Environment", {}).get("Variables", {}).get("LMI_COMMIT") + == manifest["commit"], + config.get("Environment", {}).get("Variables", {}).get("LMI_RUN_ID") + == manifest["run"], + ] + if not all(checks): + raise ProvisioningError( + f"{key}: LMI configuration/artifact/scaling readback mismatch" + ) + + +def deploy(args): + if not args.run_id or not re.fullmatch(r"[a-z0-9-]{1,24}", args.run_id): + raise ProvisioningError( + "run-id must be 1-24 lowercase letters, digits or hyphens" + ) + provider = os.environ["CAPACITY_PROVIDER_ARN"] + account = client("sts").get_caller_identity()["Account"] + if account != os.environ["TEST_ACCOUNT_ID"] or provider.split(":")[3:5] != [ + os.environ["AWS_REGION"], + account, + ]: + raise ProvisioningError( + "Authenticated account/region must match the test account and capacity provider" + ) + capacity = client("lambda").get_capacity_provider( + CapacityProviderName=provider.rsplit(":", 1)[-1].split("/")[-1] + ) + save(ARTIFACTS / "capacity-provider.json", capacity) + provider_config = capacity["CapacityProvider"] + if ( + not 2 + <= provider_config.get("CapacityProviderScalingConfig", {}).get( + "MaxVCpuCount", 0 + ) + <= 128 + ): + raise ProvisioningError( + "Use a bounded, test-owned capacity provider (2-128 maximum vCPUs)" + ) + data = (ROOT / "lmi_tests/build/function.zip").read_bytes() + digest = hashlib.sha256(data).digest() + built = json.loads((ARTIFACTS / "build.json").read_text()) + if built["sha256"] != digest.hex(): + raise ProvisioningError("Artifact changed since build") + suffix = ( + args.runtime.replace("python", "").replace(".", "") + f"-c{args.concurrency}" + ) + name = f"py-lmi-{args.run_id}-{suffix}" + manifest = { + "run": args.run_id, + "stack": name, + "bucket": name + "-" + account, + "account": account, + "region": os.environ["AWS_REGION"], + "provider": provider, + "role": os.environ["TEST_LAMBDA_EXECUTION_ROLE_ARN"], + "runtime": args.runtime, + "concurrency": args.concurrency, + "commit": built["commit"], + "codeKey": "code/" + digest.hex() + ".zip", + "codeSha256": base64.b64encode(digest).decode(), + "invocationTimeout": 60, + "deadlineTimeout": 10, + "executionTimeout": 240, + "driverTimeout": 120, + "cleanupGrace": 5, + "created": time.time(), + "expires": int(time.time()) + 4 * 3600, + "functions": {}, + } + save( + ARTIFACTS / "manifest.json", manifest + ) # enough to clean up partial provisioning + cfn = client("cloudformation") + tags = [ + {"Key": "Suite", "Value": OWNER}, + {"Key": "RunId", "Value": args.run_id}, + {"Key": "Expires", "Value": str(manifest["expires"])}, + ] + # Create only. A collision cannot take over another run or mutate shared fixtures. + cfn.create_stack( + StackName=name, TemplateBody=json.dumps(template(manifest, False)), Tags=tags + ) + manifest["owned"] = True + save(ARTIFACTS / "manifest.json", manifest) + wait_stack(cfn, name, "CREATE_COMPLETE", 180) + client("s3").put_object( + Bucket=manifest["bucket"], Key=manifest["codeKey"], Body=data + ) + cfn.update_stack( + StackName=name, TemplateBody=json.dumps(template(manifest)), Tags=tags + ) + stack = wait_stack(cfn, name, "UPDATE_COMPLETE") + for output in stack["Outputs"]: + key, arn = output["OutputKey"], output["OutputValue"] + manifest["functions"][key] = arn + save(ARTIFACTS / "manifest.json", manifest) + config = client("lambda").get_function_configuration(FunctionName=arn) + scaling = client("lambda").get_function_scaling_config( + FunctionName=arn.rsplit(":", 1)[0], Qualifier=QUALIFIER + ) + save( + ARTIFACTS / f"configuration/{key}.json", + {"function": config, "scaling": scaling}, + ) + verify(config, scaling, manifest, key) + save( + ARTIFACTS / "function-name-map.json", + {"lmi_tests.fixture.handler": manifest["functions"]["normal"]}, + ) + + +def owned_stack(cfn, manifest): + try: + stack = cfn.describe_stacks(StackName=manifest["stack"])["Stacks"][0] + except ClientError as error: + if "does not exist" in str(error): + return None + raise + tags = {t["Key"]: t["Value"] for t in stack.get("Tags", [])} + if ( + tags.get("Suite") != OWNER + or tags.get("RunId") != manifest["run"] + or not manifest["stack"].startswith("py-lmi-") + ): + raise ProvisioningError( + "Refusing to delete resources not owned by this suite/run" + ) + return stack + + +def cleanup(manifest): + if not manifest.get("owned"): + return # create failed (including a name collision): do not delete that stack + cfn, s3, lam = client("cloudformation"), client("s3"), client("lambda") + stack = owned_stack(cfn, manifest) + if stack is None: + return + resources = cfn.describe_stack_resources(StackName=manifest["stack"])[ + "StackResources" + ] + bucket_exists = any( + r["ResourceType"] == "AWS::S3::Bucket" + and r.get("PhysicalResourceId") == manifest["bucket"] + and r["ResourceStatus"] != "DELETE_COMPLETE" + for r in resources + ) + if bucket_exists: + s3.put_object( + Bucket=manifest["bucket"], Key="control/release-all", Body=b"release" + ) + errors = [] + stop_errors = [] + for resource in resources: + if ( + resource["ResourceType"] != "AWS::Lambda::Function" + or resource["ResourceStatus"] == "DELETE_COMPLETE" + ): + continue + try: + for page in lam.get_paginator( + "list_durable_executions_by_function" + ).paginate( + FunctionName=resource["PhysicalResourceId"], Statuses=["RUNNING"] + ): + for execution in page.get("DurableExecutions", []): + lam.stop_durable_execution( + DurableExecutionArn=execution["DurableExecutionArn"] + ) + except ClientError as error: + stop_errors.append(str(error)) + try: + # Delete functions BEFORE emptying the bucket. This is the retirement + # path; stopping a logical execution alone cannot stop LMI Python code. + lam.delete_function(FunctionName=resource["PhysicalResourceId"]) + except ClientError as error: + if error.response["Error"]["Code"] != "ResourceNotFoundException": + errors.append(str(error)) + if errors: + raise ProvisioningError("Function retirement failed: " + "; ".join(errors)) + if stop_errors: + save(ARTIFACTS / "cleanup-stop-errors.json", stop_errors) + # Bounded retries account for already in-flight diagnostic PUTs during retirement. + for _attempt in range(6): + if bucket_exists: + for page in s3.get_paginator("list_objects_v2").paginate( + Bucket=manifest["bucket"] + ): + keys = [{"Key": o["Key"]} for o in page.get("Contents", [])] + if keys: + deleted = s3.delete_objects( + Bucket=manifest["bucket"], Delete={"Objects": keys} + ) + if deleted.get("Errors"): + raise ProvisioningError( + "Could not empty run-owned bucket: " + + str(deleted["Errors"]) + ) + cfn.delete_stack(StackName=manifest["stack"]) + try: + wait_stack(cfn, manifest["stack"], "DELETE_COMPLETE", 180) + save( + ARTIFACTS / "cleanup.json", + {"stack": manifest["stack"], "status": "DELETED"}, + ) + return + except ProvisioningError: + # A non-bucket failure stays visible after this finite retry budget. + time.sleep(2) + raise ProvisioningError( + "Run-owned stack retirement failed; use reconcile after investigating stack-events.json" + ) + + +def reconcile(): + """Only expired, suite-tagged stacks; never mutate/delete the shared provider.""" + cfn = client("cloudformation") + for page in cfn.get_paginator("describe_stacks").paginate(): + for stack in page["Stacks"]: + tags = {t["Key"]: t["Value"] for t in stack.get("Tags", [])} + if ( + tags.get("Suite") != OWNER + or int(tags.get("Expires", "0")) >= time.time() + ): + continue + if not tags.get("Expires") or not stack["StackName"].startswith("py-lmi-"): + continue + account = client("sts").get_caller_identity()["Account"] + cleanup( + { + "owned": True, + "stack": stack["StackName"], + "bucket": stack["StackName"] + "-" + account, + "run": tags["RunId"], + } + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "command", choices=["build", "deploy", "collect", "cleanup", "reconcile"] + ) + parser.add_argument("--run-id") + parser.add_argument( + "--runtime", choices=["python3.13", "python3.14"], default="python3.13" + ) + parser.add_argument("--concurrency", type=int, choices=[1, 2], default=2) + args = parser.parse_args() + try: + if args.command == "build": + build() + elif args.command == "deploy": + deploy(args) + elif args.command == "reconcile": + reconcile() + else: + manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) + if args.command == "cleanup": + cleanup(manifest) + else: + from lmi_tests.cloud import Cloud + + Cloud(manifest).collect() + except Exception as error: + save( + ARTIFACTS / f"{args.command}-error.json", + {"type": type(error).__name__, "message": str(error)}, + ) + raise + + +if __name__ == "__main__": + main() diff --git a/lmi_tests/evidence.py b/lmi_tests/evidence.py new file mode 100644 index 000000000..4f269fbc5 --- /dev/null +++ b/lmi_tests/evidence.py @@ -0,0 +1,192 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Assertions on service and external evidence, including negative controls.""" + + +class ProvisioningError(RuntimeError): + """Unsupported configuration, unavailable capacity, or failed deployment.""" + + +class PlacementError(RuntimeError): + """The required runtime placement was not established; never a test pass.""" + + +class CollectionError(RuntimeError): + """Evidence is missing/unavailable; distinct from an SDK regression assertion.""" + + +def select(events, phase, marker=None): + return [ + e + for e in events + if e["phase"] == phase and (marker is None or e["marker"] == marker) + ] + + +def overlap(events, markers, count, environment=None): + blocked = [e for e in select(events, "BLOCKED") if e["marker"] in markers] + for start in blocked: + env = start["environment"] + if environment is not None and env != environment: + continue + active = [] + for candidate in blocked: + if candidate["environment"] != env or candidate["time"] > start["time"]: + continue + alive = [ + e + for e in events + if e["phase"] in {"ALIVE", "EFFECT"} + and all( + e.get(k) == candidate.get(k) + for k in ("request", "environment", "process", "gate") + ) + and e["time"] >= candidate["time"] + ] + # Missing exit records (e.g. a retired process) do not prove liveness. + if not alive or max(e["time"] for e in alive) < start["time"]: + continue + exits = [ + e + for e in select(events, "IO_EXIT") + if e["request"] == candidate["request"] + and e.get("gate") == candidate.get("gate") + and e["time"] >= candidate["time"] + ] + if not exits or min(e["time"] for e in exits) > start["time"]: + active.append(candidate) + if ( + len({e["request"] for e in active}) >= count + and len({e["process"] for e in active}) >= count + and len({e["pid"] for e in active}) >= count + and len({e["execution"] for e in active}) >= count + ): + return env + raise PlacementError(f"No {count}-process overlap in one execution environment") + + +def lifecycle(events): + returned = select(events, "WRAPPER_RETURN") + select(events, "WRAPPER_RAISE") + assert returned, "No wrapper exit observed" + assert not select(events, "ESCAPE"), "Test emergency timeout released SDK work" + for end in returned: + local = [e for e in events if e["request"] == end["request"]] + begins = select(local, "WRAPPER_ENTER") + assert len(begins) == 1, "Missing/duplicate wrapper entry" + baseline = {t["id"] for t in begins[0]["resources"]["threads"]} + residual = [ + t + for t in end["resources"]["threads"] + if t["id"] not in baseline + and ( + t["name"].startswith("dex-") + or t["name"].startswith("ThreadPoolExecutor") + ) + ] + assert not residual, ( + f"Invocation-owned workers survived wrapper exit: {residual}" + ) + assert any( + e["sequence"] < end["sequence"] for e in select(local, "USER_EXIT") + ), "User code still running" + assert not [ + e + for e in local + if e["sequence"] > end["sequence"] + and e["phase"] + in { + "BODY", + "EFFECT", + "CHECKPOINT_CALL", + "CHECKPOINT_ACK", + "CHECKPOINT_EXIT", + "POLL_CALL", + "POLL_EXIT", + } + ], "Late invocation activity" + + +def replay(events, history, marker): + assert len({e["request"] for e in select(events, "WRAPPER_ENTER")}) >= 2, ( + "No real replay" + ) + assert any(e["status"] == "PENDING" for e in select(events, "WRAPPER_RETURN")), ( + "No suspension" + ) + for name, terminal in [("success", "StepSucceeded"), ("failure", "StepFailed")]: + bodies = [e for e in select(events, "BODY") if e["operation"] == name] + assert len(bodies) == 1 and bodies[0]["value"] == marker, ( + f"Completed {name} repeated/contaminated" + ) + entries = [e for e in history if e.get("Name") == name] + assert sum(e["EventType"] == terminal for e in entries) == 1, ( + f"Missing {terminal}" + ) + assert len({e["Id"] for e in entries}) == 1, "Operation identity changed" + failures = select(events, "STORED_FAILURE") + assert len(failures) >= 2 and all( + e["message"] == "expected:" + marker and e["error_type"] == "ValueError" + for e in failures + ) + attempts = [ + e["attempt"] for e in select(events, "BODY") if e["operation"] == "retry" + ] + assert attempts == [1, 2], f"Unexpected retry side effects: {attempts}" + assert {"WaitSucceeded", "CallbackSucceeded", "ContextSucceeded"} <= { + e["EventType"] for e in history + } + lifecycle(events) + + +def early_completion(events, grace): + winners = select(events, "WINNER_SELECTED") + assert winners, "Completion policy did not select a winner" + winner = winners[0] + returns = [ + e for e in select(events, "WRAPPER_RETURN") if e["request"] == winner["request"] + ] + assert returns, "#741: result computed but wrapper pinned by losing branch" + assert returns[0]["time"] <= winner["time"] + grace, "#741: cleanup exceeded grace" + assert not [ + e for e in select(events, "EFFECT") if e["time"] > winner["time"] + grace + ], "#741: abandoned scope still performs side effects" + lifecycle(events) + + +def deadline(events, request, grace): + local = [e for e in events if e["request"] == request] + entry = select(local, "WRAPPER_ENTER")[0] + cutoff = entry["deadline"] + grace + assert not [ + e + for e in local + if e["time"] > cutoff + and e["phase"] in {"BODY", "EFFECT", "CHECKPOINT_CALL", "CHECKPOINT_ACK"} + ], "#741: work continued after invocation deadline + cleanup grace" + assert not select(local, "ESCAPE"), "Emergency release is not SDK deadline handling" + # A runtime-supported worker retirement can have no final diagnostic. Recovery + # and full-capacity placement must therefore also be asserted by the driver. + + +def warm_resources(events, min_reuses=3): + processes = {} + for event in select(events, "WRAPPER_RETURN"): + processes.setdefault((event["environment"], event["process"]), []).append(event) + reused = [ + sorted(v, key=lambda e: e["time"]) + for v in processes.values() + if len(v) >= min_reuses + ] + if not reused: + raise PlacementError("No observed warm Python process reuse") + for samples in reused: + # First invocation initializes legitimate client/serialization caches. + baseline = samples[1]["resources"] + for sample in samples[2:]: + current = sample["resources"] + assert len(current["threads"]) <= len(baseline["threads"]) + 1 + assert current["fds"] <= baseline["fds"] + 8, "File descriptors accumulate" + assert current["rss"] <= baseline["rss"] + 32 * 1024 * 1024, ( + "RSS grew beyond cache allowance" + ) + lifecycle(events) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py new file mode 100644 index 000000000..1a7fc145d --- /dev/null +++ b/lmi_tests/fixture.py @@ -0,0 +1,348 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Fault-injection handlers for real LMI. Never install these as SDK examples.""" + +import fcntl +import json +import os +from pathlib import Path +import threading +import time +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + +from aws_durable_execution_sdk_python import StepError, durable_execution +from aws_durable_execution_sdk_python.config import ( + CompletionConfig, + Duration, + MapConfig, + ParallelConfig, + StepConfig, +) +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInput, + DurableExecutionInvocationInputWithClient, +) +from aws_durable_execution_sdk_python.lambda_service import LambdaClient +from aws_durable_execution_sdk_python.retries import RetryDecision + + +PROCESS_ID = uuid.uuid4().hex +IO_CONFIG = Config(connect_timeout=2, read_timeout=3, retries={"total_max_attempts": 1}) +NO_RETRY = StepConfig(retry_strategy=lambda _error, _attempt: RetryDecision.no_retry()) + + +def environment_id(path="/tmp/python-lmi-environment"): + """/tmp is shared by the LMI Python workers; lock initialization across processes.""" + with open(path, "a+") as marker: + fcntl.flock(marker, fcntl.LOCK_EX) + marker.seek(0) + value = marker.read().strip() + if not value: + value = uuid.uuid4().hex + marker.write(value) + marker.flush() + os.fsync(marker.fileno()) + return value + + +def resources(): + # Current RSS, not ru_maxrss (a lifetime high-water mark). + rss_pages = int(Path("/proc/self/statm").read_text().split()[1]) + return { + "threads": [{"id": t.ident, "name": t.name} for t in threading.enumerate()], + "fds": len(list(Path("/proc/self/fd").iterdir())), + "rss": rss_pages * os.sysconf("SC_PAGE_SIZE"), + } + + +class Trace: + def __init__(self, payload, context, execution_arn): + self.payload = payload + self.context = context + self.s3 = boto3.client("s3", config=IO_CONFIG) + self.bucket = os.environ["LMI_BUCKET"] + self.lock = threading.Lock() + self.sequence = 0 + self.identity = { + "run": os.environ["LMI_RUN_ID"], + "commit": os.environ["LMI_COMMIT"], + "marker": payload["marker"], + "scenario": payload["scenario"], + "execution": execution_arn, + "request": context.aws_request_id, + "environment": environment_id(), + "process": PROCESS_ID, + "pid": os.getpid(), + } + self.deadline = time.time() + context.get_remaining_time_in_millis() / 1000 + + def emit(self, phase, **fields): + # Diagnostic I/O is outside orchestration decisions. The ledger survives a + # stuck wrapper and does not rely on the invocation response or log delivery. + with self.lock: + self.sequence += 1 + event = { + **self.identity, + "phase": phase, + "sequence": self.sequence, + "time": time.time(), + "deadline": self.deadline, + **fields, + } + print("LMI_TEST " + json.dumps(event), flush=True) + self.s3.put_object( + Bucket=self.bucket, + Key=f"events/{self.identity['request']}/{self.sequence:06d}.json", + Body=json.dumps(event).encode(), + ContentType="application/json", + ) + return event + + def signal(self, name): + self.s3.put_object(Bucket=self.bucket, Key="control/" + name, Body=b"release") + + def released(self, name): + try: + self.s3.head_object(Bucket=self.bucket, Key="control/" + name) + return True + except ClientError as error: + if error.response["Error"]["Code"] not in {"404", "NoSuchKey", "NotFound"}: + raise + return False + + def gate(self, name, *, effects=False, attempt=1): + self.emit("BLOCKED", gate=name) + end = time.monotonic() + 75 + try: + while not self.released(name) and not self.released("release-all"): + if time.monotonic() >= end: + self.emit("ESCAPE", gate=name) + raise TimeoutError( + "Test emergency release; never a passing regression" + ) + if effects: + # Each record is an independently identifiable external side effect. + # Do not stop it using the test's own Lambda deadline: that would + # mask the missing SDK cancellation/worker-retirement behavior. + self.emit( + "EFFECT", operation="blocked-io", gate=name, attempt=attempt + ) + else: + self.emit("ALIVE", gate=name) + time.sleep(0.2) # fault-injected blocking I/O, not a durable delay + except BaseException: + self.emit("IO_INTERRUPTED", gate=name) + raise + finally: + self.emit("IO_EXIT", gate=name) + + def body(self, name, value, step=None): + self.emit( + "BODY", operation=name, value=value, attempt=getattr(step, "attempt", 1) + ) + return value + + +class ObservedClient: + """Forward real SDK calls; never fabricate checkpoint acknowledgments.""" + + def __init__(self, client, trace): + self.client, self.trace = client, trace + + def get_execution_state(self, **kwargs): + self.trace.emit("POLL_CALL") + try: + return self.client.get_execution_state(**kwargs) + finally: + self.trace.emit("POLL_EXIT") + + def checkpoint(self, **kwargs): + updates = [ + { + "id": u.operation_id, + "name": u.name, + "action": u.action.value, + "type": u.operation_type.value, + } + for u in kwargs["updates"] + ] + self.trace.emit("CHECKPOINT_CALL", updates=updates) + try: + response = self.client.checkpoint(**kwargs) + self.trace.emit("CHECKPOINT_ACK", updates=updates) + if self.trace.payload["scenario"] == "checkpoint" and any( + u["name"] == "checkpoint-loser" and u["action"] == "SUCCEED" + for u in updates + ): + # Real service response, withheld while a synchronous waiter owns a + # branch. This tests settlement/join ordering without a fake service. + self.trace.emit("ACK_HELD") + self.trace.gate(self.trace.payload["marker"] + "-checkpoint") + return response + except BaseException as error: + self.trace.emit("CHECKPOINT_ERROR", error=type(error).__name__) + raise + finally: + self.trace.emit("CHECKPOINT_EXIT") + + +def workflow(event, context, trace): + marker, scenario = event["marker"], event["scenario"] + try: + value = context.step(lambda s: trace.body("success", marker, s), name="success") + if scenario == "failure": + raise ValueError("expected:" + marker) + if scenario in {"barrier", "deadline"}: + context.step( + lambda step: trace.gate( + event["gate"], effects=scenario == "deadline", attempt=step.attempt + ), + name="controlled-io", + config=NO_RETRY, + ) + context.step(lambda s: trace.body("after-io", marker, s), name="after-io") + elif scenario in {"parallel", "map", "nested"}: + + def losing(child): + def io(step): + trace.signal(marker + "-loser-started") + trace.gate(marker + "-loser", effects=True, attempt=step.attempt) + return marker + + return child.step(io, name="losing-io", config=NO_RETRY) + + def winning(child): + def io(_): + trace.gate(marker + "-loser-started") + return marker + + return child.step(io, name="winner", config=NO_RETRY) + + def race(child): + if scenario == "map": + return child.map( + [0, 1], + lambda c, item, _index, _all: ( + losing(c) if item == 0 else winning(c) + ), + name="race-map", + config=MapConfig( + max_concurrency=2, + completion_config=CompletionConfig(min_successful=1), + ), + ) + return child.parallel( + [losing, winning], + name="race-parallel", + config=ParallelConfig( + max_concurrency=2, + completion_config=CompletionConfig.first_successful(), + ), + ) + + if scenario == "nested": + context.map( + [0], + lambda c, _item, _index, _all: c.run_in_child_context( + race, name="nested-child" + ), + name="outer-map", + config=MapConfig(max_concurrency=3), + ) + else: + race(context) + trace.emit("WINNER_SELECTED") + elif scenario == "checkpoint": + context.parallel( + [ + lambda c: c.step(lambda _: marker, name="checkpoint-loser"), + lambda c: c.step(lambda _: marker, name="checkpoint-peer"), + ], + name="checkpoint-pool", + config=ParallelConfig(max_concurrency=2), + ) + elif scenario == "replay": + + def fail(step): + trace.body("failure", marker, step) + raise ValueError("expected:" + marker) + + try: + context.step(fail, name="failure", config=NO_RETRY) + except StepError as error: + trace.emit( + "STORED_FAILURE", message=error.message, error_type=error.error_type + ) + + def retry(step): + trace.body("retry", marker, step) + if step.attempt == 1: + raise ValueError("retry once") + return marker + + context.step( + retry, + name="retry", + config=StepConfig( + retry_strategy=lambda _e, n: ( + RetryDecision.retry(Duration.from_seconds(1)) + if n == 1 + else RetryDecision.no_retry() + ) + ), + ) + context.run_in_child_context( + lambda c: c.map( + [1, 2, 3], + lambda inner, item, _index, _all: inner.step( + lambda _: item, name="item" + ), + name="items", + config=MapConfig(max_concurrency=3), + ), + name="nested", + ) + context.parallel( + [ + lambda c: c.step(lambda _: value, name="left"), + lambda c: c.step(lambda _: value, name="right"), + ], + name="pair", + ) + context.wait(Duration.from_seconds(2), name="pause") + callback = context.create_callback(name="completion") + assert callback.result() == marker + trace.emit("USER_RESULT", value=value) + return value + finally: + trace.emit("USER_EXIT") + + +def handler(event, context): + invocation = DurableExecutionInvocationInput.from_json_dict(event) + payload = next( + json.loads(op.execution_details.input_payload) + for op in invocation.initial_execution_state.operations + if op.execution_details and op.execution_details.input_payload + ) + if payload["run"] != os.environ["LMI_RUN_ID"]: + raise ValueError("Input belongs to a different deployment") + trace = Trace(payload, context, invocation.durable_execution_arn) + client = ObservedClient(LambdaClient.initialize_client(), trace) + injected = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( + invocation, client + ) + decorated = durable_execution(lambda e, c: workflow(e, c, trace)) + before = resources() + trace.emit("WRAPPER_ENTER", resources=before) + try: + result = decorated(injected, context) + trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) + return result + except BaseException as error: + trace.emit("WRAPPER_RAISE", error=type(error).__name__, resources=resources()) + raise diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py new file mode 100644 index 000000000..37fba95cb --- /dev/null +++ b/lmi_tests/summary.py @@ -0,0 +1,58 @@ +"""Small, dependency-free Actions summary; no executed scenario means no pass.""" + +import json +from pathlib import Path +import xml.etree.ElementTree as ET + + +def summarize(directory): + directory = Path(directory) + manifest = directory / "manifest.json" + lines = ["LMI cloud regressions for Python SDK #741.", ""] + if manifest.exists(): + config = json.loads(manifest.read_text()) + lines.append( + f"Commit `{config['commit']}`; {config['runtime']}; environment concurrency {config['concurrency']}." + ) + lines.append( + f"Timeouts: invocation {config['deadlineTimeout']}s (fault) / {config['invocationTimeout']}s (normal), durable execution {config['executionTimeout']}s, driver {config['driverTimeout']}s; cleanup grace {config['cleanupGrace']}s." + ) + report = directory / "cloud.xml" + if report.exists(): + for case in ET.parse(report).iter("testcase"): + category = next( + ( + p.get("value") + for p in case.findall("properties/property") + if p.get("name") == "lmi_outcome" + ), + None, + ) + problem = next( + ( + case.find(k) + for k in ("error", "failure", "skipped") + if case.find(k) is not None + ), + None, + ) + state = category or (problem.tag.upper() if problem is not None else "PASS") + lines.append(f"- {case.get('name')}: {state}") + else: + lines.append( + "Cloud scenarios did not run. Inspect provisioning/build diagnostics." + ) + for path in sorted(directory.glob("*-error.json")): + data = json.loads(path.read_text()) + lines.append(f"- {path.stem}: {data['type']}") + cleanup = directory / "cleanup.json" + lines.append( + "Resources retired." + if cleanup.exists() + else "Resource retirement not confirmed; inspect cleanup diagnostics and run reconcile if needed." + ) + return "\n\n".join(lines) + + +if __name__ == "__main__": + print(summarize(Path(__file__).parent / "artifacts")) diff --git a/lmi_tests/tests/e2e/conftest.py b/lmi_tests/tests/e2e/conftest.py new file mode 100644 index 000000000..32256da13 --- /dev/null +++ b/lmi_tests/tests/e2e/conftest.py @@ -0,0 +1,53 @@ +"""Explicit cloud opt-in; infrastructure/collection exceptions stay JUnit errors.""" + +import json + +import pytest + +from lmi_tests.cloud import Cloud +from lmi_tests.deploy import ARTIFACTS +from lmi_tests.evidence import CollectionError, PlacementError, ProvisioningError + + +def pytest_addoption(parser): + parser.addoption( + "--lmi-cloud", action="store_true", help="Use a verified real LMI deployment" + ) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_makereport(item, call): + outcome = yield + report = outcome.get_result() + if call.excinfo: + category = type(call.excinfo.value).__name__ + if isinstance( + call.excinfo.value, (CollectionError, PlacementError, ProvisioningError) + ): + report.user_properties.append(("lmi_outcome", category)) + item.user_properties.append(("lmi_outcome", category)) + elif isinstance(call.excinfo.value, AssertionError): + report.user_properties.append(("lmi_outcome", "RegressionAssertion")) + item.user_properties.append(("lmi_outcome", "RegressionAssertion")) + + +@pytest.fixture(scope="session") +def deployment(request): + if not request.config.getoption("--lmi-cloud"): + pytest.skip("Real LMI deployment required; run hatch run lmi:cloud") + manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) + cloud = Cloud(manifest) + cloud.verify() + return manifest + + +@pytest.fixture +def cloud(deployment): + driver = Cloud(deployment) + try: + yield driver + finally: + try: + driver.release_all() + finally: + driver.collect() diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py new file mode 100644 index 000000000..d5a1743dc --- /dev/null +++ b/lmi_tests/tests/e2e/test_lifecycle.py @@ -0,0 +1,173 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real LMI regressions: #741 assertions intentionally fail until the SDK fix.""" + +import time +import uuid + +import pytest + +from lmi_tests import evidence + + +def test_checkpoint_replay_callback_and_retry(cloud): + item = cloud.start("replay") + callback = item["runner"].wait_for_callback( + item["arn"], name="completion", timeout=90 + ) + item["runner"].send_callback_success(callback, result=item["marker"].encode()) + history = cloud.finish(item) + evidence.replay(cloud.for_item(item), history, item["marker"]) + + +def capacity_barrier(cloud, fixture="normal", environment=None, seconds=30): + count = cloud.manifest["concurrency"] + gate = "capacity-" + uuid.uuid4().hex + items = [cloud.start("barrier", fixture, gate) for _ in range(count)] + env, events = wait_overlap(cloud, items, count, environment, seconds) + cloud.release(gate) + for item in items: + cloud.finish(item) + evidence.lifecycle(cloud.for_item(item)) + return env, evidence.select(events, "BLOCKED") + + +def wait_overlap(cloud, items, count, environment=None, seconds=30): + markers = {i["marker"] for i in items} + + def observed(): + events = [e for e in cloud.refresh() if e["marker"] in markers] + if environment is not None and any( + e["environment"] != environment for e in evidence.select(events, "BLOCKED") + ): + raise evidence.PlacementError( + "Recovery probes reached a replacement/unrelated environment" + ) + try: + return evidence.overlap(events, markers, count, environment), events + except evidence.PlacementError: + return None + + return cloud.poll( + observed, + seconds=seconds, + category=AssertionError if environment is not None else evidence.PlacementError, + message="#741: full original-environment worker capacity did not recover" + if environment is not None + else "Live worker overlap was not established in the required environment", + ) + + +def test_actual_environment_process_concurrency(cloud): + # c1 is a genuine baseline; c2 must establish different PIDs/process UUIDs + # and distinct durable execution ARNs inside the same /tmp environment. + capacity_barrier(cloud) + + +def test_warm_success_failure_and_suspension_cleanup(cloud): + items = [] + for scenario in ["success", "failure", "replay"] * 3: + item = cloud.start(scenario) + items.append(item) + if scenario == "replay": + callback = item["runner"].wait_for_callback( + item["arn"], name="completion", timeout=90 + ) + item["runner"].send_callback_success( + callback, result=item["marker"].encode() + ) + cloud.finish(item, "FAILED" if scenario == "failure" else "SUCCEEDED") + markers = {i["marker"] for i in items} + evidence.warm_resources([e for e in cloud.refresh() if e["marker"] in markers]) + + +@pytest.mark.parametrize("scenario", ["parallel", "map", "nested"]) +def test_early_completion_reclaims_losing_branches(cloud, scenario): + item = cloud.start(scenario) + winner = cloud.phase(item, "WINNER_SELECTED")[0] + # Observation is deliberately completed BEFORE any driver release signal. + cutoff = winner["time"] + cloud.manifest["cleanupGrace"] + cloud.poll(lambda: time.time() >= cutoff + 1, seconds=15) + events = cloud.for_item(item) + evidence.early_completion(events, cloud.manifest["cleanupGrace"]) + cloud.finish(item) + + +def test_real_synchronous_checkpoint_settles_before_branch_join(cloud): + item = cloud.start("checkpoint") + cloud.phase(item, "ACK_HELD") + held = cloud.for_item(item) + assert not evidence.select(held, "WRAPPER_RETURN"), ( + "Wrapper returned with an unsettled synchronous checkpoint" + ) + assert evidence.select(held, "CHECKPOINT_ACK"), ( + "A fake checkpoint cannot establish settlement" + ) + cloud.release(item["marker"] + "-checkpoint") + cloud.finish(item) + evidence.lifecycle(cloud.for_item(item)) + + +def test_invocation_deadline_isolation_and_capacity_recovery(cloud): + fault = cloud.start("deadline", "deadline", "fault-" + uuid.uuid4().hex) + blocked = cloud.phase(fault, "BLOCKED")[0] + if cloud.manifest["concurrency"] > 1: + healthy_gate = "healthy-" + uuid.uuid4().hex + healthy = cloud.start("barrier", "deadline", healthy_gate) + cloud.phase(healthy, "BLOCKED") + wait_overlap(cloud, [fault, healthy], 2, blocked["environment"]) + cloud.release(healthy_gate) + cloud.finish(healthy) + evidence.lifecycle(cloud.for_item(healthy)) + # Queue recovery demand at the runtime deadline, before waiting for the + # eventually consistent service history. History delivery latency must not + # consume the worker recovery budget. + cloud.poll(lambda: time.time() >= blocked["deadline"] - 1, seconds=15) + # Put every worker slot under demand without releasing the fault. A different + # environment cannot satisfy recovery, even if the service replaces capacity. + recovery_started = time.time() + try: + _env, probes = capacity_barrier( + cloud, "deadline", blocked["environment"], seconds=15 + ) + finally: + cloud.platform_timeout(fault, blocked["request"]) + bound = blocked["deadline"] + cloud.manifest["cleanupGrace"] + if recovery_started > bound: + raise evidence.CollectionError( + "Driver was too late to measure the recovery budget" + ) + assert max(e["time"] for e in probes) <= bound, ( + f"#741: worker recovery exceeded deadline + grace; probes started at {recovery_started}" + ) + cloud.poll(lambda: time.time() > bound + 1, seconds=15) + evidence.deadline( + cloud.for_item(fault), blocked["request"], cloud.manifest["cleanupGrace"] + ) + + +def test_service_timeout_retry_does_not_repeat_completed_step(cloud): + fault = cloud.start("deadline", "deadline", "retry-" + uuid.uuid4().hex) + first = cloud.phase(fault, "BLOCKED")[0] + cloud.platform_timeout(fault, first["request"]) + + def retry_observed(): + entries = evidence.select(cloud.for_item(fault), "WRAPPER_ENTER") + return entries if len({e["request"] for e in entries}) >= 2 else None + + # Only a retry of the SAME durable execution satisfies this assertion. A new + # execution started by pytest must not be mislabeled as a service replay. + cloud.poll( + retry_observed, + seconds=40, + message="Service retry precondition unavailable: no second request for the same execution", + ) + events = cloud.for_item(fault) + completed = [ + e for e in evidence.select(events, "BODY") if e["operation"] == "success" + ] + assert len(completed) == 1, "Completed side effect repeated across timeout/retry" + assert {e["execution"] for e in events} == {fault["arn"]} + evidence.deadline(events, first["request"], cloud.manifest["cleanupGrace"]) + # Interrupted blocked-io effects are recorded per request, not incorrectly + # constrained to exactly-once under at-least-once step semantics. diff --git a/lmi_tests/tests/regressions/conftest.py b/lmi_tests/tests/regressions/conftest.py new file mode 100644 index 000000000..cb64f770c --- /dev/null +++ b/lmi_tests/tests/regressions/conftest.py @@ -0,0 +1,11 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption("--run-lmi-regressions", action="store_true") + + +@pytest.fixture(autouse=True) +def opt_in(request): + if not request.config.getoption("--run-lmi-regressions"): + pytest.skip("#741 regressions are opt-in and expected red until the SDK fix") diff --git a/lmi_tests/tests/regressions/test_issue741.py b/lmi_tests/tests/regressions/test_issue741.py new file mode 100644 index 000000000..494194bb8 --- /dev/null +++ b/lmi_tests/tests/regressions/test_issue741.py @@ -0,0 +1,168 @@ +"""Assert fixed behavior, not the bug. All blocking fixture work has a final release.""" + +from concurrent.futures import ThreadPoolExecutor, wait +import json +import threading +import time +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.config import ( + CompletionConfig, + MapConfig, + ParallelConfig, +) +from aws_durable_execution_sdk_python.exceptions import InvocationError +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInputWithClient, + InitialExecutionState, +) +from aws_durable_execution_sdk_python.lambda_service import ( + CheckpointOutput, + CheckpointUpdatedExecutionState, + ContextDetails, + ExecutionDetails, + Operation, + OperationStatus, + OperationType, + StepDetails, +) + + +class MemoryClient: + def checkpoint(self, *, updates, **_kwargs): + changed = [ + Operation( + operation_id=u.operation_id, + operation_type=u.operation_type, + status={ + "START": OperationStatus.STARTED, + "SUCCEED": OperationStatus.SUCCEEDED, + "FAIL": OperationStatus.FAILED, + }[u.action.value], + name=u.name, + parent_id=u.parent_id, + sub_type=u.sub_type, + step_details=StepDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.STEP + else None, + context_details=ContextDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.CONTEXT + else None, + ) + for u in updates + ] + return CheckpointOutput( + "next-token", CheckpointUpdatedExecutionState(operations=changed) + ) + + +def invocation(): + return DurableExecutionInvocationInputWithClient( + durable_execution_arn="arn:test:execution/local", + checkpoint_token="token", + initial_execution_state=InitialExecutionState( + operations=[ + Operation( + operation_id="local", + operation_type=OperationType.EXECUTION, + status=OperationStatus.STARTED, + execution_details=ExecutionDetails(input_payload=json.dumps({})), + ) + ], + next_marker="", + ), + service_client=MemoryClient(), + ) + + +def lambda_context(remaining): + context = Mock() + context.aws_request_id = "local-request" + context.client_context = context.identity = context.tenant_id = None + context.invoked_function_arn = "test-function" + context._epoch_deadline_time_in_ms = int(time.time() * 1000) + remaining + context.get_remaining_time_in_millis.side_effect = lambda: ( + context._epoch_deadline_time_in_ms - int(time.time() * 1000) + ) + return context + + +def test_expired_invocation_starts_no_step_body(): + effects = [] + handler = durable_execution( + lambda _, c: c.step(lambda _: effects.append("write"), name="write") + ) + try: + handler(invocation(), lambda_context(-1000)) + except InvocationError: + pass # invocation interruption is permitted; a user operation failure is not + assert effects == [], ( + "#741: already-expired invocation still executed user side effect" + ) + + +@pytest.mark.parametrize("mode", ["parallel", "map", "nested"]) +def test_early_result_does_not_pin_wrapper_past_deadline(mode): + started, release, computed = threading.Event(), threading.Event(), threading.Event() + + def slow(child): + def io(_): + started.set() + assert release.wait(6), "emergency fixture timeout" + return "loser" + + return child.step(io, name="slow") + + def fast(child): + def io(_): + assert started.wait(3) + return "winner" + + return child.step(io, name="fast") + + def race(context): + if mode == "map": + return context.map( + [0, 1], + lambda c, item, _idx, _all: slow(c) if item == 0 else fast(c), + name="race", + config=MapConfig( + max_concurrency=2, + completion_config=CompletionConfig(min_successful=1), + ), + ) + return context.parallel( + [slow, fast], + name="race", + config=ParallelConfig( + max_concurrency=2, completion_config=CompletionConfig.first_successful() + ), + ) + + @durable_execution + def handler(_, context): + if mode == "nested": + context.map( + [0], + lambda c, _item, _idx, _all: c.run_in_child_context(race, name="child"), + name="outer", + ) + else: + race(context) + computed.set() + return "winner" + + with ThreadPoolExecutor(1) as caller: + future = caller.submit(handler, invocation(), lambda_context(1000)) + try: + assert computed.wait(2), "Fixture failed to select a winner" + done, _ = wait([future], timeout=2) + assert done, ( + "#741: user result computed but wrapper still joins losing branch past deadline" + ) + finally: + release.set() + future.result(timeout=3) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py new file mode 100644 index 000000000..61b6f1023 --- /dev/null +++ b/lmi_tests/tests/unit/test_cloud.py @@ -0,0 +1,108 @@ +import json +from unittest.mock import Mock + +import boto3 +from botocore.stub import ANY, Stubber +import pytest + +from lmi_tests import cloud as module +from lmi_tests.evidence import CollectionError +from lmi_tests.summary import summarize + + +@pytest.fixture +def driver(monkeypatch, tmp_path): + lam = boto3.client( + "lambda", + region_name="us-west-2", + aws_access_key_id="test", + aws_secret_access_key="test", + ) + monkeypatch.setattr(module, "client", lambda _: lam) + monkeypatch.setattr(module, "ARTIFACTS", tmp_path) + # CloudRunner.__init__ normally discovers credentials; the real client below + # is still used for invoke serialization/validation through botocore Stubber. + monkeypatch.setattr( + "aws_durable_execution_sdk_python_testing.runner.boto3.client", + lambda *a, **kw: lam, + ) + driver = module.Cloud( + { + "functions": { + "normal": "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" + }, + "region": "us-west-2", + "run": "unit", + } + ) + return driver + + +def test_async_cloud_runner_sends_json_to_qualified_lmi_target(driver): + arn = "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED/durable-execution/name/id" + with Stubber(driver.lam) as stub: + stub.add_response( + "invoke", + {"StatusCode": 202, "DurableExecutionArn": arn}, + { + "FunctionName": driver.manifest["functions"]["normal"], + "InvocationType": "Event", + "Payload": ANY, + }, + ) + item = driver.start("success") + assert item["arn"] == arn + stub.assert_no_pending_responses() + + +def test_missing_execution_arn_is_collection_error_not_pass(driver): + with Stubber(driver.lam) as stub: + stub.add_response("invoke", {"StatusCode": 202}) + with pytest.raises(CollectionError, match="API failed"): + driver.start("success") + + +@pytest.mark.parametrize( + "message", ["Runtime.Timeout", "Task timed out after 10 seconds"] +) +def test_requires_request_specific_service_timeout(driver, message): + good = { + "InvocationCompletedDetails": { + "RequestId": "request", + "Error": {"Payload": message}, + } + } + driver.history = Mock(return_value=[good]) + driver.poll = lambda predicate, **_: ( + predicate() or (_ for _ in ()).throw(CollectionError()) + ) + assert driver.platform_timeout({}, "request") == good + for events in ( + [{"EventType": "ExecutionTimedOut"}], + [ + { + "InvocationCompletedDetails": { + "RequestId": "different", + "Error": {"Payload": message}, + } + } + ], + ): + driver.history.return_value = events + with pytest.raises(CollectionError): + driver.platform_timeout({}, "request") + + +def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): + assert "did not run" in summarize(tmp_path) + (tmp_path / "cloud.xml").write_text( + '' + ) + summary = summarize(tmp_path) + assert "placement: PlacementError" in summary + assert "not-run: SKIPPED" in summary + assert "PASS" not in summary + (tmp_path / "deploy-error.json").write_text( + json.dumps({"type": "ProvisioningError"}) + ) + assert "ProvisioningError" in summarize(tmp_path) diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py new file mode 100644 index 000000000..ec0aa8279 --- /dev/null +++ b/lmi_tests/tests/unit/test_deploy.py @@ -0,0 +1,172 @@ +import copy +from unittest.mock import Mock + +import pytest + +from lmi_tests import deploy +from lmi_tests.evidence import ProvisioningError + + +@pytest.fixture +def manifest(): + return { + "run": "unit", + "stack": "py-lmi-unit-313-c2", + "bucket": "test-bucket", + "role": "role", + "provider": "provider", + "runtime": "python3.13", + "concurrency": 2, + "commit": "sha", + "codeKey": "code/sha.zip", + "codeSha256": "hash", + "owned": True, + "invocationTimeout": 60, + "deadlineTimeout": 10, + "executionTimeout": 240, + } + + +@pytest.mark.parametrize("runtime", ["python3.13", "python3.14"]) +@pytest.mark.parametrize("concurrency", [1, 2]) +def test_template_sets_native_scaling_durability_and_published_target( + manifest, runtime, concurrency +): + manifest.update(runtime=runtime, concurrency=concurrency) + template = deploy.template(manifest) + for key in ("normal", "deadline"): + config = template["Resources"][key + "Function"]["Properties"] + assert config["FunctionScalingConfig"] == deploy.SCALING + assert config["Runtime"] == runtime + assert ( + config["CapacityProviderConfig"][ + "LambdaManagedInstancesCapacityProviderConfig" + ]["PerExecutionEnvironmentMaxConcurrency"] + == concurrency + ) + assert config["DurableConfig"]["ExecutionTimeout"] == 240 + assert config["Timeout"] == (10 if key == "deadline" else 60) + assert ( + template["Outputs"][key]["Value"]["Fn::Join"][1][-1] == ":$LATEST.PUBLISHED" + ) + assert not any( + r["Type"] in {"AWS::Lambda::Version", "AWS::Lambda::CapacityProvider"} + for r in template["Resources"].values() + ) + + +def test_verify_rejects_standard_lambda_wrong_artifact_and_unbounded_placement( + manifest, +): + config = copy.deepcopy( + deploy.template(manifest)["Resources"]["normalFunction"]["Properties"] + ) + config.update(State="Active", Version=deploy.QUALIFIER, CodeSha256="hash") + scaling = {"AppliedFunctionScalingConfig": deploy.SCALING} + deploy.verify(config, scaling, manifest, "normal") + for field in ( + "CapacityProviderConfig", + "CodeSha256", + "Runtime", + "Timeout", + "DurableConfig", + "Environment", + ): + bad = copy.deepcopy(config) + del bad[field] + with pytest.raises(ProvisioningError): + deploy.verify(bad, scaling, manifest, "normal") + with pytest.raises(ProvisioningError): + deploy.verify( + config, + {"AppliedFunctionScalingConfig": {"MaxExecutionEnvironments": 2}}, + manifest, + "normal", + ) + + +def test_cleanup_refuses_foreign_owner_and_failed_create(manifest, monkeypatch): + cfn = Mock() + cfn.describe_stacks.return_value = { + "Stacks": [{"Tags": [{"Key": "Suite", "Value": "other"}]}] + } + monkeypatch.setattr(deploy, "client", lambda _: cfn) + with pytest.raises(ProvisioningError, match="Refusing"): + deploy.cleanup(manifest) + cfn.delete_stack.assert_not_called() + deploy.cleanup({**manifest, "owned": False}) + cfn.delete_stack.assert_not_called() + + +def test_cleanup_releases_stops_and_retires_before_bucket_deletion( + manifest, monkeypatch +): + calls = [] + cfn, s3, lam = Mock(), Mock(), Mock() + monkeypatch.setattr( + deploy, + "client", + lambda name: {"cloudformation": cfn, "s3": s3, "lambda": lam}[name], + ) + monkeypatch.setattr(deploy, "owned_stack", lambda *_: {}) + monkeypatch.setattr(deploy, "wait_stack", lambda *_: calls.append("stack-gone")) + monkeypatch.setattr(deploy, "save", lambda *_: None) + cfn.describe_stack_resources.return_value = { + "StackResources": [ + { + "ResourceType": "AWS::S3::Bucket", + "PhysicalResourceId": manifest["bucket"], + "ResourceStatus": "CREATE_COMPLETE", + }, + { + "ResourceType": "AWS::Lambda::Function", + "PhysicalResourceId": "function", + "ResourceStatus": "CREATE_COMPLETE", + }, + ] + } + s3.put_object.side_effect = lambda **_: calls.append("release") + lam.get_paginator.return_value.paginate.return_value = [ + {"DurableExecutions": [{"DurableExecutionArn": "execution"}]} + ] + lam.stop_durable_execution.side_effect = lambda **_: calls.append("stop") + lam.delete_function.side_effect = lambda **_: calls.append("retire") + s3.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": "evidence"}]} + ] + s3.delete_objects.side_effect = lambda **_: calls.append("empty") or {} + cfn.delete_stack.side_effect = lambda **_: calls.append("delete-stack") + deploy.cleanup(manifest) + assert calls == ["release", "stop", "retire", "empty", "delete-stack", "stack-gone"] + + +def test_reconcile_only_deletes_expired_suite_owned_runs(monkeypatch): + cfn = Mock() + monkeypatch.setattr(deploy, "client", lambda _: cfn) + cfn.get_caller_identity.return_value = {"Account": "123"} + + def stack(name, owner, expires): + return { + "StackName": name, + "Tags": [ + {"Key": "Suite", "Value": owner}, + {"Key": "RunId", "Value": "run"}, + {"Key": "Expires", "Value": expires}, + ], + } + + cfn.get_paginator.return_value.paginate.return_value = [ + { + "Stacks": [ + stack("py-lmi-old", deploy.OWNER, "1"), + stack("py-lmi-active", deploy.OWNER, "9999999999"), + stack("py-lmi-other", "other", "1"), + stack("unrelated", deploy.OWNER, "1"), + ] + } + ] + cleanup = Mock() + monkeypatch.setattr(deploy, "cleanup", cleanup) + deploy.reconcile() + assert cleanup.call_count == 1 + assert cleanup.call_args.args[0]["stack"] == "py-lmi-old" diff --git a/lmi_tests/tests/unit/test_evidence.py b/lmi_tests/tests/unit/test_evidence.py new file mode 100644 index 000000000..65aee11cb --- /dev/null +++ b/lmi_tests/tests/unit/test_evidence.py @@ -0,0 +1,191 @@ +import copy + +import pytest + +from lmi_tests import evidence + + +def event(phase, time=1, **fields): + return { + "phase": phase, + "time": time, + "sequence": int(time * 10), + "marker": "a", + "request": "r1", + "execution": "exec1", + "environment": "env1", + "process": "p1", + "pid": 10, + "deadline": 10, + "gate": "gate", + **fields, + } + + +def resource_snapshot(extra=False): + return { + "threads": [{"id": 1, "name": "MainThread"}] + + ([{"id": 2, "name": "dex-handler_0"}] if extra else []), + "fds": 8, + "rss": 1000, + } + + +def lifecycle_events(): + return [ + event("WRAPPER_ENTER", resources=resource_snapshot()), + event("WINNER_SELECTED", 2), + event("USER_EXIT", 3), + event("WRAPPER_RETURN", 4, status="SUCCEEDED", resources=resource_snapshot()), + ] + + +def alive(events): + return events + [ + {**e, "phase": "ALIVE", "time": max(v["time"] for v in events) + 1} + for e in events + if e["phase"] == "BLOCKED" + ] + + +def test_overlap_requires_distinct_processes_requests_and_executions(): + events = [ + event("BLOCKED"), + event( + "BLOCKED", + 2, + marker="b", + process="p2", + pid=11, + request="r2", + execution="exec2", + ), + ] + assert evidence.overlap(alive(events), {"a", "b"}, 2) == "env1" + with pytest.raises(evidence.PlacementError): + evidence.overlap(events, {"a", "b"}, 2) # starts alone do not prove overlap + for key in ("process", "pid", "request", "execution"): + invalid = copy.deepcopy(events) + invalid[1][key] = invalid[0][key] + with pytest.raises(evidence.PlacementError): + evidence.overlap(alive(invalid), {"a", "b"}, 2) + + +def test_different_environment_and_sequential_invocations_cannot_pass(): + a = event("BLOCKED") + b = event( + "BLOCKED", 3, marker="b", process="p2", pid=11, request="r2", execution="exec2" + ) + for events in ([a, {**b, "environment": "env2"}], [a, event("IO_EXIT", 2), b]): + with pytest.raises(evidence.PlacementError): + evidence.overlap(alive(events), {"a", "b"}, 2) + with pytest.raises(evidence.PlacementError): + evidence.overlap(alive([a, b]), {"a", "b"}, 2, environment="original-env") + + +@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "PENDING"]) +def test_lifecycle_accepts_clean_success_failure_and_suspension(status): + events = lifecycle_events() + events[-1]["status"] = status + evidence.lifecycle(events) + + +@pytest.mark.parametrize( + "phase", + [ + "BODY", + "EFFECT", + "CHECKPOINT_CALL", + "CHECKPOINT_ACK", + "CHECKPOINT_EXIT", + "POLL_EXIT", + ], +) +def test_late_activity_fails(phase): + with pytest.raises(AssertionError, match="Late invocation"): + evidence.lifecycle(lifecycle_events() + [event(phase, 5)]) + + +def test_return_with_threads_or_without_user_exit_fails(): + events = lifecycle_events() + events[-1]["resources"] = resource_snapshot(True) + with pytest.raises(AssertionError, match="workers survived"): + evidence.lifecycle(events) + with pytest.raises(AssertionError, match="User code still running"): + evidence.lifecycle([e for e in lifecycle_events() if e["phase"] != "USER_EXIT"]) + + +def test_winner_without_wrapper_return_and_emergency_release_fail(): + with pytest.raises(AssertionError, match="wrapper pinned"): + evidence.early_completion(lifecycle_events()[:-1], 5) + with pytest.raises(AssertionError, match="emergency"): + evidence.early_completion(lifecycle_events() + [event("ESCAPE", 3)], 5) + with pytest.raises(AssertionError, match="grace"): + evidence.early_completion(lifecycle_events(), 1) + evidence.early_completion(lifecycle_events(), 5) + + +@pytest.mark.parametrize( + "phase", ["BODY", "EFFECT", "CHECKPOINT_CALL", "CHECKPOINT_ACK"] +) +def test_expired_invocation_cannot_continue_activity(phase): + with pytest.raises(AssertionError, match="#741"): + evidence.deadline([event("WRAPPER_ENTER"), event(phase, 16)], "r1", 5) + # A distinct attempt may repeat interrupted at-least-once work. + evidence.deadline([event("WRAPPER_ENTER"), event(phase, 16, request="r2")], "r1", 5) + + +def test_warm_reuse_is_required_and_resource_growth_fails(): + with pytest.raises(evidence.PlacementError): + evidence.warm_resources(lifecycle_events()) + events = [] + for index in range(3): + events += [ + {**e, "request": f"r{index}", "time": e["time"] + index * 10} + for e in lifecycle_events() + ] + evidence.warm_resources(events) + events[-1]["resources"]["fds"] = 100 + with pytest.raises(AssertionError, match="descriptors"): + evidence.warm_resources(events) + + +def test_replay_rejects_repeated_bodies_changed_identity_and_missing_history(): + events = lifecycle_events() + events[-1]["status"] = "PENDING" + events += [ + {**e, "request": "r2", "time": e["time"] + 10} for e in lifecycle_events() + ] + events += [ + event("BODY", operation=name, value="a", attempt=attempt) + for name, attempt in [ + ("success", 1), + ("failure", 1), + ("retry", 1), + ("retry", 2), + ] + ] + events += [ + event("STORED_FAILURE", message="expected:a", error_type="ValueError") + for _ in range(2) + ] + history = [ + {"Name": "success", "Id": "s", "EventType": "StepSucceeded"}, + {"Name": "failure", "Id": "f", "EventType": "StepFailed"}, + *[ + {"EventType": kind} + for kind in ("WaitSucceeded", "CallbackSucceeded", "ContextSucceeded") + ], + ] + evidence.replay(events, history, "a") + for bad_events, bad_history in [ + (events + [event("BODY", operation="success", value="a")], history), + (events, history[1:]), + ( + events, + history + + [{"Name": "success", "Id": "different", "EventType": "StepStarted"}], + ), + ]: + with pytest.raises(AssertionError): + evidence.replay(bad_events, bad_history, "a") diff --git a/lmi_tests/tests/unit/test_fixture.py b/lmi_tests/tests/unit/test_fixture.py new file mode 100644 index 000000000..96e2919c6 --- /dev/null +++ b/lmi_tests/tests/unit/test_fixture.py @@ -0,0 +1,155 @@ +from concurrent.futures import ThreadPoolExecutor +import json +import subprocess +import sys +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.lambda_service import ( + OperationAction, + OperationType, + OperationUpdate, +) +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests.fixture import ObservedClient, workflow +from lmi_tests import fixture +from lmi_tests.tests.regressions.test_issue741 import ( + MemoryClient, + invocation, + lambda_context, +) + + +class LocalTrace: + def __init__(self): + self.events = [] + self.payload = {"scenario": "checkpoint", "marker": "local"} + + def emit(self, phase, **fields): + self.events.append({"phase": phase, **fields}) + + def body(self, name, value, step): + self.emit("BODY", operation=name, value=value, attempt=step.attempt) + return value + + def gate(self, name): + self.emit("GATE", name=name) + + +def test_environment_marker_is_shared_across_processes(tmp_path): + path = str(tmp_path / "marker") + workers = [ + subprocess.Popen( + [ + sys.executable, + "-c", + "import sys; from lmi_tests.fixture import environment_id; print(environment_id(sys.argv[1]))", + path, + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + for _ in range(4) + ] + try: + values = [] + for worker in workers: + stdout, stderr = worker.communicate(timeout=15) + assert worker.returncode == 0, stderr + values.append(stdout.strip()) + finally: + for worker in workers: + if worker.poll() is None: + worker.kill() + worker.wait(timeout=5) + assert len(set(values)) == 1 + assert len(values[0]) == 32 + + +def test_checkpoint_proxy_preserves_real_result_and_errors(): + trace, backend = LocalTrace(), Mock() + observed = ObservedClient(backend, trace) + update = OperationUpdate( + "id", + OperationType.STEP, + OperationAction.SUCCEED, + name="checkpoint-loser", + payload="result", + ) + result = observed.checkpoint( + updates=[update], checkpoint_token="secret", durable_execution_arn="arn" + ) + assert result is backend.checkpoint.return_value + backend.checkpoint.assert_called_once_with( + updates=[update], checkpoint_token="secret", durable_execution_arn="arn" + ) + assert [e["phase"] for e in trace.events] == [ + "CHECKPOINT_CALL", + "CHECKPOINT_ACK", + "ACK_HELD", + "GATE", + "CHECKPOINT_EXIT", + ] + assert "secret" not in json.dumps(trace.events) + trace.events.clear() + backend.checkpoint.side_effect = RuntimeError("network failure") + with pytest.raises(RuntimeError, match="network failure"): + observed.checkpoint(updates=[update]) + assert [e["phase"] for e in trace.events] == [ + "CHECKPOINT_CALL", + "CHECKPOINT_ERROR", + "CHECKPOINT_EXIT", + ] + + +def test_replay_fixture_uses_real_public_apis_and_preserves_outcomes(): + trace = LocalTrace() + handler = durable_execution(lambda e, c: workflow(e, c, trace)) + with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: + arn = runner.run_async(input={"marker": "local", "scenario": "replay"}) + callback = runner.wait_for_callback(arn, name="completion", timeout=10) + runner.send_callback_success(callback, result=b"local") + result = runner.wait_for_result(arn, timeout=10) + assert result.status.value == "SUCCEEDED" + assert json.loads(result.result) == "local" + bodies = [e for e in trace.events if e["phase"] == "BODY"] + assert sum(e["operation"] == "success" for e in bodies) == 1 + assert sum(e["operation"] == "failure" for e in bodies) == 1 + assert [e["attempt"] for e in bodies if e["operation"] == "retry"] == [1, 2] + failures = [e for e in trace.events if e["phase"] == "STORED_FAILURE"] + assert len(failures) >= 2 + assert all(e["message"] == "expected:local" for e in failures) + + +def test_parallel_checkpoints_finish_with_local_runner(): + trace = LocalTrace() + handler = durable_execution(lambda e, c: workflow(e, c, trace)) + with DurableFunctionTestRunner(handler) as runner, ThreadPoolExecutor(1) as caller: + result = caller.submit( + runner.run, input={"marker": "local", "scenario": "checkpoint"} + ).result(timeout=10) + assert result.status.value == "SUCCEEDED" + + +def test_cloud_entrypoint_unwraps_runtime_event_and_observes_real_wrapper(monkeypatch): + trace = LocalTrace() + trace.payload["scenario"] = "success" + monkeypatch.setenv("LMI_RUN_ID", "unit") + monkeypatch.setattr(fixture, "Trace", lambda *_: trace) + monkeypatch.setattr( + fixture.LambdaClient, "initialize_client", lambda: MemoryClient() + ) + event = invocation().to_json_dict() + event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps({"run": "unit", "marker": "local", "scenario": "success"}) + result = fixture.handler(event, lambda_context(30000)) + assert result["Status"] == "SUCCEEDED" + assert json.loads(result["Result"]) == "local" + phases = [e["phase"] for e in trace.events] + assert phases[0] == "WRAPPER_ENTER" and phases[-1] == "WRAPPER_RETURN" + assert phases.index("USER_EXIT") < phases.index("WRAPPER_RETURN") + assert "CHECKPOINT_ACK" in phases diff --git a/pyproject.toml b/pyproject.toml index 428d680e3..1da4855cd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,21 @@ all = "pytest {args}" cov = "pytest --cov-report=term-missing --cov {args}" examples-integration = "pytest --runner-mode=cloud -m example packages/aws-durable-execution-sdk-python-examples/test/ -v {args}" +[tool.hatch.envs.lmi] +detached = true +workspace.members = [ + "packages/aws-durable-execution-sdk-python", + "packages/aws-durable-execution-sdk-python-testing", +] +dependencies = ["boto3>=1.43.0", "pytest", "ruff", "build"] + +[tool.hatch.envs.lmi.scripts] +unit = "pytest lmi_tests/tests/unit {args}" +cloud = "pytest lmi_tests/tests/e2e --lmi-cloud {args}" +regressions = "pytest lmi_tests/tests/regressions --run-lmi-regressions {args}" +lint = "ruff check lmi_tests && ruff format --check lmi_tests" +build = "python -m lmi_tests.deploy build {args}" + [tool.pytest.ini_options] # Declare custom markers to avoid warnings with --strict-markers addopts = "-v --strict-markers --import-mode=importlib" From f93a3c8e10c5f84a1b0447f42c0f43ea93543bba Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 00:33:07 +0000 Subject: [PATCH 02/32] fix: skip installing the workspace root The root only coordinates workspace packages. Disable its implicit editable install so the LMI test directory does not trigger setuptools' multiple top-level package discovery error when starting examples. Exercise examples environment initialization in the unprivileged LMI harness job and verify that it imports the checked-out SDK. --- .github/workflows/lmi-e2e-tests.yml | 9 +++++++++ pyproject.toml | 2 ++ 2 files changed, 11 insertions(+) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 50d4ee1ca..bdd353529 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -27,6 +27,15 @@ jobs: - run: pip install hatch==1.16.5 - run: hatch run lmi:lint - run: hatch run lmi:unit --junitxml=lmi_tests/artifacts/unit.xml + - name: Verify examples workspace initialization + run: | + hatch run examples:python - <<'PY' + from pathlib import Path + import aws_durable_execution_sdk_python as sdk + + source = Path('packages/aws-durable-execution-sdk-python/src').resolve() + assert Path(sdk.__file__).resolve().is_relative_to(source), sdk.__file__ + PY - uses: actions/upload-artifact@v7 if: always() with: diff --git a/pyproject.toml b/pyproject.toml index 1da4855cd..7c48f373e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,6 +4,8 @@ version = "0.0.0" requires-python = ">=3.11" [tool.hatch.envs.default] +# The root coordinates workspace packages; it is not an installable distribution. +skip-install = true workspace.members = ["packages/*"] [tool.hatch.envs.test] From d8005d2b181c8ede78aabf2c465bd4e2b5d55565 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 03:51:01 +0000 Subject: [PATCH 03/32] ci: run LMI tests on PRs and main Run the complete cloud matrix on every trusted PR update, including drafts, and every main push without labels or path filters. Preserve existing cloud credential restrictions for forks and Dependabot. Use run-owned resources without a cross-run concurrency group so newer commits cannot replace another commit's pending cloud job. --- .github/workflows/lmi-e2e-tests.yml | 16 ++++++---------- lmi_tests/README.md | 29 +++++++++++++++++++---------- 2 files changed, 25 insertions(+), 20 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index bdd353529..a38e616c0 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -4,7 +4,7 @@ on: push: branches: [main] pull_request: - types: [opened, synchronize, reopened, labeled] + types: [opened, synchronize, reopened] workflow_dispatch: inputs: region: @@ -44,13 +44,12 @@ jobs: cloud: needs: harness - # Start with manual runs and explicit trusted PR opt-in; no scheduled cloud job. + # Run on every main push and trusted PR update, including drafts. + # Match the repository's cloud credential restrictions for forks/Dependabot. if: >- - github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && - github.actor != 'dependabot[bot]' && - github.event.pull_request.head.repo.full_name == github.repository && - contains(github.event.pull_request.labels.*.name, 'run-lmi-e2e')) + github.event_name != 'pull_request' || + (github.actor != 'dependabot[bot]' && + github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest timeout-minutes: 55 permissions: @@ -62,9 +61,6 @@ jobs: matrix: python: ['3.13', '3.14'] concurrency: [1, 2] - concurrency: - group: python-lmi-e2e-${{ matrix.python }}-${{ matrix.concurrency }} - cancel-in-progress: false env: AWS_REGION: ${{ inputs.region || 'us-west-2' }} CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} diff --git a/lmi_tests/README.md b/lmi_tests/README.md index b072ac9c3..483123acb 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -7,9 +7,10 @@ It follows the separate cloud-suite approach in [Java PR #728](https://github.com/aws/aws-durable-execution-sdk-java/pull/728). It changes no production SDK code. The deadline and early-completion regressions are **expected to fail on the current SDK**. They are ordinary failing assertions, -with no `xfail`, swallowed failure, or expected-failure success status. Cloud tests -and local red regressions require explicit opt-in; the ordinary test suite remains -usable while the fix is designed. +with no `xfail`, swallowed failure, or expected-failure success status. CI runs the +cloud suite automatically on every trusted PR update and main push. Local cloud +runs and local red regressions require explicit commands; the ordinary local test +suite remains usable while the fix is designed. ## Run locally @@ -156,13 +157,21 @@ An unexecuted scenario is never summarized as passing. ## CI and resource ownership -The dedicated workflow runs the green harness checks on PRs and main pushes. -Cloud tests initially run on `workflow_dispatch` or same-repository PRs labeled -`run-lmi-e2e`, excluding Dependabot and forks. It reuses `TEST_ROLE_ARN`, -`TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. Cloud checks are -intentionally not required green until #741 is implemented and cloud placement / -timing has been validated. Scheduled and larger stress runs can be added after -this suite is stable. +The dedicated workflow runs the harness and full LMI cloud matrix automatically +when a same-repository PR is opened, updated, or reopened (including Draft PRs), +and on every push to `main`, including merged changes. There are no path filters, +label requirements, or ready-for-review requirements. `workflow_dispatch` remains +available for manual reruns; there are no scheduled jobs. + +Each workflow run has its own resources and runs its four matrix entries +sequentially. Runs do not share a GitHub concurrency group, so a new PR update or +main push cannot replace another commit's pending cloud job. + +Privileged cloud jobs retain the repository's existing restrictions for forked PRs +and Dependabot; the harness still runs for those PRs. Cloud jobs reuse +`TEST_ROLE_ARN`, `TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. +The #741 regression assertions remain visibly failing until its fix lands; they +are not skipped or converted into expected-success results to keep CI green. The workflow collects evidence before teardown, even after test failure or ordinary cancellation. Cleanup sends external releases, stops running durable executions, From 70e24c6b9bee33d8bb6169caea4a73649462c80c Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 05:04:39 +0000 Subject: [PATCH 04/32] fix: repair LMI control barriers and evidence Initialize explicit hold/release objects before invoking fixtures and report missing or forbidden controls as infrastructure failures. Signal the winner only after the losing step has entered held I/O, and reject cleanup assertions when that loser exited before the winner was ready. Poll events by case with bounded parallel reads, correlate checkpoint holds to their request, and check stale effects before waiting for a retry on a potentially pinned worker. Cover the observed false positives and full fixture handshakes with regression tests. --- lmi_tests/README.md | 17 +++ lmi_tests/cloud.py | 86 ++++++++++---- lmi_tests/deploy.py | 3 + lmi_tests/evidence.py | 48 ++++++++ lmi_tests/fixture.py | 43 ++++--- lmi_tests/tests/e2e/test_lifecycle.py | 16 +-- lmi_tests/tests/unit/test_cloud.py | 62 +++++++++++ lmi_tests/tests/unit/test_control.py | 154 ++++++++++++++++++++++++++ lmi_tests/tests/unit/test_evidence.py | 53 ++++++++- 9 files changed, 434 insertions(+), 48 deletions(-) create mode 100644 lmi_tests/tests/unit/test_control.py diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 483123acb..5d59a70f4 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -31,6 +31,8 @@ waits. They establish the SDK defects, not real LMI worker recovery. The green harness tests exercise negative evidence controls, process-shared marker initialization, real public-API replay/callback/retry with the local runner, checkpoint response/error forwarding, deployment readback, and ownership/retirement. +They also exercise the real parallel/map/nested fixtures with controlled I/O and +reject an early-completion pass when the losing branch failed before the winner. ## Cloud prerequisites and configuration @@ -135,6 +137,21 @@ assertions. Missing service retry evidence is a collection/precondition failure, never a passing retry test. No explicit second logical execution is mislabeled as a service retry. +Control objects are created with `hold` before invocation and updated to `release` +explicitly. Fixtures read their content with `GetObject`; missing objects, 403s, +and invalid states emit `CONTROL_ERROR` and fail as test infrastructure errors. +This avoids relying on `HeadObject` returning 404 for missing keys when the +function role has no `s3:ListBucket` permission. The winner is released only after +the losing step has validated its controls and emitted an actual side-effect +record. A loser that exits before `WINNER_READY` cannot satisfy the regression. + +Lifecycle objects are partitioned by case marker and request ID. Live polling +reads only the current case's prefixes with a bounded pool of readers; final +collection retrieves the complete run. Earlier cases therefore do not consume a +later case's short observation budget. Checkpoint holds and returns are correlated +to the same request, and stale-attempt effects are checked before waiting for a +retry that might itself be unable to start on a pinned worker. + ## Independent budgets and results | Budget | Default | diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index f64f90b3c..a06931bec 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -3,6 +3,7 @@ """Bounded real-service driver using the repository's cloud runner.""" import json +from concurrent.futures import ThreadPoolExecutor import time import uuid @@ -10,7 +11,7 @@ from aws_durable_execution_sdk_python_testing import DurableFunctionCloudTestRunner from lmi_tests.deploy import ARTIFACTS, QUALIFIER, client, save, verify -from lmi_tests.evidence import CollectionError, select +from lmi_tests.evidence import CollectionError, check_controls, select class Cloud: @@ -34,8 +35,12 @@ def start(self, scenario, fixture="normal", gate=None): payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} if gate: payload["gate"] = gate - self.gates.add(gate) - self.gates.update({marker + "-loser", marker + "-checkpoint"}) + self.hold(gate) + if scenario in {"parallel", "map", "nested"}: + self.hold(marker + "-loser") + self.hold(marker + "-loser-started") + elif scenario == "checkpoint": + self.hold(marker + "-checkpoint") runner = DurableFunctionCloudTestRunner( self.manifest["functions"][fixture], region=self.manifest["region"] ) @@ -70,30 +75,62 @@ def start(self, scenario, fixture="normal", gate=None): ) return item + def hold(self, gate): + if gate in self.gates: + return # another invocation in this case uses the same barrier + try: + self.s3.put_object( + Bucket=self.manifest["bucket"], + Key="control/" + gate, + Body=b"hold", + IfNoneMatch="*", + ) + except ClientError as error: + if error.response["Error"]["Code"] != "PreconditionFailed": + raise CollectionError( + f"Cannot initialize control {gate}: {error}" + ) from error + self.gates.add(gate) + def release(self, gate): self.s3.put_object( Bucket=self.manifest["bucket"], Key="control/" + gate, Body=b"release" ) - def refresh(self): + def refresh(self, markers=None): + # A new pytest fixture must not scan every earlier case before it can + # observe its first event. Explicit [] requests the complete run for collection. + if markers is None: + markers = [item["marker"] for item in self.invocations] + prefixes = ( + [f"events/{marker}/" for marker in markers] if markers else ["events/"] + ) + missing = [] try: - for page in self.s3.get_paginator("list_objects_v2").paginate( - Bucket=self.manifest["bucket"], Prefix="events/" - ): - for obj in page.get("Contents", []): - if obj["Key"] in self.events: - continue - response = self.s3.get_object( - Bucket=self.manifest["bucket"], Key=obj["Key"] + for prefix in prefixes: + for page in self.s3.get_paginator("list_objects_v2").paginate( + Bucket=self.manifest["bucket"], Prefix=prefix + ): + missing.extend( + obj["Key"] + for obj in page.get("Contents", []) + if obj["Key"] not in self.events ) - with response["Body"] as stream: - event = json.loads(stream.read()) - if ( - event["run"] != self.manifest["run"] - or event["commit"] != self.manifest["commit"] - ): - raise CollectionError("Event came from another build/run") - self.events[obj["Key"]] = event + + def read(key): + response = self.s3.get_object(Bucket=self.manifest["bucket"], Key=key) + with response["Body"] as stream: + event = json.loads(stream.read()) + if ( + event["run"] != self.manifest["run"] + or event["commit"] != self.manifest["commit"] + ): + raise CollectionError("Event came from another build/run") + return key, event + + if missing: + with ThreadPoolExecutor(max_workers=8) as readers: + self.events.update(readers.map(read, missing)) except ClientError as error: raise CollectionError( "Could not retrieve external lifecycle/side-effect ledger" @@ -102,10 +139,13 @@ def refresh(self): self.events.values(), key=lambda e: (e["time"], e["request"], e["sequence"]) ) save(ARTIFACTS / "events.json", events) - return events + selected = [e for e in events if not markers or e["marker"] in markers] + if markers: + check_controls(selected) + return selected def for_item(self, item): - return [e for e in self.refresh() if e["marker"] == item["marker"]] + return self.refresh(markers=[item["marker"]]) def poll( self, @@ -183,7 +223,7 @@ def observed(): def collect(self): errors = [] try: - events = self.refresh() + events = self.refresh(markers=[]) # Independent collect steps can recover execution ARNs without the # pytest process or successful invoke response. items = { diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 289c994bb..462f9fe70 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -372,6 +372,9 @@ def deploy(args): client("s3").put_object( Bucket=manifest["bucket"], Key=manifest["codeKey"], Body=data ) + client("s3").put_object( + Bucket=manifest["bucket"], Key="control/release-all", Body=b"hold" + ) cfn.update_stack( StackName=name, TemplateBody=json.dumps(template(manifest)), Tags=tags ) diff --git a/lmi_tests/evidence.py b/lmi_tests/evidence.py index 4f269fbc5..76f88442e 100644 --- a/lmi_tests/evidence.py +++ b/lmi_tests/evidence.py @@ -23,6 +23,52 @@ def select(events, phase, marker=None): ] +def check_controls(events): + errors = select(events, "CONTROL_ERROR") + if errors: + error = errors[0] + raise CollectionError(f"Control {error['gate']} failed: {error['error']}") + + +def held_loser(events): + """Establish the fault before asserting SDK cleanup; a failed loser is invalid.""" + check_controls(events) + ready = select(events, "WINNER_READY") + if not ready: + raise CollectionError( + "Missing winner-ready evidence for the blocked-loser fixture" + ) + cutoff = ready[0] + gate = cutoff["marker"] + "-loser" + local = [ + e for e in events if e["request"] == cutoff["request"] and e.get("gate") == gate + ] + if not any( + e["phase"] == "EFFECT" and e["sequence"] < cutoff["sequence"] for e in local + ) or any( + e["phase"] == "IO_EXIT" and e["sequence"] < cutoff["sequence"] for e in local + ): + raise CollectionError("Losing branch was not held when the winner became ready") + + +def checkpoint_held(events, gate): + check_controls(events) + blocked = [e for e in select(events, "BLOCKED") if e.get("gate") == gate] + if not blocked: + raise CollectionError("Checkpoint response was not held by the control gate") + held = blocked[0] + local = [e for e in events if e["request"] == held["request"]] + assert any(e["sequence"] < held["sequence"] for e in select(local, "ACK_HELD")), ( + "No real acknowledged checkpoint was held" + ) + assert not [ + e + for e in local + if e["sequence"] > held["sequence"] + and e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE", "CHECKPOINT_EXIT"} + ], "Wrapper/checkpoint exited before the held response settled" + + def overlap(events, markers, count, environment=None): blocked = [e for e in select(events, "BLOCKED") if e["marker"] in markers] for start in blocked: @@ -139,6 +185,7 @@ def replay(events, history, marker): def early_completion(events, grace): + held_loser(events) winners = select(events, "WINNER_SELECTED") assert winners, "Completion policy did not select a winner" winner = winners[0] @@ -155,6 +202,7 @@ def early_completion(events, grace): def deadline(events, request, grace): local = [e for e in events if e["request"] == request] + check_controls(local) entry = select(local, "WRAPPER_ENTER")[0] cutoff = entry["deadline"] + grace assert not [ diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 1a7fc145d..92e2ca45f 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -96,7 +96,7 @@ def emit(self, phase, **fields): print("LMI_TEST " + json.dumps(event), flush=True) self.s3.put_object( Bucket=self.bucket, - Key=f"events/{self.identity['request']}/{self.sequence:06d}.json", + Key=f"events/{self.identity['marker']}/{self.identity['request']}/{self.sequence:06d}.json", Body=json.dumps(event).encode(), ContentType="application/json", ) @@ -107,18 +107,25 @@ def signal(self, name): def released(self, name): try: - self.s3.head_object(Bucket=self.bucket, Key="control/" + name) - return True - except ClientError as error: - if error.response["Error"]["Code"] not in {"404", "NoSuchKey", "NotFound"}: - raise - return False - - def gate(self, name, *, effects=False, attempt=1): - self.emit("BLOCKED", gate=name) + response = self.s3.get_object(Bucket=self.bucket, Key="control/" + name) + with response["Body"] as body: + state = body.read() + if state not in {b"hold", b"release"}: + raise ValueError(f"Invalid control state for {name}") + return state == b"release" + except (ClientError, ValueError) as error: + # Every key must exist before Invoke. Missing/forbidden objects are + # fixture errors, never an implicit hold or release signal. + self.emit("CONTROL_ERROR", gate=name, error=str(error)) + raise + + def gate(self, name, *, effects=False, attempt=1, ready=None): end = time.monotonic() + 75 + entered = False try: while not self.released(name) and not self.released("release-all"): + if not entered: + self.emit("BLOCKED", gate=name) if time.monotonic() >= end: self.emit("ESCAPE", gate=name) raise TimeoutError( @@ -133,9 +140,12 @@ def gate(self, name, *, effects=False, attempt=1): ) else: self.emit("ALIVE", gate=name) + if not entered and ready: + self.signal(ready) + entered = True time.sleep(0.2) # fault-injected blocking I/O, not a durable delay - except BaseException: - self.emit("IO_INTERRUPTED", gate=name) + except BaseException as error: + self.emit("IO_INTERRUPTED", gate=name, error=type(error).__name__) raise finally: self.emit("IO_EXIT", gate=name) @@ -209,8 +219,12 @@ def workflow(event, context, trace): def losing(child): def io(step): - trace.signal(marker + "-loser-started") - trace.gate(marker + "-loser", effects=True, attempt=step.attempt) + trace.gate( + marker + "-loser", + effects=True, + attempt=step.attempt, + ready=marker + "-loser-started", + ) return marker return child.step(io, name="losing-io", config=NO_RETRY) @@ -218,6 +232,7 @@ def io(step): def winning(child): def io(_): trace.gate(marker + "-loser-started") + trace.emit("WINNER_READY") return marker return child.step(io, name="winner", config=NO_RETRY) diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py index d5a1743dc..aaaf13ade 100644 --- a/lmi_tests/tests/e2e/test_lifecycle.py +++ b/lmi_tests/tests/e2e/test_lifecycle.py @@ -95,14 +95,9 @@ def test_early_completion_reclaims_losing_branches(cloud, scenario): def test_real_synchronous_checkpoint_settles_before_branch_join(cloud): item = cloud.start("checkpoint") - cloud.phase(item, "ACK_HELD") + cloud.phase(item, "BLOCKED") held = cloud.for_item(item) - assert not evidence.select(held, "WRAPPER_RETURN"), ( - "Wrapper returned with an unsettled synchronous checkpoint" - ) - assert evidence.select(held, "CHECKPOINT_ACK"), ( - "A fake checkpoint cannot establish settlement" - ) + evidence.checkpoint_held(held, item["marker"] + "-checkpoint") cloud.release(item["marker"] + "-checkpoint") cloud.finish(item) evidence.lifecycle(cloud.for_item(item)) @@ -150,6 +145,13 @@ def test_service_timeout_retry_does_not_repeat_completed_step(cloud): fault = cloud.start("deadline", "deadline", "retry-" + uuid.uuid4().hex) first = cloud.phase(fault, "BLOCKED")[0] cloud.platform_timeout(fault, first["request"]) + # Diagnose stale side effects before waiting for a retry that may itself be + # unable to start because the old attempt has pinned the only worker. + cutoff = first["deadline"] + cloud.manifest["cleanupGrace"] + cloud.poll(lambda: time.time() > cutoff + 1, seconds=15) + evidence.deadline( + cloud.for_item(fault), first["request"], cloud.manifest["cleanupGrace"] + ) def retry_observed(): entries = evidence.select(cloud.for_item(fault), "WRAPPER_ENTER") diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index 61b6f1023..d4f81bf00 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -1,4 +1,5 @@ import json +import io from unittest.mock import Mock import boto3 @@ -33,8 +34,11 @@ def driver(monkeypatch, tmp_path): }, "region": "us-west-2", "run": "unit", + "bucket": "unit-bucket", + "commit": "commit", } ) + driver.s3 = Mock() return driver @@ -106,3 +110,61 @@ def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): json.dumps({"type": "ProvisioningError"}) ) assert "ProvisioningError" in summarize(tmp_path) + + +def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset(driver): + driver.lam.invoke = Mock( + return_value={"StatusCode": 202, "DurableExecutionArn": "arn"} + ) + calls = [] + driver.s3.put_object.side_effect = lambda **kw: calls.append( + ("hold", kw["Key"], kw["Body"]) + ) + driver.lam.invoke.side_effect = lambda **_: ( + calls.append(("invoke",)) or {"StatusCode": 202, "DurableExecutionArn": "arn"} + ) + item = driver.start("parallel") + assert calls == [ + ("hold", "control/" + item["marker"] + "-loser", b"hold"), + ("hold", "control/" + item["marker"] + "-loser-started", b"hold"), + ("invoke",), + ] + calls.clear() + driver.start("barrier", gate="shared") + driver.start("barrier", gate="shared") + assert calls == [("hold", "control/shared", b"hold"), ("invoke",), ("invoke",)] + assert "release-all" not in driver.gates + + +def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): + driver.invocations = [{"marker": "current"}] + event = { + "run": "unit", + "commit": "commit", + "marker": "current", + "request": "r", + "sequence": 1, + "time": 1, + "phase": "CONTROL_ERROR", + "gate": "g", + "error": "403 Forbidden", + } + key = "events/current/r/000001.json" + driver.s3.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": key}]} + ] + driver.s3.get_object.side_effect = lambda **_: { + "Body": io.BytesIO(json.dumps(event).encode()) + } + with pytest.raises(CollectionError, match="403"): + driver.refresh() + driver.s3.get_paginator.return_value.paginate.assert_called_once_with( + Bucket="unit-bucket", Prefix="events/current/" + ) + driver.s3.get_object.assert_called_once_with(Bucket="unit-bucket", Key=key) + # Full diagnostic collection preserves errors without turning itself into a + # second failing test or omitting evidence from earlier cases. + assert driver.refresh(markers=[]) == [event] + driver.s3.get_paginator.return_value.paginate.assert_called_with( + Bucket="unit-bucket", Prefix="events/" + ) diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py new file mode 100644 index 000000000..edab68e03 --- /dev/null +++ b/lmi_tests/tests/unit/test_control.py @@ -0,0 +1,154 @@ +"""Regressions for the real-cloud 403 and premature-loser false positives.""" + +import io +import json +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from unittest.mock import Mock + +import boto3 +from botocore.exceptions import ClientError +from botocore.stub import Stubber +import pytest + +from lmi_tests.fixture import Trace, workflow +from lmi_tests import evidence +from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context +from aws_durable_execution_sdk_python import durable_execution + + +@pytest.fixture +def trace(): + value = Trace.__new__(Trace) + value.s3 = boto3.client( + "s3", + region_name="us-west-2", + aws_access_key_id="test", + aws_secret_access_key="test", + ) + value.bucket = "test-bucket" + value.emit = Mock() + return value + + +@pytest.mark.parametrize("state,released", [(b"hold", False), (b"release", True)]) +def test_control_reads_explicit_state_and_closes_body(trace, state, released): + body = io.BytesIO(state) + with Stubber(trace.s3) as stub: + stub.add_response( + "get_object", + {"Body": body}, + {"Bucket": "test-bucket", "Key": "control/gate"}, + ) + assert trace.released("gate") is released + assert body.closed + trace.emit.assert_not_called() + + +@pytest.mark.parametrize("code,status", [("403", 403), ("NoSuchKey", 404)]) +def test_missing_or_forbidden_control_is_not_a_hold_or_release(trace, code, status): + with Stubber(trace.s3) as stub: + stub.add_client_error( + "get_object", service_error_code=code, http_status_code=status + ) + with pytest.raises(ClientError): + trace.gate("gate", effects=True, ready="ready") + phases = [call.args[0] for call in trace.emit.call_args_list] + assert "CONTROL_ERROR" in phases + assert not {"BLOCKED", "ALIVE", "EFFECT"} & set(phases) + + +def test_invalid_control_content_cannot_release_work(trace): + with Stubber(trace.s3) as stub: + stub.add_response("get_object", {"Body": io.BytesIO(b"unexpected")}) + with pytest.raises(ValueError, match="Invalid control state"): + trace.released("gate") + assert trace.emit.call_args.args[0] == "CONTROL_ERROR" + + +def test_winner_is_signalled_only_after_loser_actually_enters_held_io(trace): + controls = { + "control/loser": b"hold", + "control/ready": b"hold", + "control/release-all": b"hold", + } + trace.s3 = Mock() + trace.s3.get_object.side_effect = lambda **kw: { + "Body": io.BytesIO(controls[kw["Key"]]) + } + + def signal(**kwargs): + assert kwargs["Key"] == "control/ready" and kwargs["Body"] == b"release" + assert [c.args[0] for c in trace.emit.call_args_list] == ["BLOCKED", "EFFECT"] + controls[kwargs["Key"]] = kwargs["Body"] + controls["control/loser"] = b"release" + + trace.s3.put_object.side_effect = signal + trace.gate("loser", effects=True, ready="ready") + assert [c.args[0] for c in trace.emit.call_args_list] == [ + "BLOCKED", + "EFFECT", + "IO_EXIT", + ] + assert controls["control/release-all"] == b"hold" + + +def test_emitted_records_are_partitioned_by_case_and_request(trace): + trace.s3 = Mock() + trace.identity = {"marker": "case", "request": "request"} + trace.sequence = 0 + trace.deadline = 10 + trace.lock = threading.Lock() + Trace.emit(trace, "BODY") + assert ( + trace.s3.put_object.call_args.kwargs["Key"] == "events/case/request/000001.json" + ) + + +@pytest.mark.parametrize("scenario", ["parallel", "map", "nested"]) +def test_full_fixture_establishes_a_real_blocked_loser(scenario): + """Exercise fixture wiring and SDK APIs; valid evidence survives future SDK fixes.""" + objects = { + "control/release-all": b"hold", + "control/local-loser": b"hold", + "control/local-loser-started": b"hold", + } + events = [] + computed = threading.Event() + lock = threading.Lock() + + def put(**kwargs): + with lock: + objects[kwargs["Key"]] = kwargs["Body"] + if kwargs["Key"].startswith("events/"): + event = json.loads(kwargs["Body"]) + events.append(event) + if event["phase"] == "WINNER_SELECTED": + computed.set() + + def get(**kwargs): + with lock: + return {"Body": io.BytesIO(objects[kwargs["Key"]])} + + trace = Trace.__new__(Trace) + trace.s3, trace.bucket = Mock(), "test-bucket" + trace.s3.put_object.side_effect, trace.s3.get_object.side_effect = put, get + trace.identity = {"marker": "local", "request": "request"} + trace.lock, trace.sequence, trace.deadline = threading.Lock(), 0, time.time() + 30 + handler = durable_execution( + lambda _e, c: workflow({"marker": "local", "scenario": scenario}, c, trace) + ) + with ThreadPoolExecutor(1) as caller: + future = caller.submit(handler, invocation(), lambda_context(30000)) + try: + assert computed.wait(3), ( + "The controlled fixture did not reach winner selection" + ) + with lock: + snapshot = list(events) + evidence.held_loser(snapshot) + assert any(e["phase"] == "EFFECT" for e in snapshot) + finally: + put(Key="control/local-loser", Body=b"release") + assert future.result(timeout=3)["Status"] == "SUCCEEDED" diff --git a/lmi_tests/tests/unit/test_evidence.py b/lmi_tests/tests/unit/test_evidence.py index 65aee11cb..c36e89a25 100644 --- a/lmi_tests/tests/unit/test_evidence.py +++ b/lmi_tests/tests/unit/test_evidence.py @@ -40,6 +40,14 @@ def lifecycle_events(): ] +def race_events(): + return lifecycle_events() + [ + event("BLOCKED", 1.1, gate="a-loser"), + event("EFFECT", 1.2, gate="a-loser"), + event("WINNER_READY", 1.3), + ] + + def alive(events): return events + [ {**e, "phase": "ALIVE", "time": max(v["time"] for v in events) + 1} @@ -117,12 +125,49 @@ def test_return_with_threads_or_without_user_exit_fails(): def test_winner_without_wrapper_return_and_emergency_release_fail(): with pytest.raises(AssertionError, match="wrapper pinned"): - evidence.early_completion(lifecycle_events()[:-1], 5) + evidence.early_completion( + [e for e in race_events() if e["phase"] != "WRAPPER_RETURN"], 5 + ) with pytest.raises(AssertionError, match="emergency"): - evidence.early_completion(lifecycle_events() + [event("ESCAPE", 3)], 5) + evidence.early_completion(race_events() + [event("ESCAPE", 3)], 5) with pytest.raises(AssertionError, match="grace"): - evidence.early_completion(lifecycle_events(), 1) - evidence.early_completion(lifecycle_events(), 5) + evidence.early_completion(race_events(), 1) + evidence.early_completion(race_events(), 5) + + +def test_early_completion_rejects_the_cloud_false_positive(): + with pytest.raises(evidence.CollectionError, match="not held"): + evidence.early_completion( + [e for e in race_events() if e["phase"] != "EFFECT"], 5 + ) + with pytest.raises(evidence.CollectionError, match="not held"): + evidence.early_completion( + race_events() + [event("IO_EXIT", 1.25, gate="a-loser")], 5 + ) + with pytest.raises(evidence.CollectionError, match="403"): + evidence.early_completion( + race_events() + + [event("CONTROL_ERROR", gate="a-loser", error="403 Forbidden")], + 5, + ) + # SDK cancellation after the winning step is ready, before the public + # parallel call returns, is legitimate cleanup rather than a fixture error. + evidence.early_completion( + race_events() + [event("IO_EXIT", 1.5, gate="a-loser")], 5 + ) + + +def test_checkpoint_hold_requires_real_blocking_and_correlates_the_request(): + held = [event("ACK_HELD", 1), event("BLOCKED", 2, gate="checkpoint")] + evidence.checkpoint_held(held, "checkpoint") + with pytest.raises(evidence.CollectionError, match="not held"): + evidence.checkpoint_held(held[:1], "checkpoint") + with pytest.raises(AssertionError, match="before the held"): + evidence.checkpoint_held(held + [event("WRAPPER_RETURN", 3)], "checkpoint") + # An unrelated request's return is not evidence that this waiter settled. + evidence.checkpoint_held( + held + [event("WRAPPER_RETURN", 3, request="other")], "checkpoint" + ) @pytest.mark.parametrize( From 9150c871f8e4f011725bc9cd7fb7c336fb59994d Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:05:57 +0000 Subject: [PATCH 05/32] test: add Java lifecycle guards on Python 3.14 Keep LMI on Python 3.14 with environment concurrency one and two. Exercise root finally before PENDING, bounded nested progress, concurrent shared-handler invocations, in-flight return/failure cleanup, and late operations from abandoned children using Python public APIs. Document the Java/Python mapping and preserve the newly exposed late-step scope defect as an explicit issue 741 regression. --- .github/workflows/lmi-e2e-tests.yml | 10 +- lmi_tests/README.md | 51 ++++- lmi_tests/cloud.py | 11 +- lmi_tests/deploy.py | 4 +- lmi_tests/evidence.py | 81 ++++++++ lmi_tests/fixture.py | 84 +++++++- lmi_tests/tests/e2e/test_lifecycle.py | 108 +++++++++++ .../tests/regressions/test_late_operation.py | 9 + lmi_tests/tests/unit/test_cloud.py | 27 ++- lmi_tests/tests/unit/test_deploy.py | 10 +- lmi_tests/tests/unit/test_evidence.py | 70 +++++++ lmi_tests/tests/unit/test_java_parity.py | 180 ++++++++++++++++++ pyproject.toml | 1 + 13 files changed, 618 insertions(+), 28 deletions(-) create mode 100644 lmi_tests/tests/regressions/test_late_operation.py create mode 100644 lmi_tests/tests/unit/test_java_parity.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index a38e616c0..a425001da 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: '3.13' + python-version: '3.14' - run: pip install hatch==1.16.5 - run: hatch run lmi:lint - run: hatch run lmi:unit --junitxml=lmi_tests/artifacts/unit.xml @@ -43,6 +43,7 @@ jobs: path: lmi_tests/artifacts/ cloud: + name: cloud (Python 3.14, concurrency ${{ matrix.concurrency }}) needs: harness # Run on every main push and trusted PR update, including drafts. # Match the repository's cloud credential restrictions for forks/Dependabot. @@ -59,7 +60,6 @@ jobs: fail-fast: false max-parallel: 1 matrix: - python: ['3.13', '3.14'] concurrency: [1, 2] env: AWS_REGION: ${{ inputs.region || 'us-west-2' }} @@ -70,7 +70,7 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: ${{ matrix.python }} + python-version: '3.14' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ secrets.TEST_ROLE_ARN }} @@ -91,7 +91,7 @@ jobs: - name: Deploy verified LMI functions timeout-minutes: 28 env: - LMI_RUNTIME: python${{ matrix.python }} + LMI_RUNTIME: python3.14 LMI_CONCURRENCY: ${{ matrix.concurrency }} run: | hatch run lmi:python -m lmi_tests.deploy deploy \ @@ -118,7 +118,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: lmi-${{ matrix.python }}-c${{ matrix.concurrency }}-${{ github.run_id }}-${{ github.run_attempt }} + name: lmi-3.14-c${{ matrix.concurrency }}-${{ github.run_id }}-${{ github.run_attempt }} retention-days: 7 path: lmi_tests/artifacts/ - name: Summarize outcomes diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 5d59a70f4..6e1fd1e2d 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -14,7 +14,7 @@ suite remains usable while the fix is designed. ## Run locally -Use Python 3.13 or 3.14 and Hatch from the repository root: +Use Python 3.14 and Hatch from the repository root: ```sh hatch run lmi:unit @@ -22,10 +22,11 @@ hatch run lmi:lint hatch run lmi:regressions --junitxml=lmi_tests/artifacts/regressions.xml ``` -The last command asserts four fixed behaviors against the real decorator/step/map/ +The last command asserts five fixed behaviors against the real decorator/step/map/ parallel APIs with an in-memory checkpoint service: an expired invocation starts no step, and an early result does not pin a parallel, map, or nested-map wrapper past -its invocation budget. These local tests have final releases and bounded fixture +its invocation budget. A further regression checks that an abandoned child cannot +start another step in its `finally` block. These local tests have final releases and bounded fixture waits. They establish the SDK defects, not real LMI worker recovery. The green harness tests exercise negative evidence controls, process-shared marker @@ -37,7 +38,7 @@ reject an early-completion pass when the losing branch failed before the winner. ## Cloud prerequisites and configuration Use a **test account** and region supporting durable functions on LMI with Python -3.13/3.14 and arm64. Provision an existing test-owned capacity provider with an +3.14 and arm64. Provision an existing test-owned capacity provider with an explicit 2–128 vCPU maximum. Its `VpcConfig` supplies subnet/security-group configuration; change networking on that provider before the run. Workers need access to Lambda's checkpoint API, S3, and CloudWatch Logs through NAT or endpoints. @@ -65,7 +66,7 @@ is substituted. The artifact hash, SDK wheel version, and commit are retained. hatch run lmi:build hatch run lmi:python -m lmi_tests.deploy reconcile hatch run lmi:python -m lmi_tests.deploy deploy \ - --run-id local-20260923-a --runtime python3.13 --concurrency 2 + --run-id local-20260923-a --runtime python3.14 --concurrency 2 hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml # Run both commands below even when a scenario fails: hatch run lmi:python -m lmi_tests.deploy collect @@ -75,9 +76,10 @@ python -m lmi_tests.summary For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls `collect` followed by `cleanup`; the workflow already has unconditional steps. -Use a fresh run ID for each deployment. Run all four combinations of runtime -`python3.13`/`python3.14` and environment concurrency `1`/`2`, as CI does. -The independent SDK branch-concurrency settings include 2 and 3. +Use a fresh run ID for each deployment. Run both environment-concurrency settings, +`1` and `2`, with Python 3.14, as CI does. The LMI suite and its Hatch environment +target only Python 3.14. Independent SDK branch-concurrency settings include 1, 2, +and 3. Each combination creates a unique stack, private bucket, log groups, and two functions (`normal`, `deadline`). Both functions declare native scaling limits of @@ -106,6 +108,37 @@ waiting for results, with finite boto transport timeouts. | Invocation deadline (#741) | A real short function timeout, service invocation-completed error for the exact request, external side effects, full original-environment worker capacity recovery, and healthy concurrent work on another process for c2 | | Timeout/retry | A second service invocation of the **same** durable execution, completed step skipped, original attempt effects cease; repeated interrupted at-least-once work is recorded separately and is not called exactly-once | +### Java lifecycle coverage in Python + +The analogous cases from Java PR #728 are mapped to Python's public APIs: + +| Java case | Python coverage | +| --- | --- | +| Root `finally` before PENDING, with a healthy peer | `test_pending_waits_for_root_finally_and_then_replays`: externally hold root cleanup, require cleanup exit before PENDING, verify actual wait/resume and no repeated step body; c2 also observes a healthy worker progressing during cleanup | +| Two roots on a fixed executor | Local subprocess test invokes the same decorated handler concurrently through the real local runner, with distinct inputs/checkpoints and a bounded rendezvous | +| Fixed-pool nested child/map/parallel progress | `test_nested_single_lane_pools_progress_for_all_runtime_workers`: saturate runtime lanes, release a shared barrier, and finish nested child/map/parallel work with each SDK branch pool limited to one worker; results and a 15-second progress budget are asserted | +| Return/failure with an in-flight step | `test_root_return_or_failure_settles_inflight_work`: `first_successful` leaves a controlled losing branch in flight, then the root returns or raises; the wrapper must settle that work before returning SUCCEEDED/FAILED, retaining the original failure; c2 keeps a healthy peer active | +| Non-cooperative residual child attempts later SDK work | `test_abandoned_child_rejects_late_durable_operation`: after parent completion, explicitly attempt another step in the losing branch's `finally`; rejection must precede any body execution or service checkpoint | +| Deadline, worker recovery, repeated warm batches | Existing deadline/isolation/retry and warm-resource tests remain enabled | + +There is no Python equivalent of Java's public shared-executor injection or +`stepAsync`. The Python cases use separate invocation scopes, public child/map/ +parallel APIs, and LMI's distinct Python worker processes. They do not inject a +Java-style executor into Python or manufacture an async step API. + +The root-finally gate is test-only lifecycle instrumentation after a durable wait +has requested suspension; it does not schedule another durable operation or change +the handler's result/operation sequence. Ordinary orchestration delays still use +`context.wait`. Local shared-root tests run under a subprocess watchdog, so an +executor-starvation regression cannot hang the test process. Cloud progress tests +have bounded driver budgets and the existing run-owned-function retirement path. + +The late-operation guard currently exposes an additional #741 scope-lifetime gap: +a newly created operation can escape the SDK's set of already-known orphaned +operation IDs. Its local regression is explicitly enabled with `lmi:regressions`; +its real-cloud assertion always runs. The green harness validates fixture behavior +and negative evidence controls without treating the SDK defect as a passing result. + The current SDK has no public cooperative cancellation API. The fault fixture therefore deliberately remains blocked until interrupted/retired by the eventual SDK/runtime policy or released externally. Its 75-second emergency limit always @@ -180,7 +213,7 @@ and on every push to `main`, including merged changes. There are no path filters label requirements, or ready-for-review requirements. `workflow_dispatch` remains available for manual reruns; there are no scheduled jobs. -Each workflow run has its own resources and runs its four matrix entries +Each workflow run has its own resources and runs its two matrix entries sequentially. Runs do not share a GitHub concurrency group, so a new PR update or main push cannot replace another commit's pending cloud job. diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index a06931bec..e0ccd5a33 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -36,11 +36,20 @@ def start(self, scenario, fixture="normal", gate=None): if gate: payload["gate"] = gate self.hold(gate) - if scenario in {"parallel", "map", "nested"}: + if scenario in { + "parallel", + "map", + "nested", + "return-inflight", + "failure-inflight", + "late-operation", + }: self.hold(marker + "-loser") self.hold(marker + "-loser-started") elif scenario == "checkpoint": self.hold(marker + "-checkpoint") + elif scenario == "suspend-cleanup": + self.hold(marker + "-cleanup") runner = DurableFunctionCloudTestRunner( self.manifest["functions"][fixture], region=self.manifest["region"] ) diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 462f9fe70..74d5c370c 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -530,9 +530,7 @@ def main(): "command", choices=["build", "deploy", "collect", "cleanup", "reconcile"] ) parser.add_argument("--run-id") - parser.add_argument( - "--runtime", choices=["python3.13", "python3.14"], default="python3.13" - ) + parser.add_argument("--runtime", choices=["python3.14"], default="python3.14") parser.add_argument("--concurrency", type=int, choices=[1, 2], default=2) args = parser.parse_args() try: diff --git a/lmi_tests/evidence.py b/lmi_tests/evidence.py index 76f88442e..38c9c0abd 100644 --- a/lmi_tests/evidence.py +++ b/lmi_tests/evidence.py @@ -69,6 +69,87 @@ def checkpoint_held(events, gate): ], "Wrapper/checkpoint exited before the held response settled" +def scope_exit(events, gate, status=None): + """A result or failure must not abandon live user work (Java in-flight cases).""" + check_controls(events) + assert not select(events, "ESCAPE"), "Fixture emergency exit is not SDK cleanup" + starts = [e for e in select(events, "BLOCKED") if e.get("gate") == gate] + if not starts: + raise CollectionError("Missing in-flight work/cleanup precondition") + request = starts[0]["request"] + local = [e for e in events if e["request"] == request] + ends = select(local, "WRAPPER_RETURN") + select(local, "WRAPPER_RAISE") + for end in ends: + exits = [e for e in select(local, "IO_EXIT") if e.get("gate") == gate] + assert exits and exits[0]["sequence"] < end["sequence"], ( + "Wrapper exited while invocation-owned work was still active" + ) + if status is not None: + assert any(e.get("status") == status for e in ends), ( + f"No {status} return for the original request" + ) + lifecycle(events) + + +def suspension_cleanup(events, history): + check_controls(events) + pending = [e for e in select(events, "WRAPPER_RETURN") if e["status"] == "PENDING"] + assert pending, "No real PENDING invocation observed" + assert len({e["request"] for e in select(events, "WRAPPER_ENTER")}) >= 2, ( + "No real resume" + ) + for end in pending: + local = [e for e in events if e["request"] == end["request"]] + enters, exits = select(local, "CLEANUP_ENTER"), select(local, "CLEANUP_EXIT") + assert ( + enters + and exits + and enters[0]["sequence"] < exits[0]["sequence"] < end["sequence"] + ), "PENDING preceded root finally cleanup" + bodies = [e for e in select(events, "BODY") if e["operation"] == "success"] + assert len(bodies) == 1, "Completed body repeated after cleanup/suspension" + assert any(e.get("EventType") == "WaitSucceeded" for e in history) + lifecycle(events) + + +def progress(events, marker, seconds=15): + check_controls(events) + entered, exited = select(events, "PROGRESS_BEGIN"), select(events, "PROGRESS") + assert entered and exited, ( + "Nested work did not progress with single-lane branch pools" + ) + assert exited[0]["time"] - entered[0]["time"] <= seconds, ( + "Nested orchestration exhausted its progress budget" + ) + assert exited[0]["value"] == [[marker + ":0", marker + ":1"], marker], ( + "Nested results crossed invocation boundaries" + ) + lifecycle(events) + + +def late_operation(events, history): + held_loser(events) + ready = select(events, "WINNER_READY")[0] + assert select(events, "WINNER_SELECTED"), "Parent completion was not observed" + attempts, rejected = select(events, "LATE_ATTEMPT"), select(events, "LATE_REJECTED") + assert attempts and attempts[0]["sequence"] > ready["sequence"], ( + "Late operation was not attempted after winner readiness" + ) + assert not [e for e in select(events, "BODY") if e["operation"] == "late-work"], ( + "#741: abandoned child executed a late user side effect" + ) + assert rejected and rejected[0]["error"] == "OrphanedChildException", ( + "#741: completed child scope did not reject subsequent work" + ) + assert not select(events, "LATE_ACCEPTED"), ( + "#741: completed child scope accepted a late operation" + ) + assert not [e for e in history if e.get("Name") == "late-work"], ( + "#741: late operation reached the service" + ) + lifecycle(events) + + def overlap(events, markers, count, environment=None): blocked = [e for e in select(events, "BLOCKED") if e["marker"] in markers] for start in blocked: diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 92e2ca45f..d9033a861 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -28,6 +28,7 @@ ) from aws_durable_execution_sdk_python.lambda_service import LambdaClient from aws_durable_execution_sdk_python.retries import RetryDecision +from aws_durable_execution_sdk_python.exceptions import OrphanedChildException PROCESS_ID = uuid.uuid4().hex @@ -200,13 +201,51 @@ def checkpoint(self, **kwargs): self.trace.emit("CHECKPOINT_EXIT") +def nested_progress(context, trace, marker): + """Nested public operations must progress even when each branch pool has one lane.""" + trace.emit("PROGRESS_BEGIN") + + def children(child): + def mapped(branch): + return branch.map( + [0, 1], + lambda inner, item, _index, _all: inner.run_in_child_context( + lambda leaf: leaf.step( + lambda step: trace.body("leaf", f"{marker}:{item}", step), + name="leaf", + ), + name="grandchild", + ), + name="inner-map", + config=MapConfig(max_concurrency=1), + ).get_results() + + return child.parallel( + [ + mapped, + lambda branch: branch.run_in_child_context( + lambda leaf: leaf.step( + lambda step: trace.body("sibling", marker, step), name="sibling" + ), + name="sibling-child", + ), + ], + name="outer-parallel", + config=ParallelConfig(max_concurrency=1), + ).get_results() + + result = context.run_in_child_context(children, name="progress-child") + assert result == [[marker + ":0", marker + ":1"], marker] + trace.emit("PROGRESS", value=result) + + def workflow(event, context, trace): marker, scenario = event["marker"], event["scenario"] try: value = context.step(lambda s: trace.body("success", marker, s), name="success") if scenario == "failure": raise ValueError("expected:" + marker) - if scenario in {"barrier", "deadline"}: + if scenario in {"barrier", "deadline", "nested-progress"}: context.step( lambda step: trace.gate( event["gate"], effects=scenario == "deadline", attempt=step.attempt @@ -215,7 +254,18 @@ def workflow(event, context, trace): config=NO_RETRY, ) context.step(lambda s: trace.body("after-io", marker, s), name="after-io") - elif scenario in {"parallel", "map", "nested"}: + if scenario == "nested-progress": + nested_progress(context, trace, marker) + elif scenario == "suspend-cleanup": + context.wait(Duration.from_seconds(5), name="pause") + elif scenario in { + "parallel", + "map", + "nested", + "return-inflight", + "failure-inflight", + "late-operation", + }: def losing(child): def io(step): @@ -227,7 +277,24 @@ def io(step): ) return marker - return child.step(io, name="losing-io", config=NO_RETRY) + try: + return child.step(io, name="losing-io", config=NO_RETRY) + finally: + if scenario == "late-operation": + # The preceding step's final checkpoint can itself be + # rejected after the parent finishes. Still test the next + # SDK boundary, without swallowing unrelated failures. + trace.emit("LATE_ATTEMPT") + try: + child.step( + lambda step: trace.body("late-work", marker, step), + name="late-work", + config=NO_RETRY, + ) + except OrphanedChildException as error: + trace.emit("LATE_REJECTED", error=type(error).__name__) + else: + trace.emit("LATE_ACCEPTED") def winning(child): def io(_): @@ -271,6 +338,9 @@ def race(child): else: race(context) trace.emit("WINNER_SELECTED") + if scenario == "failure-inflight": + trace.emit("ROOT_FAILURE_READY") + raise ValueError("expected:" + marker) elif scenario == "checkpoint": context.parallel( [ @@ -334,6 +404,14 @@ def retry(step): trace.emit("USER_RESULT", value=value) return value finally: + if scenario == "suspend-cleanup": + # Test-only root-finally diagnostics. This gate changes only exit + # timing; it introduces no durable operations during suspension. + trace.emit("CLEANUP_ENTER") + try: + trace.gate(marker + "-cleanup") + finally: + trace.emit("CLEANUP_EXIT") trace.emit("USER_EXIT") diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py index aaaf13ade..eed607727 100644 --- a/lmi_tests/tests/e2e/test_lifecycle.py +++ b/lmi_tests/tests/e2e/test_lifecycle.py @@ -173,3 +173,111 @@ def retry_observed(): evidence.deadline(events, first["request"], cloud.manifest["cleanupGrace"]) # Interrupted blocked-io effects are recorded per request, not incorrectly # constrained to exactly-once under at-least-once step semantics. + + +def start_healthy_anchor(cloud): + if cloud.manifest["concurrency"] == 1: + return None, None + gate = "anchor-" + uuid.uuid4().hex + item = cloud.start("barrier", gate=gate) + cloud.phase(item, "BLOCKED") + return item, gate + + +def healthy_during(cloud, anchor, victim, boundary): + if anchor is None: + return + wait_overlap(cloud, [anchor, victim], 2) + cloud.poll( + lambda: [ + e + for e in cloud.for_item(anchor) + if e["phase"] == "ALIVE" and e["time"] >= boundary["time"] + ], + message="Healthy peer did not progress while victim cleanup was held", + ) + + +def finish_anchor(cloud, anchor, gate): + if anchor is not None: + cloud.release(gate) + cloud.finish(anchor) + evidence.lifecycle(cloud.for_item(anchor)) + + +def test_pending_waits_for_root_finally_and_then_replays(cloud): + anchor, anchor_gate = start_healthy_anchor(cloud) + item = cloud.start("suspend-cleanup") + blocked = cloud.phase(item, "BLOCKED")[0] + healthy_during(cloud, anchor, item, blocked) + gate = item["marker"] + "-cleanup" + evidence.scope_exit(cloud.for_item(item), gate) + cloud.release(gate) + history = cloud.finish(item) + evidence.suspension_cleanup(cloud.for_item(item), history) + finish_anchor(cloud, anchor, anchor_gate) + + +def test_nested_single_lane_pools_progress_for_all_runtime_workers(cloud): + gate = "progress-" + uuid.uuid4().hex + items = [ + cloud.start("nested-progress", gate=gate) + for _ in range(cloud.manifest["concurrency"]) + ] + wait_overlap(cloud, items, len(items)) + cloud.release(gate) + for item in items: + cloud.phase( + item, + "PROGRESS", + seconds=20, + category=AssertionError, + message="Nested child/map/parallel work starved behind its own branch pool", + ) + cloud.finish(item) + evidence.progress(cloud.for_item(item), item["marker"]) + + +@pytest.mark.parametrize( + "scenario,status", + [("return-inflight", "SUCCEEDED"), ("failure-inflight", "FAILED")], +) +def test_root_return_or_failure_settles_inflight_work(cloud, scenario, status): + anchor, anchor_gate = start_healthy_anchor(cloud) + item = cloud.start(scenario) + cloud.phase(item, "WINNER_SELECTED") + root_exit = cloud.phase(item, "USER_EXIT")[0] + healthy_during(cloud, anchor, item, root_exit) + gate = item["marker"] + "-loser" + evidence.held_loser(cloud.for_item(item)) + evidence.scope_exit(cloud.for_item(item), gate) + cloud.release(gate) + cloud.finish(item, status) + if status == "FAILED": + failure = cloud.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ + "Error" + ] + assert ( + failure["ErrorType"] == "ValueError" + and failure["ErrorMessage"] == "expected:" + item["marker"] + ) + evidence.scope_exit(cloud.for_item(item), gate, status) + finish_anchor(cloud, anchor, anchor_gate) + + +def test_abandoned_child_rejects_late_durable_operation(cloud): + item = cloud.start("late-operation") + cloud.phase(item, "WINNER_SELECTED") + evidence.held_loser(cloud.for_item(item)) + cloud.release(item["marker"] + "-loser") + cloud.poll( + lambda: [ + e + for e in cloud.for_item(item) + if e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + ], + message="Residual branch did not settle after release", + ) + history = cloud.history(item) + evidence.late_operation(cloud.for_item(item), history) + cloud.finish(item) diff --git a/lmi_tests/tests/regressions/test_late_operation.py b/lmi_tests/tests/regressions/test_late_operation.py new file mode 100644 index 000000000..53acf1143 --- /dev/null +++ b/lmi_tests/tests/regressions/test_late_operation.py @@ -0,0 +1,9 @@ +"""A stopped parent must reject new work created in a residual branch's finally.""" + +from lmi_tests import evidence +from lmi_tests.tests.unit.test_java_parity import run_inflight_case + + +def test_abandoned_child_cannot_start_another_step(): + events, history = run_inflight_case("late-operation", "SUCCEEDED") + evidence.late_operation(events, history) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index d4f81bf00..5aaaf1782 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -112,7 +112,12 @@ def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): assert "ProvisioningError" in summarize(tmp_path) -def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset(driver): +@pytest.mark.parametrize( + "scenario", ["parallel", "return-inflight", "failure-inflight", "late-operation"] +) +def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset( + driver, scenario +): driver.lam.invoke = Mock( return_value={"StatusCode": 202, "DurableExecutionArn": "arn"} ) @@ -123,7 +128,7 @@ def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset(driv driver.lam.invoke.side_effect = lambda **_: ( calls.append(("invoke",)) or {"StatusCode": 202, "DurableExecutionArn": "arn"} ) - item = driver.start("parallel") + item = driver.start(scenario) assert calls == [ ("hold", "control/" + item["marker"] + "-loser", b"hold"), ("hold", "control/" + item["marker"] + "-loser-started", b"hold"), @@ -136,6 +141,24 @@ def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset(driv assert "release-all" not in driver.gates +def test_finally_control_is_initialized_before_invocation(driver): + calls = [] + driver.s3.put_object.side_effect = lambda **kw: calls.append( + ("hold", kw["Key"], kw["Body"]) + ) + driver.lam.invoke = Mock( + side_effect=lambda **_: ( + calls.append(("invoke",)) + or {"StatusCode": 202, "DurableExecutionArn": "arn"} + ) + ) + item = driver.start("suspend-cleanup") + assert calls == [ + ("hold", "control/" + item["marker"] + "-cleanup", b"hold"), + ("invoke",), + ] + + def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): driver.invocations = [{"marker": "current"}] event = { diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index ec0aa8279..85dded018 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -11,11 +11,11 @@ def manifest(): return { "run": "unit", - "stack": "py-lmi-unit-313-c2", + "stack": "py-lmi-unit-314-c2", "bucket": "test-bucket", "role": "role", "provider": "provider", - "runtime": "python3.13", + "runtime": "python3.14", "concurrency": 2, "commit": "sha", "codeKey": "code/sha.zip", @@ -27,12 +27,12 @@ def manifest(): } -@pytest.mark.parametrize("runtime", ["python3.13", "python3.14"]) @pytest.mark.parametrize("concurrency", [1, 2]) def test_template_sets_native_scaling_durability_and_published_target( - manifest, runtime, concurrency + manifest, concurrency ): - manifest.update(runtime=runtime, concurrency=concurrency) + runtime = "python3.14" + manifest.update(concurrency=concurrency) template = deploy.template(manifest) for key in ("normal", "deadline"): config = template["Resources"][key + "Function"]["Properties"] diff --git a/lmi_tests/tests/unit/test_evidence.py b/lmi_tests/tests/unit/test_evidence.py index c36e89a25..a0dbf9685 100644 --- a/lmi_tests/tests/unit/test_evidence.py +++ b/lmi_tests/tests/unit/test_evidence.py @@ -234,3 +234,73 @@ def test_replay_rejects_repeated_bodies_changed_identity_and_missing_history(): ]: with pytest.raises(AssertionError): evidence.replay(bad_events, bad_history, "a") + + +@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "PENDING"]) +def test_scope_exit_rejects_return_before_owned_work_exits(status): + events = lifecycle_events() + [event("BLOCKED", 1.1)] + events[3]["status"] = status + with pytest.raises(AssertionError, match="still active"): + evidence.scope_exit(events, "gate", status) + evidence.scope_exit(events + [event("IO_EXIT", 3.5)], "gate", status) + + +def test_suspension_requires_finally_before_pending_and_preserves_replay(): + events = [ + event("WRAPPER_ENTER", resources=resource_snapshot()), + event("BODY", 1.1, operation="success"), + event("CLEANUP_ENTER", 1.2), + event("CLEANUP_EXIT", 2), + event("USER_EXIT", 2.1), + event("WRAPPER_RETURN", 2.2, status="PENDING", resources=resource_snapshot()), + event("WRAPPER_ENTER", 3, request="r2", resources=resource_snapshot()), + event("USER_EXIT", 4, request="r2"), + event( + "WRAPPER_RETURN", + 5, + request="r2", + status="SUCCEEDED", + resources=resource_snapshot(), + ), + ] + history = [{"EventType": "WaitSucceeded"}] + evidence.suspension_cleanup(events, history) + wrong = [ + {**e, "sequence": 30} if e["phase"] == "CLEANUP_EXIT" else e for e in events + ] + with pytest.raises(AssertionError, match="finally"): + evidence.suspension_cleanup(wrong, history) + with pytest.raises(AssertionError, match="repeated"): + evidence.suspension_cleanup( + events + [event("BODY", 3.1, request="r2", operation="success")], history + ) + + +def test_progress_requires_correct_nested_results_within_budget(): + events = lifecycle_events() + [ + event("PROGRESS_BEGIN", 1.1), + event("PROGRESS", 2, value=[["a:0", "a:1"], "a"]), + ] + evidence.progress(events, "a") + with pytest.raises(AssertionError, match="crossed"): + evidence.progress(events, "different") + with pytest.raises(AssertionError, match="budget"): + evidence.progress(events, "a", seconds=0.1) + + +def test_late_rejection_after_a_side_effect_is_not_a_pass(): + events = race_events() + [ + event("LATE_ATTEMPT", 2.1), + event("LATE_REJECTED", 2.2, error="OrphanedChildException"), + ] + evidence.late_operation(events, []) + with pytest.raises(AssertionError, match="late user side effect"): + evidence.late_operation( + events + [event("BODY", 2.15, operation="late-work")], [] + ) + with pytest.raises(AssertionError, match="accepted"): + evidence.late_operation(events + [event("LATE_ACCEPTED", 2.2)], []) + with pytest.raises(AssertionError, match="reached the service"): + evidence.late_operation( + events, [{"Name": "late-work", "EventType": "StepStarted"}] + ) diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py new file mode 100644 index 000000000..c65724ccc --- /dev/null +++ b/lmi_tests/tests/unit/test_java_parity.py @@ -0,0 +1,180 @@ +"""Public-API counterparts of Java's root-finally and executor-starvation guards.""" + +import io +import json +import subprocess +import sys +import threading +import time +import uuid + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import evidence +from lmi_tests.fixture import Trace, nested_progress, resources, workflow + + +class Controls: + def __init__(self): + self.condition = threading.Condition() + self.objects = {"control/release-all": b"hold"} + self.events = [] + + def put_object(self, *, Key, Body, **_kwargs): + with self.condition: + self.objects[Key] = Body + if Key.startswith("events/"): + self.events.append(json.loads(Body)) + self.condition.notify_all() + + def get_object(self, *, Key, **_kwargs): + with self.condition: + return {"Body": io.BytesIO(self.objects[Key])} + + def wait(self, phase): + with self.condition: + assert self.condition.wait_for( + lambda: any(e["phase"] == phase for e in self.events), timeout=5 + ), phase + return next(e for e in self.events if e["phase"] == phase) + + def snapshot(self): + with self.condition: + return list(self.events) + + def trace(self, marker, scenario): + trace = Trace.__new__(Trace) + trace.s3, trace.bucket = self, "unit" + trace.payload = {"marker": marker, "scenario": scenario} + trace.identity = { + "marker": marker, + "request": uuid.uuid4().hex, + "environment": "local", + "process": "local", + "pid": 1, + } + trace.lock, trace.sequence, trace.deadline = ( + threading.Lock(), + 0, + time.time() + 30, + ) + return trace + + +def test_pending_waits_for_finally_then_replays_without_repeating_body(): + controls = Controls() + controls.put_object(Key="control/local-cleanup", Body=b"hold") + + def entry(event, context): + trace = controls.trace("local", "suspend-cleanup") + trace.emit("WRAPPER_ENTER", resources=resources()) + result = durable_execution(lambda e, c: workflow(e, c, trace))(event, context) + trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) + return result + + with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: + arn = runner.run_async(input={"marker": "local", "scenario": "suspend-cleanup"}) + try: + controls.wait("BLOCKED") + evidence.scope_exit(controls.snapshot(), "local-cleanup") + assert not [ + e for e in controls.snapshot() if e["phase"] == "WRAPPER_RETURN" + ], "PENDING returned before finally completed" + finally: + controls.put_object(Key="control/local-cleanup", Body=b"release") + result = runner.wait_for_result(arn, timeout=10) + history = runner.get_execution_history(arn).to_dict()["Events"] + assert result.status.value == "SUCCEEDED" + evidence.suspension_cleanup(controls.snapshot(), history) + + +def run_inflight_case(scenario, status): + controls = Controls() + for suffix in ("loser", "loser-started"): + controls.put_object(Key="control/local-" + suffix, Body=b"hold") + + def entry(event, context): + trace = controls.trace("local", scenario) + trace.emit("WRAPPER_ENTER", resources=resources()) + result = durable_execution(lambda e, c: workflow(e, c, trace))(event, context) + trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) + return result + + with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: + arn = runner.run_async(input={"marker": "local", "scenario": scenario}) + try: + controls.wait("USER_EXIT") + evidence.held_loser(controls.snapshot()) + evidence.scope_exit(controls.snapshot(), "local-loser") + finally: + controls.put_object(Key="control/local-loser", Body=b"release") + result = runner.wait_for_result(arn, timeout=5) + history = runner.get_execution_history(arn).to_dict()["Events"] + assert result.status.value == status + if status == "FAILED": + assert ( + result.error.message == "expected:local" + and result.error.type == "ValueError" + ) + evidence.scope_exit(controls.snapshot(), "local-loser", status) + return controls.snapshot(), history + + +@pytest.mark.parametrize( + "scenario,status", + [("return-inflight", "SUCCEEDED"), ("failure-inflight", "FAILED")], +) +def test_inflight_return_and_failure_use_real_sdk_scopes(scenario, status): + run_inflight_case(scenario, status) + + +def run_shared_roots(nested): + """Runs in a child process so a future executor deadlock cannot hang pytest.""" + rendezvous = threading.Barrier(2, timeout=3) + + class QuietTrace: + def emit(self, *_args, **_kwargs): + pass + + def body(self, _name, value, _step=None): + return value + + @durable_execution + def handler(event, context): + context.step(lambda _: rendezvous.wait(), name="root-rendezvous") + if nested: + nested_progress(context, QuietTrace(), event["marker"]) + return context.step(lambda _: event["marker"], name="result") + + with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: + arns = [ + runner.run_async(input={"marker": marker}, execution_name=marker) + for marker in ("left", "right") + ] + for marker, arn in zip(("left", "right"), arns): + result = runner.wait_for_result(arn, timeout=5) + assert result.status.value == "SUCCEEDED" + assert json.loads(result.result) == marker + + +@pytest.mark.parametrize("nested", [False, True]) +def test_shared_decorator_roots_progress_without_starving_children(nested): + try: + result = subprocess.run( + [ + sys.executable, + "-c", + "from lmi_tests.tests.unit.test_java_parity import run_shared_roots; " + f"run_shared_roots({nested!r})", + ], + capture_output=True, + text=True, + timeout=15, + ) + except subprocess.TimeoutExpired: + pytest.fail( + "Shared root/nested work exceeded its bounded executor progress test" + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/pyproject.toml b/pyproject.toml index 7c48f373e..5461d93fa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ examples-integration = "pytest --runner-mode=cloud -m example packages/aws-durab [tool.hatch.envs.lmi] detached = true +python = "3.14" workspace.members = [ "packages/aws-durable-execution-sdk-python", "packages/aws-durable-execution-sdk-python-testing", From 3ebcf7d559dca9cfc565ccb4efe1af9e346f3cba Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:32:33 +0000 Subject: [PATCH 06/32] test: pace LMI execution history reads Collect each case's own executions during teardown and leave full-run history/log collection to the final workflow step. Pace history reads and bound throttle-only retries, including reads through the public cloud runner, without retrying invocation or callback writes. Cover pagination, throttling exhaustion, permission errors, and scoped versus full-run collection with focused tests. --- lmi_tests/README.md | 6 ++ lmi_tests/cloud.py | 71 +++++++++++++++++++---- lmi_tests/tests/e2e/conftest.py | 2 +- lmi_tests/tests/unit/test_cloud.py | 90 ++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 11 deletions(-) diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 6e1fd1e2d..5183a9a9e 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -185,6 +185,12 @@ later case's short observation budget. Checkpoint holds and returns are correlat to the same request, and stale-attempt effects are checked before waiting for a retry that might itself be unable to start on a pinned worker. +Per-case teardown collects only that case's executions. Complete-run histories +and CloudWatch logs are collected once by the final workflow step. Both runner +and artifact history reads are paced at one request per second, with at most five +throttling attempts within a 20-second retry budget (in addition to the finite +transport timeout). Invocation and callback writes are not retried by this layer. + ## Independent budgets and results | Budget | Default | diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index e0ccd5a33..c98af4417 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -14,6 +14,19 @@ from lmi_tests.evidence import CollectionError, check_controls, select +class RunnerClient: + """Share paced history reads with the public runner; other APIs are unchanged.""" + + def __init__(self, cloud): + self.cloud = cloud + + def __getattr__(self, name): + return getattr(self.cloud.lam, name) + + def get_durable_execution_history(self, **kwargs): + return self.cloud.history_page(**kwargs) + + class Cloud: def __init__(self, manifest): self.manifest = manifest @@ -21,6 +34,7 @@ def __init__(self, manifest): self.events = {} self.invocations = [] self.gates = set() + self._next_history_read = 0.0 def verify(self): for key, arn in self.manifest["functions"].items(): @@ -53,7 +67,7 @@ def start(self, scenario, fixture="normal", gate=None): runner = DurableFunctionCloudTestRunner( self.manifest["functions"][fixture], region=self.manifest["region"] ) - runner.lambda_client = self.lam # bounded transport, including async start + runner.lambda_client = RunnerClient(self) try: arn = runner.run_async(input=payload) if not arn: @@ -106,7 +120,7 @@ def release(self, gate): Bucket=self.manifest["bucket"], Key="control/" + gate, Body=b"release" ) - def refresh(self, markers=None): + def refresh(self, markers=None, *, validate_controls=True): # A new pytest fixture must not scan every earlier case before it can # observe its first event. Explicit [] requests the complete run for collection. if markers is None: @@ -149,7 +163,7 @@ def read(key): ) save(ARTIFACTS / "events.json", events) selected = [e for e in events if not markers or e["marker"] in markers] - if markers: + if markers and validate_controls: check_controls(selected) return selected @@ -196,13 +210,39 @@ def finish(self, item, status="SUCCEEDED"): self.phase(item, "WRAPPER_RETURN") return self.history(item) + def history_page(self, **request): + """Bound read-only throttling retries without replaying invocation writes.""" + deadline = time.monotonic() + 20 + last_error = None + for attempt in range(5): + delay = max(0, self._next_history_read - time.monotonic()) + if time.monotonic() + delay >= deadline: + break + if delay: + time.sleep(delay) + try: + return self.lam.get_durable_execution_history(**request) + except ClientError as error: + if error.response["Error"]["Code"] not in { + "TooManyRequestsException", + "ThrottlingException", + }: + raise + last_error = error + finally: + self._next_history_read = time.monotonic() + 1 + self._next_history_read = time.monotonic() + min(2**attempt, 4) + raise CollectionError( + "Execution-history API throttle budget exhausted" + ) from last_error + def history(self, item): events, marker = [], None while True: request = {"DurableExecutionArn": item["arn"], "IncludeExecutionData": True} if marker: request["Marker"] = marker - result = self.lam.get_durable_execution_history(**request) + result = self.history_page(**request) events.extend(result.get("Events", [])) marker = result.get("NextMarker") if not marker: @@ -229,25 +269,36 @@ def observed(): message="No service invocation timeout evidence for " + request, ) - def collect(self): + def collect(self, *, full_run=True): errors = [] + if not full_run and not self.invocations: + return try: - events = self.refresh(markers=[]) + events = self.refresh( + markers=[] if full_run else [i["marker"] for i in self.invocations], + validate_controls=False, + ) # Independent collect steps can recover execution ARNs without the # pytest process or successful invoke response. items = { e["execution"]: {"arn": e["execution"], "marker": e["marker"]} for e in events } - for item in items.values(): + except Exception as error: + errors.append(str(error)) + items = {} + if not full_run: + items.update({i["arn"]: i for i in self.invocations}) + for item in items.values(): + try: self.history(item) save( ARTIFACTS / f"executions/{item['marker']}.json", self.lam.get_durable_execution(DurableExecutionArn=item["arn"]), ) - except Exception as error: - errors.append(str(error)) - for key, arn in self.manifest["functions"].items(): + except Exception as error: + errors.append(str(error)) + for key, arn in self.manifest["functions"].items() if full_run else []: try: config = self.lam.get_function_configuration(FunctionName=arn) logs = [] diff --git a/lmi_tests/tests/e2e/conftest.py b/lmi_tests/tests/e2e/conftest.py index 32256da13..cae08df4b 100644 --- a/lmi_tests/tests/e2e/conftest.py +++ b/lmi_tests/tests/e2e/conftest.py @@ -50,4 +50,4 @@ def cloud(deployment): try: driver.release_all() finally: - driver.collect() + driver.collect(full_run=False) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index 5aaaf1782..d861df583 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -4,6 +4,7 @@ import boto3 from botocore.stub import ANY, Stubber +from botocore.exceptions import ClientError import pytest from lmi_tests import cloud as module @@ -191,3 +192,92 @@ def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): driver.s3.get_paginator.return_value.paginate.assert_called_with( Bucket="unit-bucket", Prefix="events/" ) + + +class Clock: + def __init__(self): + self.now = 0.0 + + def monotonic(self): + return self.now + + def time(self): + return self.now + + def sleep(self, seconds): + self.now += seconds + + +def throttle_error(): + return ClientError( + {"Error": {"Code": "TooManyRequestsException", "Message": "Rate exceeded"}}, + "GetDurableExecutionHistory", + ) + + +def test_history_retries_preserve_pages_and_pace_requests(driver, monkeypatch): + clock = Clock() + monkeypatch.setattr(module, "time", clock) + driver.lam.get_durable_execution_history = Mock( + side_effect=[ + throttle_error(), + {"Events": [{"EventId": 1}], "NextMarker": "page2"}, + {"Events": [{"EventId": 2}]}, + ] + ) + assert driver.history({"arn": "execution", "marker": "case"}) == [ + {"EventId": 1}, + {"EventId": 2}, + ] + calls = driver.lam.get_durable_execution_history.call_args_list + assert calls[0] == calls[1] + assert calls[2].kwargs["Marker"] == "page2" + assert clock.now >= 2 + + +def test_history_throttling_has_a_finite_budget(driver, monkeypatch): + clock = Clock() + monkeypatch.setattr(module, "time", clock) + driver.lam.get_durable_execution_history = Mock(side_effect=throttle_error()) + with pytest.raises(CollectionError, match="throttle budget"): + driver.history_page(DurableExecutionArn="execution") + assert driver.lam.get_durable_execution_history.call_count == 5 + assert clock.now < 20 + + +def test_history_does_not_retry_permission_errors_or_invocations(driver, monkeypatch): + monkeypatch.setattr(module, "time", Clock()) + driver.lam.invoke = Mock( + return_value={"StatusCode": 202, "DurableExecutionArn": "execution"} + ) + item = driver.start("success") + error = ClientError( + {"Error": {"Code": "AccessDeniedException"}}, "GetDurableExecutionHistory" + ) + driver.lam.get_durable_execution_history = Mock(side_effect=error) + with pytest.raises(ClientError): + item["runner"].lambda_client.get_durable_execution_history( + DurableExecutionArn="execution" + ) + driver.lam.invoke.assert_called_once() + driver.lam.get_durable_execution_history.assert_called_once() + + +def test_case_collection_is_scoped_but_final_collection_is_complete(driver): + driver.invocations = [{"arn": "execution", "marker": "case"}] + driver.refresh = Mock(return_value=[{"execution": "execution", "marker": "case"}]) + driver.history = Mock(return_value=[]) + driver.lam.get_durable_execution = Mock(return_value={"Status": "SUCCEEDED"}) + driver.logs = Mock() + driver.collect(full_run=False) + driver.refresh.assert_called_once_with(markers=["case"], validate_controls=False) + driver.history.assert_called_once() + driver.logs.get_paginator.assert_not_called() + driver.manifest["created"] = 0 + driver.lam.get_function_configuration = Mock( + return_value={"LoggingConfig": {"LogGroup": "group"}} + ) + driver.logs.get_paginator.return_value.paginate.return_value = [] + driver.collect() + driver.refresh.assert_called_with(markers=[], validate_controls=False) + driver.logs.get_paginator.assert_called_once_with("filter_log_events") From ef1314622645791163f48aa8f222a97a36c1c44b Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 18:47:38 +0000 Subject: [PATCH 07/32] test: bound LMI peer placement before admission Checkpoint a victim's placement decision before starting fault work. If a runtime environment rotates between anchor and victim admission, replace the pair with a bounded retry. Never retry an admitted lifecycle assertion. Keep the admission decision stable during replay. --- lmi_tests/README.md | 5 +++ lmi_tests/cloud.py | 4 +- lmi_tests/fixture.py | 17 +++++++- lmi_tests/tests/e2e/test_lifecycle.py | 38 +++++++++++++----- lmi_tests/tests/unit/test_java_parity.py | 50 ++++++++++++++++++++++++ 5 files changed, 102 insertions(+), 12 deletions(-) diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 5183a9a9e..3d48b6141 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -133,6 +133,11 @@ the handler's result/operation sequence. Ordinary orchestration delays still use executor-starvation regression cannot hang the test process. Cloud progress tests have bounded driver budgets and the existing run-owned-function retirement path. +Healthy-peer cases checkpoint an admission decision before running fault work. +If an environment rotation places the victim elsewhere, it returns without +entering the scenario and the driver replaces the pair, up to three attempts. +Once admitted, lifecycle assertions are final and are never retried to get a pass. + The late-operation guard currently exposes an additional #741 scope-lifetime gap: a newly created operation can escape the SDK's set of already-known orphaned operation IDs. Its local regression is explicitly enabled with `lmi:regressions`; diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index c98af4417..1818856ad 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -44,9 +44,11 @@ def verify(self): ) verify(config, scaling, self.manifest, key) - def start(self, scenario, fixture="normal", gate=None): + def start(self, scenario, fixture="normal", gate=None, target_environment=None): marker = scenario + "-" + uuid.uuid4().hex[:12] payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} + if target_environment is not None: + payload["target_environment"] = target_environment if gate: payload["gate"] = gate self.hold(gate) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index d9033a861..f5dc44fd5 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -241,7 +241,22 @@ def mapped(branch): def workflow(event, context, trace): marker, scenario = event["marker"], event["scenario"] + admitted = False try: + if target := event.get("target_environment"): + # Record placement before any fault work. Replays consume the same + # decision even if Lambda later resumes in another environment. + accepted = context.step( + lambda _: trace.identity["environment"] == target, + name="placement-admission", + config=NO_RETRY, + ) + trace.emit( + "PLACEMENT_ACCEPTED" if accepted else "PLACEMENT_MISS", target=target + ) + if not accepted: + return marker + admitted = True value = context.step(lambda s: trace.body("success", marker, s), name="success") if scenario == "failure": raise ValueError("expected:" + marker) @@ -404,7 +419,7 @@ def retry(step): trace.emit("USER_RESULT", value=value) return value finally: - if scenario == "suspend-cleanup": + if scenario == "suspend-cleanup" and admitted: # Test-only root-finally diagnostics. This gate changes only exit # timing; it introduces no durable operations during suspension. trace.emit("CLEANUP_ENTER") diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py index eed607727..1253d5d25 100644 --- a/lmi_tests/tests/e2e/test_lifecycle.py +++ b/lmi_tests/tests/e2e/test_lifecycle.py @@ -175,13 +175,33 @@ def retry_observed(): # constrained to exactly-once under at-least-once step semantics. -def start_healthy_anchor(cloud): +def place_with_healthy_anchor(cloud, scenario): if cloud.manifest["concurrency"] == 1: - return None, None - gate = "anchor-" + uuid.uuid4().hex - item = cloud.start("barrier", gate=gate) - cloud.phase(item, "BLOCKED") - return item, gate + return None, None, cloud.start(scenario) + for _attempt in range(3): + gate = "anchor-" + uuid.uuid4().hex + anchor = cloud.start("barrier", gate=gate) + environment = cloud.phase(anchor, "BLOCKED")[0]["environment"] + victim = cloud.start(scenario, target_environment=environment) + admission = cloud.poll( + lambda: [ + e + for e in cloud.for_item(victim) + if e["phase"] in {"PLACEMENT_ACCEPTED", "PLACEMENT_MISS"} + ], + seconds=15, + category=evidence.PlacementError, + message="Victim placement was not established before fault admission", + )[0] + if admission["phase"] == "PLACEMENT_ACCEPTED": + return anchor, gate, victim + # Only a rejected precondition is retried. The victim has checkpointed + # admission=False and returns without executing the scenario. + cloud.finish(victim) + finish_anchor(cloud, anchor, gate) + raise evidence.PlacementError( + "No same-environment pair after three placement attempts" + ) def healthy_during(cloud, anchor, victim, boundary): @@ -206,8 +226,7 @@ def finish_anchor(cloud, anchor, gate): def test_pending_waits_for_root_finally_and_then_replays(cloud): - anchor, anchor_gate = start_healthy_anchor(cloud) - item = cloud.start("suspend-cleanup") + anchor, anchor_gate, item = place_with_healthy_anchor(cloud, "suspend-cleanup") blocked = cloud.phase(item, "BLOCKED")[0] healthy_during(cloud, anchor, item, blocked) gate = item["marker"] + "-cleanup" @@ -243,8 +262,7 @@ def test_nested_single_lane_pools_progress_for_all_runtime_workers(cloud): [("return-inflight", "SUCCEEDED"), ("failure-inflight", "FAILED")], ) def test_root_return_or_failure_settles_inflight_work(cloud, scenario, status): - anchor, anchor_gate = start_healthy_anchor(cloud) - item = cloud.start(scenario) + anchor, anchor_gate, item = place_with_healthy_anchor(cloud, scenario) cloud.phase(item, "WINNER_SELECTED") root_exit = cloud.phase(item, "USER_EXIT")[0] healthy_during(cloud, anchor, item, root_exit) diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py index c65724ccc..35012df9f 100644 --- a/lmi_tests/tests/unit/test_java_parity.py +++ b/lmi_tests/tests/unit/test_java_parity.py @@ -90,6 +90,56 @@ def entry(event, context): evidence.suspension_cleanup(controls.snapshot(), history) +@pytest.mark.parametrize( + "scenario", ["suspend-cleanup", "return-inflight", "failure-inflight"] +) +def test_placement_miss_runs_no_fault_or_finally_gate(scenario): + controls = Controls() + trace = controls.trace("local", scenario) + handler = durable_execution(lambda event, context: workflow(event, context, trace)) + with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: + result = runner.run( + input={ + "marker": "local", + "scenario": scenario, + "target_environment": "another-environment", + }, + execution_timeout=10, + ) + assert result.status.value == "SUCCEEDED" + phases = {e["phase"] for e in controls.snapshot()} + assert "PLACEMENT_MISS" in phases + assert not phases & {"BODY", "BLOCKED", "WINNER_READY", "CLEANUP_ENTER"} + + +def test_only_missed_admission_retries_environment_pair(monkeypatch): + from unittest.mock import Mock + from lmi_tests.tests.e2e import test_lifecycle + + cloud = Mock() + cloud.manifest = {"concurrency": 2} + first, missed, second, admitted = [ + dict(marker=name) for name in ("first", "missed", "second", "admitted") + ] + cloud.start.side_effect = [first, missed, second, admitted] + cloud.phase.side_effect = [[{"environment": "old"}], [{"environment": "new"}]] + cloud.for_item.side_effect = [ + [{"phase": "PLACEMENT_MISS"}], + [{"phase": "PLACEMENT_ACCEPTED"}], + ] + cloud.poll.side_effect = lambda predicate, **_: predicate() + finish = Mock() + monkeypatch.setattr(test_lifecycle, "finish_anchor", finish) + anchor, _gate, victim = test_lifecycle.place_with_healthy_anchor( + cloud, "return-inflight" + ) + assert anchor is second and victim is admitted + cloud.finish.assert_called_once_with(missed) + assert finish.call_count == 1 + assert cloud.start.call_args_list[1].kwargs["target_environment"] == "old" + assert cloud.start.call_args_list[3].kwargs["target_environment"] == "new" + + def run_inflight_case(scenario, status): controls = Controls() for suffix in ("loser", "loser-started"): From f385a1dfb24cdd91a5cbf3e3c00926b7919ce175 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 20:48:31 +0000 Subject: [PATCH 08/32] ci: serialize LMI cloud jobs across runs Hold a repository-wide concurrency slot through deployment, testing, and cleanup without increasing shared provider capacity. Retain up to 100 pending cloud jobs with queue: max instead of evicting older jobs. Keep the harness parallel and document the queue and repository scope. --- .github/workflows/lmi-e2e-tests.yml | 6 ++++++ lmi_tests/README.md | 15 ++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index a425001da..a2448801f 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -56,6 +56,12 @@ jobs: permissions: contents: read id-token: write + # All runs and matrix entries use the same provider. Hold this slot through + # cleanup; do not include a PR/ref/run ID or matrix value in the group. + concurrency: + group: lmi-e2e-shared-capacity-provider + cancel-in-progress: false + queue: max strategy: fail-fast: false max-parallel: 1 diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 3d48b6141..910f3f8b8 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -224,9 +224,18 @@ and on every push to `main`, including merged changes. There are no path filters label requirements, or ready-for-review requirements. `workflow_dispatch` remains available for manual reruns; there are no scheduled jobs. -Each workflow run has its own resources and runs its two matrix entries -sequentially. Runs do not share a GitHub concurrency group, so a new PR update or -main push cannot replace another commit's pending cloud job. +Each workflow run has its own resources. All cloud jobs share the repository-wide +`lmi-e2e-shared-capacity-provider` concurrency group, across PRs, main pushes, +manual runs, and both matrix entries. Only one cloud job can deploy, test, or clean +up at a time; the slot is held until the entire job finishes. Harness jobs can run +in parallel. The capacity provider's configuration is not changed. + +`cancel-in-progress: false` preserves running jobs, and +[`queue: max`](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency#example-queueing-multiple-pending-runs) +retains up to 100 pending jobs instead of replacing an older pending job on each update. GitHub +cancels additional arrivals when that queue is full. This repository-wide group +does not coordinate manual local deployments or jobs in other repositories using +the same provider; those consumers must also leave sufficient capacity available. Privileged cloud jobs retain the repository's existing restrictions for forked PRs and Dependabot; the harness still runs for those PRs. Cloud jobs reuse From 31d9a307f54407eb4ac9a99bfa6fbb654e726dff Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:47:31 +0000 Subject: [PATCH 09/32] test: share LMI functions in one deployment Deploy one function per native concurrency setting and reuse it for ordinary and deadline scenarios. Run all 28 cases in one cloud job without changing provider capacity or losing regression assertions. Release controls, stop remaining executions, and observe wrapper exit before reusing each function. Extend fault observation for the shared 60-second timeout and retain a bounded emergency release. --- .github/workflows/lmi-e2e-tests.yml | 24 +++--- lmi_tests/README.md | 59 +++++++++----- lmi_tests/cloud.py | 88 +++++++++++++++++++-- lmi_tests/deploy.py | 31 ++++---- lmi_tests/fixture.py | 4 +- lmi_tests/summary.py | 4 +- lmi_tests/tests/e2e/conftest.py | 19 ++++- lmi_tests/tests/e2e/test_lifecycle.py | 28 +++---- lmi_tests/tests/unit/test_cloud.py | 99 ++++++++++++++++++++++-- lmi_tests/tests/unit/test_control.py | 25 +++++- lmi_tests/tests/unit/test_deploy.py | 36 +++++---- lmi_tests/tests/unit/test_java_parity.py | 2 +- 12 files changed, 317 insertions(+), 102 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index a2448801f..81d781a00 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -43,7 +43,7 @@ jobs: path: lmi_tests/artifacts/ cloud: - name: cloud (Python 3.14, concurrency ${{ matrix.concurrency }}) + name: cloud (Python 3.14, concurrency 1 and 2) needs: harness # Run on every main push and trusted PR update, including drafts. # Match the repository's cloud credential restrictions for forks/Dependabot. @@ -52,21 +52,16 @@ jobs: (github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest - timeout-minutes: 55 + timeout-minutes: 70 permissions: contents: read id-token: write - # All runs and matrix entries use the same provider. Hold this slot through - # cleanup; do not include a PR/ref/run ID or matrix value in the group. + # All runs use the same provider. Hold this slot through cleanup; do not + # include a PR/ref/run ID in the group. concurrency: group: lmi-e2e-shared-capacity-provider cancel-in-progress: false queue: max - strategy: - fail-fast: false - max-parallel: 1 - matrix: - concurrency: [1, 2] env: AWS_REGION: ${{ inputs.region || 'us-west-2' }} CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} @@ -94,21 +89,20 @@ jobs: run: | hatch run lmi:build hatch run lmi:python -m lmi_tests.deploy reconcile - - name: Deploy verified LMI functions + - name: Deploy both shared LMI functions once timeout-minutes: 28 env: LMI_RUNTIME: python3.14 - LMI_CONCURRENCY: ${{ matrix.concurrency }} run: | hatch run lmi:python -m lmi_tests.deploy deploy \ --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ - --runtime "$LMI_RUNTIME" --concurrency "$LMI_CONCURRENCY" + --runtime "$LMI_RUNTIME" - name: Run cloud regressions (red until issue 741 is fixed) - timeout-minutes: 12 + timeout-minutes: 20 run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml - name: Collect service history and runtime evidence if: always() - timeout-minutes: 3 + timeout-minutes: 5 run: | if test -f lmi_tests/artifacts/manifest.json; then hatch run lmi:python -m lmi_tests.deploy collect @@ -124,7 +118,7 @@ jobs: if: always() uses: actions/upload-artifact@v7 with: - name: lmi-3.14-c${{ matrix.concurrency }}-${{ github.run_id }}-${{ github.run_attempt }} + name: lmi-3.14-${{ github.run_id }}-${{ github.run_attempt }} retention-days: 7 path: lmi_tests/artifacts/ - name: Summarize outcomes diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 910f3f8b8..df337f044 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -66,7 +66,7 @@ is substituted. The artifact hash, SDK wheel version, and commit are retained. hatch run lmi:build hatch run lmi:python -m lmi_tests.deploy reconcile hatch run lmi:python -m lmi_tests.deploy deploy \ - --run-id local-20260923-a --runtime python3.14 --concurrency 2 + --run-id local-20260923-a --runtime python3.14 hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml # Run both commands below even when a scenario fails: hatch run lmi:python -m lmi_tests.deploy collect @@ -76,14 +76,22 @@ python -m lmi_tests.summary For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls `collect` followed by `cleanup`; the workflow already has unconditional steps. -Use a fresh run ID for each deployment. Run both environment-concurrency settings, -`1` and `2`, with Python 3.14, as CI does. The LMI suite and its Hatch environment -target only Python 3.14. Independent SDK branch-concurrency settings include 1, 2, -and 3. - -Each combination creates a unique stack, private bucket, log groups, and two -functions (`normal`, `deadline`). Both functions declare native scaling limits of -exactly one environment and use 2 GiB / 1 vCPU. LMI automatically publishes +Use a fresh run ID for each deployment. This one deployment supplies both native +environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud +command runs all 14 scenarios for each setting (28 cases), sequentially, without +redeploying or changing function configuration between cases. Independent SDK +branch-concurrency settings include 1, 2, and 3. + +Each run creates one unique stack, one private bucket, and two functions/log +groups (`c1`, `c2`). All scenarios, including deadlines and healthy-peer probes, +reuse the function for their concurrency setting and the same handler artifact. +Both functions use a 60-second invocation timeout. Deadline tests wait for the +real platform timeout, leaving ordinary scenarios their existing invocation +budget. A single fixed function configuration cannot exercise both native +concurrency limits; merely sending one request to `c2` would not establish `c1`. + +Both functions declare native scaling limits of exactly one environment and use +2 GiB / 1 vCPU. LMI automatically publishes `$LATEST.PUBLISHED`; every invocation uses that qualified target. Creating an extra numbered version would allocate another set of environments. Before assertions, the suite checks Active state, runtime/architecture, durability, timeout, provider, @@ -91,8 +99,9 @@ process concurrency, applied scaling limits, commit/run identity, and code hash. Unsupported configurations, insufficient permissions, and unavailable capacity fail provisioning. There is no fallback to standard Lambda and no passing cloud skip. -The normal handler mapping is saved in `artifacts/function-name-map.json`, using -the existing `PYTEST_FUNCTION_NAME_MAP` convention. The test driver reuses +The handler mappings are saved under `c1` and `c2` in +`artifacts/function-name-map.json`; each entry has the existing +`PYTEST_FUNCTION_NAME_MAP` shape. The test driver reuses `DurableFunctionCloudTestRunner` for async invocation, callback completion, and waiting for results, with finite boto transport timeouts. @@ -200,13 +209,14 @@ transport timeout). Invocation and callback writes are not retried by this layer | Budget | Default | | --- | --- | -| Lambda invocation | Normal 60 s; deadline fixture 10 s | +| Lambda invocation | 60 s for all scenarios, including deadlines | | Durable logical execution | 240 s | | Driver result polling | 120 s | | Cleanup acceptance grace | 5 s | -| Fault-fixture emergency I/O release | 75 s | -| Workflow cloud assertions | 12 min | -| Matrix job including provisioning/retirement | 55 min | +| Fault-fixture emergency I/O release | 150 s | +| Per-case retirement after assertions | 30 s | +| Workflow cloud assertions (all 28 cases) | 20 min | +| Single cloud job including provisioning/retirement | 70 min | Artifacts contain JUnit, configuration/provider readbacks, qualified targets, commit/wheel/code hash, invocation inputs/ARNs, all lifecycle and side-effect records, @@ -218,16 +228,17 @@ An unexecuted scenario is never summarized as passing. ## CI and resource ownership -The dedicated workflow runs the harness and full LMI cloud matrix automatically +The dedicated workflow runs the harness and full LMI suite automatically when a same-repository PR is opened, updated, or reopened (including Draft PRs), and on every push to `main`, including merged changes. There are no path filters, label requirements, or ready-for-review requirements. `workflow_dispatch` remains available for manual reruns; there are no scheduled jobs. -Each workflow run has its own resources. All cloud jobs share the repository-wide -`lmi-e2e-shared-capacity-provider` concurrency group, across PRs, main pushes, -manual runs, and both matrix entries. Only one cloud job can deploy, test, or clean -up at a time; the slot is held until the entire job finishes. Harness jobs can run +Each workflow run has one harness job and one cloud job with its own resources. +All cloud jobs share the repository-wide `lmi-e2e-shared-capacity-provider` +concurrency group, across PRs, main pushes, and manual runs. Only one cloud job can +deploy, test, or clean up at a time; the slot is held until the entire job finishes. +Harness jobs can run in parallel. The capacity provider's configuration is not changed. `cancel-in-progress: false` preserves running jobs, and @@ -243,6 +254,14 @@ and Dependabot; the harness still runs for those PRs. Cloud jobs reuse The #741 regression assertions remain visibly failing until its fix lands; they are not skipped or converted into expected-success results to keep CI green. +After each case's assertions, teardown releases that case's controls, stops any +remaining logical executions to prevent retries, and waits for every observed +wrapper to return or raise. A terminal service status alone does not prove worker +retirement. If retirement fails, the next case must first retire that pending work +before invoking the shared function. Collection remains scoped to the case, and +the final collector gathers the entire run. These post-assertion releases and +stops cannot satisfy a deadline, cleanup-grace, or capacity-recovery assertion. + The workflow collects evidence before teardown, even after test failure or ordinary cancellation. Cleanup sends external releases, stops running durable executions, deletes the run's functions to retire their environments, empties the bucket, and diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 1818856ad..b9f96fc0c 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -10,8 +10,13 @@ from botocore.exceptions import ClientError from aws_durable_execution_sdk_python_testing import DurableFunctionCloudTestRunner -from lmi_tests.deploy import ARTIFACTS, QUALIFIER, client, save, verify -from lmi_tests.evidence import CollectionError, check_controls, select +from lmi_tests.deploy import ARTIFACTS, CONCURRENCIES, QUALIFIER, client, save, verify +from lmi_tests.evidence import ( + CollectionError, + ProvisioningError, + check_controls, + select, +) class RunnerClient: @@ -28,8 +33,9 @@ def get_durable_execution_history(self, **kwargs): class Cloud: - def __init__(self, manifest): + def __init__(self, manifest, configuration=None): self.manifest = manifest + self.configuration = configuration self.lam, self.s3, self.logs = client("lambda"), client("s3"), client("logs") self.events = {} self.invocations = [] @@ -37,6 +43,12 @@ def __init__(self, manifest): self._next_history_read = 0.0 def verify(self): + if self.manifest.get("concurrencies") != CONCURRENCIES or set( + self.manifest["functions"] + ) != set(CONCURRENCIES): + raise ProvisioningError( + "One deployment must contain both c1 and c2 functions" + ) for key, arn in self.manifest["functions"].items(): config = self.lam.get_function_configuration(FunctionName=arn) scaling = self.lam.get_function_scaling_config( @@ -44,7 +56,15 @@ def verify(self): ) verify(config, scaling, self.manifest, key) - def start(self, scenario, fixture="normal", gate=None, target_environment=None): + @property + def concurrency(self): + return self.manifest["concurrencies"][self.configuration] + + def start(self, scenario, gate=None, target_environment=None): + if self.configuration not in self.manifest["functions"]: + raise ProvisioningError( + "Select a deployed concurrency configuration before Invoke" + ) marker = scenario + "-" + uuid.uuid4().hex[:12] payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} if target_environment is not None: @@ -67,7 +87,8 @@ def start(self, scenario, fixture="normal", gate=None, target_environment=None): elif scenario == "suspend-cleanup": self.hold(marker + "-cleanup") runner = DurableFunctionCloudTestRunner( - self.manifest["functions"][fixture], region=self.manifest["region"] + self.manifest["functions"][self.configuration], + region=self.manifest["region"], ) runner.lambda_client = RunnerClient(self) try: @@ -83,7 +104,7 @@ def start(self, scenario, fixture="normal", gate=None, target_environment=None): item = { "marker": marker, "arn": arn, - "fixture": fixture, + "configuration": self.configuration, "runner": runner, "started": time.time(), } @@ -267,7 +288,7 @@ def observed(): return self.poll( observed, - seconds=40, + seconds=self.manifest["invocationTimeout"] + 40, message="No service invocation timeout evidence for " + request, ) @@ -320,3 +341,56 @@ def release_all(self): for gate in self.gates: self.release(gate) self.gates.clear() + + def settle_case(self): + """Release fault work and drain this case before reusing its function.""" + self.release_all() + pending = [] + for item in self.invocations: + result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) + if result["Status"] == "RUNNING": + try: + self.lam.stop_durable_execution(DurableExecutionArn=item["arn"]) + pending.append(item) + except ClientError as error: + if ( + error.response["Error"]["Code"] + != "InvalidParameterValueException" + ): + raise + # Completion can race the stop request. A still-running + # execution must not be ignored or leak into the next case. + if ( + self.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ + "Status" + ] + == "RUNNING" + ): + raise + + def settled(): + pending[:] = [ + item + for item in pending + if self.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ + "Status" + ] + == "RUNNING" + ] + if pending: + return False + events = self.refresh(validate_controls=False) + entered = {e["request"] for e in select(events, "WRAPPER_ENTER")} + exited = { + e["request"] + for e in events + if e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + } + return entered <= exited + + if self.invocations: + self.poll( + settled, + seconds=30, + message="Released case still owns runtime workers; shared function is not ready for reuse", + ) diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 74d5c370c..97c3946ed 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -27,6 +27,7 @@ OWNER = "python-sdk-lmi-e2e" SCALING = {"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1} QUALIFIER = "$LATEST.PUBLISHED" +CONCURRENCIES = {"c1": 1, "c2": 2} def client(service, region=None): @@ -171,10 +172,7 @@ def template(manifest, functions=True): } outputs = {} if functions: - for key, timeout in ( - ("normal", manifest["invocationTimeout"]), - ("deadline", manifest["deadlineTimeout"]), - ): + for key, concurrency in manifest["concurrencies"].items(): name = manifest["stack"] + "-" + key resources[key + "Logs"] = { "Type": "AWS::Logs::LogGroup", @@ -193,7 +191,7 @@ def template(manifest, functions=True): "Handler": "lmi_tests.fixture.handler", "Role": manifest["role"], "MemorySize": 2048, - "Timeout": timeout, + "Timeout": manifest["invocationTimeout"], "Code": {"S3Bucket": bucket, "S3Key": manifest["codeKey"]}, "FunctionScalingConfig": SCALING, "DurableConfig": { @@ -203,9 +201,7 @@ def template(manifest, functions=True): "CapacityProviderConfig": { "LambdaManagedInstancesCapacityProviderConfig": { "CapacityProviderArn": manifest["provider"], - "PerExecutionEnvironmentMaxConcurrency": manifest[ - "concurrency" - ], + "PerExecutionEnvironmentMaxConcurrency": concurrency, "ExecutionEnvironmentMemoryGiBPerVCpu": 2, } }, @@ -270,7 +266,8 @@ def verify(config, scaling, manifest, key): ) checks = [ actual.get("CapacityProviderArn") == manifest["provider"], - actual.get("PerExecutionEnvironmentMaxConcurrency") == manifest["concurrency"], + actual.get("PerExecutionEnvironmentMaxConcurrency") + == manifest["concurrencies"][key], config.get("Runtime") == manifest["runtime"], config.get("Architectures") == ["arm64"], config.get("Version") == QUALIFIER, @@ -279,8 +276,7 @@ def verify(config, scaling, manifest, key): config.get("State") == "Active", config.get("DurableConfig", {}).get("ExecutionTimeout") == manifest["executionTimeout"], - config.get("Timeout") - == manifest["deadlineTimeout" if key == "deadline" else "invocationTimeout"], + config.get("Timeout") == manifest["invocationTimeout"], scaling.get("AppliedFunctionScalingConfig") == SCALING, config.get("Environment", {}).get("Variables", {}).get("LMI_COMMIT") == manifest["commit"], @@ -327,9 +323,7 @@ def deploy(args): built = json.loads((ARTIFACTS / "build.json").read_text()) if built["sha256"] != digest.hex(): raise ProvisioningError("Artifact changed since build") - suffix = ( - args.runtime.replace("python", "").replace(".", "") + f"-c{args.concurrency}" - ) + suffix = args.runtime.replace("python", "").replace(".", "") name = f"py-lmi-{args.run_id}-{suffix}" manifest = { "run": args.run_id, @@ -340,12 +334,11 @@ def deploy(args): "provider": provider, "role": os.environ["TEST_LAMBDA_EXECUTION_ROLE_ARN"], "runtime": args.runtime, - "concurrency": args.concurrency, + "concurrencies": CONCURRENCIES, "commit": built["commit"], "codeKey": "code/" + digest.hex() + ".zip", "codeSha256": base64.b64encode(digest).decode(), "invocationTimeout": 60, - "deadlineTimeout": 10, "executionTimeout": 240, "driverTimeout": 120, "cleanupGrace": 5, @@ -394,7 +387,10 @@ def deploy(args): verify(config, scaling, manifest, key) save( ARTIFACTS / "function-name-map.json", - {"lmi_tests.fixture.handler": manifest["functions"]["normal"]}, + { + key: {"lmi_tests.fixture.handler": arn} + for key, arn in manifest["functions"].items() + }, ) @@ -531,7 +527,6 @@ def main(): ) parser.add_argument("--run-id") parser.add_argument("--runtime", choices=["python3.14"], default="python3.14") - parser.add_argument("--concurrency", type=int, choices=[1, 2], default=2) args = parser.parse_args() try: if args.command == "build": diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index f5dc44fd5..3fc56caf5 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -121,7 +121,9 @@ def released(self, name): raise def gate(self, name, *, effects=False, attempt=1, ready=None): - end = time.monotonic() + 75 + # Leave time to observe the real 60-second invocation timeout and + # recovery. This safety escape must never masquerade as SDK cancellation. + end = time.monotonic() + 150 entered = False try: while not self.released(name) and not self.released("release-all"): diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py index 37fba95cb..ae5b60a66 100644 --- a/lmi_tests/summary.py +++ b/lmi_tests/summary.py @@ -12,10 +12,10 @@ def summarize(directory): if manifest.exists(): config = json.loads(manifest.read_text()) lines.append( - f"Commit `{config['commit']}`; {config['runtime']}; environment concurrency {config['concurrency']}." + f"Commit `{config['commit']}`; {config['runtime']}; one deployment, environment concurrency {list(config['concurrencies'].values())}." ) lines.append( - f"Timeouts: invocation {config['deadlineTimeout']}s (fault) / {config['invocationTimeout']}s (normal), durable execution {config['executionTimeout']}s, driver {config['driverTimeout']}s; cleanup grace {config['cleanupGrace']}s." + f"Timeouts: invocation {config['invocationTimeout']}s (all scenarios), durable execution {config['executionTimeout']}s, driver {config['driverTimeout']}s; cleanup grace {config['cleanupGrace']}s." ) report = directory / "cloud.xml" if report.exists(): diff --git a/lmi_tests/tests/e2e/conftest.py b/lmi_tests/tests/e2e/conftest.py index cae08df4b..899949e98 100644 --- a/lmi_tests/tests/e2e/conftest.py +++ b/lmi_tests/tests/e2e/conftest.py @@ -5,7 +5,7 @@ import pytest from lmi_tests.cloud import Cloud -from lmi_tests.deploy import ARTIFACTS +from lmi_tests.deploy import ARTIFACTS, CONCURRENCIES from lmi_tests.evidence import CollectionError, PlacementError, ProvisioningError @@ -41,13 +41,24 @@ def deployment(request): return manifest +@pytest.fixture(scope="session", params=CONCURRENCIES) +def configuration(request, deployment): + return {"key": request.param, "pending": None} + + @pytest.fixture -def cloud(deployment): - driver = Cloud(deployment) +def cloud(deployment, configuration): + # A failed teardown cannot silently hand a still-busy function to another + # case. Retry only retirement; never retry the regression assertions. + if configuration["pending"] is not None: + configuration["pending"].settle_case() + driver = Cloud(deployment, configuration["key"]) + configuration["pending"] = driver try: yield driver finally: try: - driver.release_all() + driver.settle_case() + configuration["pending"] = None finally: driver.collect(full_run=False) diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py index 1253d5d25..525dc15eb 100644 --- a/lmi_tests/tests/e2e/test_lifecycle.py +++ b/lmi_tests/tests/e2e/test_lifecycle.py @@ -20,10 +20,10 @@ def test_checkpoint_replay_callback_and_retry(cloud): evidence.replay(cloud.for_item(item), history, item["marker"]) -def capacity_barrier(cloud, fixture="normal", environment=None, seconds=30): - count = cloud.manifest["concurrency"] +def capacity_barrier(cloud, environment=None, seconds=30): + count = cloud.concurrency gate = "capacity-" + uuid.uuid4().hex - items = [cloud.start("barrier", fixture, gate) for _ in range(count)] + items = [cloud.start("barrier", gate=gate) for _ in range(count)] env, events = wait_overlap(cloud, items, count, environment, seconds) cloud.release(gate) for item in items: @@ -104,11 +104,11 @@ def test_real_synchronous_checkpoint_settles_before_branch_join(cloud): def test_invocation_deadline_isolation_and_capacity_recovery(cloud): - fault = cloud.start("deadline", "deadline", "fault-" + uuid.uuid4().hex) + fault = cloud.start("deadline", gate="fault-" + uuid.uuid4().hex) blocked = cloud.phase(fault, "BLOCKED")[0] - if cloud.manifest["concurrency"] > 1: + if cloud.concurrency > 1: healthy_gate = "healthy-" + uuid.uuid4().hex - healthy = cloud.start("barrier", "deadline", healthy_gate) + healthy = cloud.start("barrier", gate=healthy_gate) cloud.phase(healthy, "BLOCKED") wait_overlap(cloud, [fault, healthy], 2, blocked["environment"]) cloud.release(healthy_gate) @@ -117,14 +117,15 @@ def test_invocation_deadline_isolation_and_capacity_recovery(cloud): # Queue recovery demand at the runtime deadline, before waiting for the # eventually consistent service history. History delivery latency must not # consume the worker recovery budget. - cloud.poll(lambda: time.time() >= blocked["deadline"] - 1, seconds=15) + cloud.poll( + lambda: time.time() >= blocked["deadline"] - 1, + seconds=cloud.manifest["invocationTimeout"] + 5, + ) # Put every worker slot under demand without releasing the fault. A different # environment cannot satisfy recovery, even if the service replaces capacity. recovery_started = time.time() try: - _env, probes = capacity_barrier( - cloud, "deadline", blocked["environment"], seconds=15 - ) + _env, probes = capacity_barrier(cloud, blocked["environment"], seconds=15) finally: cloud.platform_timeout(fault, blocked["request"]) bound = blocked["deadline"] + cloud.manifest["cleanupGrace"] @@ -142,7 +143,7 @@ def test_invocation_deadline_isolation_and_capacity_recovery(cloud): def test_service_timeout_retry_does_not_repeat_completed_step(cloud): - fault = cloud.start("deadline", "deadline", "retry-" + uuid.uuid4().hex) + fault = cloud.start("deadline", gate="retry-" + uuid.uuid4().hex) first = cloud.phase(fault, "BLOCKED")[0] cloud.platform_timeout(fault, first["request"]) # Diagnose stale side effects before waiting for a retry that may itself be @@ -176,7 +177,7 @@ def retry_observed(): def place_with_healthy_anchor(cloud, scenario): - if cloud.manifest["concurrency"] == 1: + if cloud.concurrency == 1: return None, None, cloud.start(scenario) for _attempt in range(3): gate = "anchor-" + uuid.uuid4().hex @@ -240,8 +241,7 @@ def test_pending_waits_for_root_finally_and_then_replays(cloud): def test_nested_single_lane_pools_progress_for_all_runtime_workers(cloud): gate = "progress-" + uuid.uuid4().hex items = [ - cloud.start("nested-progress", gate=gate) - for _ in range(cloud.manifest["concurrency"]) + cloud.start("nested-progress", gate=gate) for _ in range(cloud.concurrency) ] wait_overlap(cloud, items, len(items)) cloud.release(gate) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index d861df583..926b61936 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -31,35 +31,124 @@ def driver(monkeypatch, tmp_path): driver = module.Cloud( { "functions": { - "normal": "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" + "c1": "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" }, "region": "us-west-2", "run": "unit", "bucket": "unit-bucket", "commit": "commit", - } + "invocationTimeout": 60, + "concurrencies": {"c1": 1}, + }, + "c1", ) driver.s3 = Mock() return driver -def test_async_cloud_runner_sends_json_to_qualified_lmi_target(driver): +@pytest.mark.parametrize("configuration", ["c1", "c2"]) +@pytest.mark.parametrize("scenario", ["success", "deadline"]) +def test_scenarios_reuse_the_qualified_function_for_their_concurrency( + driver, configuration, scenario +): + driver.configuration = configuration + driver.manifest["concurrencies"] = module.CONCURRENCIES + driver.manifest["functions"]["c2"] = ( + "arn:aws:lambda:us-west-2:123456789012:function:lmi-c2:$LATEST.PUBLISHED" + ) + assert driver.concurrency == module.CONCURRENCIES[configuration] arn = "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED/durable-execution/name/id" with Stubber(driver.lam) as stub: stub.add_response( "invoke", {"StatusCode": 202, "DurableExecutionArn": arn}, { - "FunctionName": driver.manifest["functions"]["normal"], + "FunctionName": driver.manifest["functions"][configuration], "InvocationType": "Event", "Payload": ANY, }, ) - item = driver.start("success") + item = driver.start(scenario, gate="fault" if scenario == "deadline" else None) assert item["arn"] == arn stub.assert_no_pending_responses() +def test_verify_rejects_a_deployment_missing_a_concurrency_configuration(driver): + driver.manifest["concurrencies"] = module.CONCURRENCIES + with pytest.raises(module.ProvisioningError, match="both c1 and c2"): + driver.verify() + + +def test_settlement_releases_controls_stops_retries_and_waits_for_every_wrapper(driver): + driver.invocations = [{"arn": "fault"}, {"arn": "completed"}] + calls = [] + driver.release_all = Mock(side_effect=lambda: calls.append("release")) + statuses = {"fault": "RUNNING", "completed": "SUCCEEDED"} + driver.lam.get_durable_execution = Mock( + side_effect=lambda **kw: {"Status": statuses[kw["DurableExecutionArn"]]} + ) + + def stop(**kw): + calls.append("stop") + statuses[kw["DurableExecutionArn"]] = "STOPPED" + + driver.lam.stop_durable_execution = Mock(side_effect=stop) + events = [ + {"phase": "WRAPPER_ENTER", "request": "first"}, + {"phase": "WRAPPER_ENTER", "request": "retry"}, + {"phase": "WRAPPER_RETURN", "request": "first"}, + ] + driver.refresh = Mock(return_value=events) + + def poll(predicate, **_): + # A terminal durable execution does not prove a stopped LMI worker. + assert not predicate() + events.append({"phase": "WRAPPER_RAISE", "request": "retry"}) + assert predicate() + calls.append("settled") + + driver.poll = poll + driver.settle_case() + assert calls == ["release", "stop", "settled"] + driver.lam.stop_durable_execution.assert_called_once_with( + DurableExecutionArn="fault" + ) + + +def test_settlement_reports_a_worker_still_running_after_release(driver): + driver.invocations = [{"arn": "execution"}] + driver.lam.get_durable_execution = Mock(return_value={"Status": "STOPPED"}) + driver.refresh = Mock(return_value=[{"phase": "WRAPPER_ENTER", "request": "stuck"}]) + + def poll(predicate, **kwargs): + assert not predicate() + raise CollectionError(kwargs["message"]) + + driver.poll = poll + with pytest.raises(CollectionError, match="not ready for reuse"): + driver.settle_case() + + +@pytest.mark.parametrize("status", ["SUCCEEDED", "RUNNING"]) +def test_stop_race_is_ignored_only_after_confirmed_completion(driver, status): + driver.invocations = [{"arn": "execution"}] + driver.lam.get_durable_execution = Mock( + side_effect=[{"Status": "RUNNING"}, {"Status": status}] + ) + driver.lam.stop_durable_execution = Mock( + side_effect=ClientError( + {"Error": {"Code": "InvalidParameterValueException"}}, + "StopDurableExecution", + ) + ) + driver.refresh = Mock(return_value=[]) + if status == "RUNNING": + with pytest.raises(ClientError): + driver.settle_case() + else: + driver.settle_case() + + def test_missing_execution_arn_is_collection_error_not_pass(driver): with Stubber(driver.lam) as stub: stub.add_response("invoke", {"StatusCode": 202}) diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py index edab68e03..76a12e22f 100644 --- a/lmi_tests/tests/unit/test_control.py +++ b/lmi_tests/tests/unit/test_control.py @@ -5,6 +5,7 @@ import threading import time from concurrent.futures import ThreadPoolExecutor +from types import SimpleNamespace from unittest.mock import Mock import boto3 @@ -13,7 +14,7 @@ import pytest from lmi_tests.fixture import Trace, workflow -from lmi_tests import evidence +from lmi_tests import evidence, fixture from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context from aws_durable_execution_sdk_python import durable_execution @@ -67,6 +68,28 @@ def test_invalid_control_content_cannot_release_work(trace): assert trace.emit.call_args.args[0] == "CONTROL_ERROR" +@pytest.mark.parametrize("elapsed,escaped", [(90, False), (180, True)]) +def test_fault_gate_outlives_invocation_deadline_but_remains_bounded( + trace, monkeypatch, elapsed, escaped +): + trace.deadline = 60 + times = iter([0, elapsed]) + trace.released = Mock(side_effect=[False, False, True]) + monkeypatch.setattr( + fixture, + "time", + SimpleNamespace(monotonic=lambda: next(times), sleep=lambda _: None), + ) + if escaped: + with pytest.raises(TimeoutError, match="emergency release"): + trace.gate("fault", effects=True) + else: + trace.gate("fault", effects=True) + phases = [call.args[0] for call in trace.emit.call_args_list] + assert ("ESCAPE" in phases) is escaped + assert ("EFFECT" in phases) is not escaped + + def test_winner_is_signalled_only_after_loser_actually_enters_held_io(trace): controls = { "control/loser": b"hold", diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index 85dded018..950f66d9e 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -11,30 +11,35 @@ def manifest(): return { "run": "unit", - "stack": "py-lmi-unit-314-c2", + "stack": "py-lmi-unit-314", "bucket": "test-bucket", "role": "role", "provider": "provider", "runtime": "python3.14", - "concurrency": 2, + "concurrencies": deploy.CONCURRENCIES, "commit": "sha", "codeKey": "code/sha.zip", "codeSha256": "hash", "owned": True, "invocationTimeout": 60, - "deadlineTimeout": 10, "executionTimeout": 240, } -@pytest.mark.parametrize("concurrency", [1, 2]) -def test_template_sets_native_scaling_durability_and_published_target( - manifest, concurrency -): +def test_one_deployment_shares_two_functions_across_all_scenarios(manifest): runtime = "python3.14" - manifest.update(concurrency=concurrency) template = deploy.template(manifest) - for key in ("normal", "deadline"): + functions = { + key: resource + for key, resource in template["Resources"].items() + if resource["Type"] == "AWS::Lambda::Function" + } + assert set(functions) == {"c1Function", "c2Function"} + assert ( + functions["c1Function"]["Properties"]["Code"] + == functions["c2Function"]["Properties"]["Code"] + ) + for key, concurrency in deploy.CONCURRENCIES.items(): config = template["Resources"][key + "Function"]["Properties"] assert config["FunctionScalingConfig"] == deploy.SCALING assert config["Runtime"] == runtime @@ -45,7 +50,8 @@ def test_template_sets_native_scaling_durability_and_published_target( == concurrency ) assert config["DurableConfig"]["ExecutionTimeout"] == 240 - assert config["Timeout"] == (10 if key == "deadline" else 60) + assert config["Timeout"] == 60 + assert config["Handler"] == "lmi_tests.fixture.handler" assert ( template["Outputs"][key]["Value"]["Fn::Join"][1][-1] == ":$LATEST.PUBLISHED" ) @@ -59,11 +65,13 @@ def test_verify_rejects_standard_lambda_wrong_artifact_and_unbounded_placement( manifest, ): config = copy.deepcopy( - deploy.template(manifest)["Resources"]["normalFunction"]["Properties"] + deploy.template(manifest)["Resources"]["c1Function"]["Properties"] ) config.update(State="Active", Version=deploy.QUALIFIER, CodeSha256="hash") scaling = {"AppliedFunctionScalingConfig": deploy.SCALING} - deploy.verify(config, scaling, manifest, "normal") + deploy.verify(config, scaling, manifest, "c1") + with pytest.raises(ProvisioningError): + deploy.verify(config, scaling, manifest, "c2") for field in ( "CapacityProviderConfig", "CodeSha256", @@ -75,13 +83,13 @@ def test_verify_rejects_standard_lambda_wrong_artifact_and_unbounded_placement( bad = copy.deepcopy(config) del bad[field] with pytest.raises(ProvisioningError): - deploy.verify(bad, scaling, manifest, "normal") + deploy.verify(bad, scaling, manifest, "c1") with pytest.raises(ProvisioningError): deploy.verify( config, {"AppliedFunctionScalingConfig": {"MaxExecutionEnvironments": 2}}, manifest, - "normal", + "c1", ) diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py index 35012df9f..80ece91a5 100644 --- a/lmi_tests/tests/unit/test_java_parity.py +++ b/lmi_tests/tests/unit/test_java_parity.py @@ -117,7 +117,7 @@ def test_only_missed_admission_retries_environment_pair(monkeypatch): from lmi_tests.tests.e2e import test_lifecycle cloud = Mock() - cloud.manifest = {"concurrency": 2} + cloud.concurrency = 2 first, missed, second, admitted = [ dict(marker=name) for name in ("first", "missed", "second", "admitted") ] From 7bd33b18c5320effbacb86206c8ab12ed9c8ddc3 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Wed, 23 Sep 2026 21:57:54 +0000 Subject: [PATCH 10/32] test: retain LMI functions across runs Create the persistent stack once and update the same two functions after previous durable executions finish. Preserve failed stacks, code artifacts, and infrastructure after every test run. Isolate controls and events by run ID, expire only run data, and scope cleanup to the current run without deleting functions or buckets. Remove the infrastructure janitor and verify create/update behavior. --- .github/workflows/lmi-e2e-tests.yml | 17 +- lmi_tests/README.md | 63 +++--- lmi_tests/cloud.py | 37 +++- lmi_tests/deploy.py | 258 ++++++++++++----------- lmi_tests/fixture.py | 12 +- lmi_tests/summary.py | 4 +- lmi_tests/tests/unit/test_cloud.py | 58 ++++- lmi_tests/tests/unit/test_control.py | 34 +-- lmi_tests/tests/unit/test_deploy.py | 257 ++++++++++++++++------ lmi_tests/tests/unit/test_java_parity.py | 13 +- 10 files changed, 494 insertions(+), 259 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 81d781a00..a128f11c1 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -52,7 +52,7 @@ jobs: (github.actor != 'dependabot[bot]' && github.event.pull_request.head.repo.full_name == github.repository) runs-on: ubuntu-latest - timeout-minutes: 70 + timeout-minutes: 80 permissions: contents: read id-token: write @@ -63,6 +63,7 @@ jobs: cancel-in-progress: false queue: max env: + LMI_STACK_NAME: python-lmi-e2e AWS_REGION: ${{ inputs.region || 'us-west-2' }} CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} TEST_ACCOUNT_ID: ${{ secrets.TEST_ACCOUNT_ID }} @@ -84,19 +85,17 @@ jobs: test -n "$TEST_LAMBDA_EXECUTION_ROLE_ARN" test "$(aws sts get-caller-identity --query Account --output text)" = "$TEST_ACCOUNT_ID" - run: pip install hatch==1.16.5 - - name: Build SDK checkout and reconcile expired test stacks + - name: Build SDK checkout timeout-minutes: 5 - run: | - hatch run lmi:build - hatch run lmi:python -m lmi_tests.deploy reconcile - - name: Deploy both shared LMI functions once - timeout-minutes: 28 + run: hatch run lmi:build + - name: Create or update persistent LMI functions + timeout-minutes: 35 env: LMI_RUNTIME: python3.14 run: | hatch run lmi:python -m lmi_tests.deploy deploy \ --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ - --runtime "$LMI_RUNTIME" + --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" - name: Run cloud regressions (red until issue 741 is fixed) timeout-minutes: 20 run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml @@ -107,7 +106,7 @@ jobs: if test -f lmi_tests/artifacts/manifest.json; then hatch run lmi:python -m lmi_tests.deploy collect fi - - name: Release blocked work and retire run-owned resources + - name: Release this run's work and retain persistent functions if: always() timeout-minutes: 5 run: | diff --git a/lmi_tests/README.md b/lmi_tests/README.md index df337f044..465e99671 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -31,7 +31,8 @@ waits. They establish the SDK defects, not real LMI worker recovery. The green harness tests exercise negative evidence controls, process-shared marker initialization, real public-API replay/callback/retry with the local runner, -checkpoint response/error forwarding, deployment readback, and ownership/retirement. +checkpoint response/error forwarding, persistent create/update, ownership checks, +and run-scoped cleanup. They also exercise the real parallel/map/nested fixtures with controlled I/O and reject an early-completion pass when the losing branch failed before the winner. @@ -55,18 +56,17 @@ Set: | `TEST_LAMBDA_EXECUTION_ROLE_ARN` | Lambda role with LMI/durable execution and logging permissions | The deployment identity needs CloudFormation stack operations, S3 bucket/policy/ -object operations, Lambda deployment/configuration/invoke/history/stop/delete, +object operations, Lambda deployment/configuration/invoke/history/list/stop, `iam:PassRole` for the supplied role, CloudWatch Logs read/create/delete, and read access to the provider. The stack's private bucket policy grants that function -role access **only** to this run's event and control objects. The SDK and dependencies +role access **only** to the suite's `runs/` event and control objects. The SDK and dependencies come from a wheel built from the checked-out core package; no published SDK wheel is substituted. The artifact hash, SDK wheel version, and commit are retained. ```sh hatch run lmi:build -hatch run lmi:python -m lmi_tests.deploy reconcile hatch run lmi:python -m lmi_tests.deploy deploy \ - --run-id local-20260923-a --runtime python3.14 + --stack-name python-lmi-e2e --run-id local-20260923-a --runtime python3.14 hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml # Run both commands below even when a scenario fails: hatch run lmi:python -m lmi_tests.deploy collect @@ -76,14 +76,17 @@ python -m lmi_tests.summary For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls `collect` followed by `cleanup`; the workflow already has unconditional steps. -Use a fresh run ID for each deployment. This one deployment supplies both native +Use a fresh run ID for each run and reuse the same stack name. This deployment supplies both native environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud command runs all 14 scenarios for each setting (28 cases), sequentially, without redeploying or changing function configuration between cases. Independent SDK branch-concurrency settings include 1, 2, and 3. -Each run creates one unique stack, one private bucket, and two functions/log -groups (`c1`, `c2`). All scenarios, including deadlines and healthy-peer probes, +The first run creates the persistent `python-lmi-e2e` stack, one private bucket, +and two functions/log groups (`python-lmi-e2e-c1`, `python-lmi-e2e-c2`). Later runs +update these same resources and preserve their names/ARNs. Select another dedicated +stack with `--stack-name`; its bucket name is stable for the account/region/stack. +All scenarios, including deadlines and healthy-peer probes, reuse the function for their concurrency setting and the same handler artifact. Both functions use a 60-second invocation timeout. Deadline tests wait for the real platform timeout, leaving ordinary scenarios their existing invocation @@ -140,7 +143,7 @@ has requested suspension; it does not schedule another durable operation or chan the handler's result/operation sequence. Ordinary orchestration delays still use `context.wait`. Local shared-root tests run under a subprocess watchdog, so an executor-starvation regression cannot hang the test process. Cloud progress tests -have bounded driver budgets and the existing run-owned-function retirement path. +have bounded driver budgets and post-assertion release/settlement of run-owned work. Healthy-peer cases checkpoint an admission decision before running fault work. If an environment rotation places the victim elsewhere, it returns without @@ -216,7 +219,8 @@ transport timeout). Invocation and callback writes are not retried by this layer | Fault-fixture emergency I/O release | 150 s | | Per-case retirement after assertions | 30 s | | Workflow cloud assertions (all 28 cases) | 20 min | -| Single cloud job including provisioning/retirement | 70 min | +| Waiting for prior executions before updating | 270 s | +| Single cloud job including deployment/run cleanup | 80 min | Artifacts contain JUnit, configuration/provider readbacks, qualified targets, commit/wheel/code hash, invocation inputs/ARNs, all lifecycle and side-effect records, @@ -234,7 +238,7 @@ and on every push to `main`, including merged changes. There are no path filters label requirements, or ready-for-review requirements. `workflow_dispatch` remains available for manual reruns; there are no scheduled jobs. -Each workflow run has one harness job and one cloud job with its own resources. +Each workflow run has one harness job and one cloud job using the persistent resources. All cloud jobs share the repository-wide `lmi-e2e-shared-capacity-provider` concurrency group, across PRs, main pushes, and manual runs. Only one cloud job can deploy, test, or clean up at a time; the slot is held until the entire job finishes. @@ -262,16 +266,27 @@ before invoking the shared function. Collection remains scoped to the case, and the final collector gathers the entire run. These post-assertion releases and stops cannot satisfy a deadline, cleanup-grace, or capacity-recovery assertion. -The workflow collects evidence before teardown, even after test failure or ordinary -cancellation. Cleanup sends external releases, stops running durable executions, -deletes the run's functions to retire their environments, empties the bucket, and -waits for stack deletion. Stopping an execution alone is not evidence that Python -code stopped. Only suite/run-tagged resources are eligible for deletion. A failed -create/name collision cannot cause another run's stack to be deleted. - -Forced runner termination may prevent an `always()` step. Each stack has a four-hour -expiry tag; `reconcile` runs before the next cloud deployment and can also be invoked -manually. It retires only expired stacks owned by this suite. S3 objects additionally -expire after two days; object expiry does not retire compute. Inspect cleanup errors -and run reconciliation if `cleanup.json` does not confirm deletion. The shared -capacity provider remains under its owner's explicit lifecycle policy. +Before updating, the driver checks persistent ownership tags and waits for earlier +durable executions on both functions to finish. Incomplete execution listings, +active executions past the wait budget, and failed/busy stacks block updates. +The driver never deletes and recreates a failed stack; CloudFormation's normal +deployment rollback behavior still applies. Every run updates `LMI_RUN_ID` and +`LMI_COMMIT` in the function configuration, refreshing runtime state while keeping +function names and ARNs stable, as in Java PR #728. Readback and trace identity must +match the tested run and commit. No code/configuration changes occur during tests. + +The workflow collects evidence before final run cleanup, even after test failure or +ordinary cancellation. `cleanup` releases `runs/RUN_ID/control/release-all`, discovers +only the current run's invocations from its ledger/artifacts, stops unfinished +executions, and waits for observed wrappers to exit. It never deletes functions, +the stack, bucket, or code artifacts. A terminal service status alone does not prove +that LMI worker code stopped. Other runs' controls and executions are not released +or stopped by this cleanup. + +Events and controls use `runs/RUN_ID/events/` and `runs/RUN_ID/control/`; only `runs/` +objects expire after one day. Content-addressed `code/` artifacts are retained for +updates and CloudFormation rollback. Logs and durable histories retain one day, +and GitHub artifacts retain seven days. There is no automatic infrastructure deletion +or janitor. The test-account owner manages final retirement of the persistent stack, +functions, bucket, and provider. A forced runner termination can interrupt run +cleanup; inspect artifacts and let prior executions settle before the next update. diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index b9f96fc0c..0cbbe7ef9 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -127,7 +127,7 @@ def hold(self, gate): try: self.s3.put_object( Bucket=self.manifest["bucket"], - Key="control/" + gate, + Key=f"runs/{self.manifest['run']}/control/{gate}", Body=b"hold", IfNoneMatch="*", ) @@ -140,7 +140,9 @@ def hold(self, gate): def release(self, gate): self.s3.put_object( - Bucket=self.manifest["bucket"], Key="control/" + gate, Body=b"release" + Bucket=self.manifest["bucket"], + Key=f"runs/{self.manifest['run']}/control/{gate}", + Body=b"release", ) def refresh(self, markers=None, *, validate_controls=True): @@ -148,9 +150,8 @@ def refresh(self, markers=None, *, validate_controls=True): # observe its first event. Explicit [] requests the complete run for collection. if markers is None: markers = [item["marker"] for item in self.invocations] - prefixes = ( - [f"events/{marker}/" for marker in markers] if markers else ["events/"] - ) + prefix = f"runs/{self.manifest['run']}/events/" + prefixes = [f"{prefix}{marker}/" for marker in markers] if markers else [prefix] missing = [] try: for prefix in prefixes: @@ -342,6 +343,32 @@ def release_all(self): self.release(gate) self.gates.clear() + def run_invocations(self): + """Recover only this run's invocations, including failed Invoke responses.""" + events = self.refresh(markers=[], validate_controls=False) + items = { + e["execution"]: {"arn": e["execution"], "marker": e["marker"]} + for e in events + } + for path in (ARTIFACTS / "invocations").glob("*.json"): + record = json.loads(path.read_text()) + if ( + record.get("arn") + and record.get("payload", {}).get("run") == self.manifest["run"] + ): + items[record["arn"]] = { + "arn": record["arn"], + "marker": record["payload"]["marker"], + } + targets = tuple( + arn + "/durable-execution/" for arn in self.manifest["functions"].values() + ) + if any(not item["arn"].startswith(targets) for item in items.values()): + raise CollectionError( + "Refusing to stop an execution outside this deployment" + ) + return list(items.values()) + def settle_case(self): """Release fault work and drain this case before reusing its function.""" self.release_all() diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 97c3946ed..dac1f4860 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Build, provision, verify, collect, and retire run-owned LMI resources.""" +"""Build, update, verify, and exercise persistent LMI test resources.""" import argparse import base64 @@ -28,6 +28,7 @@ SCALING = {"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1} QUALIFIER = "$LATEST.PUBLISHED" CONCURRENCIES = {"c1": 1, "c2": 2} +DEFAULT_STACK = "python-lmi-e2e" def client(service, region=None): @@ -128,9 +129,10 @@ def template(manifest, functions=True): "LifecycleConfiguration": { "Rules": [ { - "Id": "expiry", + "Id": "expire-run-data", "Status": "Enabled", - "ExpirationInDays": 2, + "Prefix": "runs/", + "ExpirationInDays": 1, "AbortIncompleteMultipartUpload": { "DaysAfterInitiation": 1 }, @@ -151,8 +153,7 @@ def template(manifest, functions=True): "Principal": {"AWS": manifest["role"]}, "Action": ["s3:GetObject", "s3:PutObject"], "Resource": [ - f"arn:aws:s3:::{bucket}/events/*", - f"arn:aws:s3:::{bucket}/control/*", + f"arn:aws:s3:::{bucket}/runs/*", ], }, { @@ -257,7 +258,7 @@ def wait_stack(cfn, name, expected, seconds=1500): ) raise ProvisioningError(f"{name}: {status}; no fallback to standard Lambda") time.sleep(3) - raise ProvisioningError(f"{name}: provisioning/retirement deadline exceeded") + raise ProvisioningError(f"{name}: provisioning deadline exceeded; stack retained") def verify(config, scaling, manifest, key): @@ -294,6 +295,10 @@ def deploy(args): raise ProvisioningError( "run-id must be 1-24 lowercase letters, digits or hyphens" ) + if not re.fullmatch(r"[a-z][a-z0-9-]{0,49}", args.stack_name): + raise ProvisioningError( + "stack-name must be 1-50 lowercase letters, digits or hyphens, starting with a letter" + ) provider = os.environ["CAPACITY_PROVIDER_ARN"] account = client("sts").get_caller_identity()["Account"] if account != os.environ["TEST_ACCOUNT_ID"] or provider.split(":")[3:5] != [ @@ -323,12 +328,15 @@ def deploy(args): built = json.loads((ARTIFACTS / "build.json").read_text()) if built["sha256"] != digest.hex(): raise ProvisioningError("Artifact changed since build") - suffix = args.runtime.replace("python", "").replace(".", "") - name = f"py-lmi-{args.run_id}-{suffix}" + name = args.stack_name + bucket_scope = hashlib.sha256( + f"{os.environ['AWS_REGION']}:{name}".encode() + ).hexdigest()[:12] manifest = { "run": args.run_id, "stack": name, - "bucket": name + "-" + account, + "bucket": f"py-lmi-e2e-{account}-{bucket_scope}", + "persistent": True, "account": account, "region": os.environ["AWS_REGION"], "provider": provider, @@ -343,35 +351,70 @@ def deploy(args): "driverTimeout": 120, "cleanupGrace": 5, "created": time.time(), - "expires": int(time.time()) + 4 * 3600, "functions": {}, } - save( - ARTIFACTS / "manifest.json", manifest - ) # enough to clean up partial provisioning + save(ARTIFACTS / "manifest.json", manifest) + deploy_stack(manifest, data) + + +def deploy_stack(manifest, data): + """Bootstrap the bucket once, then update the same two functions in place.""" cfn = client("cloudformation") tags = [ {"Key": "Suite", "Value": OWNER}, - {"Key": "RunId", "Value": args.run_id}, - {"Key": "Expires", "Value": str(manifest["expires"])}, + {"Key": "Stack", "Value": manifest["stack"]}, + {"Key": "Persistent", "Value": "true"}, ] - # Create only. A collision cannot take over another run or mutate shared fixtures. - cfn.create_stack( - StackName=name, TemplateBody=json.dumps(template(manifest, False)), Tags=tags - ) + stack = owned_stack(cfn, manifest) + if stack is None: + cfn.create_stack( + StackName=manifest["stack"], + TemplateBody=json.dumps(template(manifest, False)), + Tags=tags, + ) + manifest["owned"] = True + save(ARTIFACTS / "manifest.json", manifest) + wait_stack(cfn, manifest["stack"], "CREATE_COMPLETE", 180) + else: + if stack["StackStatus"] not in { + "CREATE_COMPLETE", + "UPDATE_COMPLETE", + "UPDATE_ROLLBACK_COMPLETE", + }: + raise ProvisioningError( + f"Persistent stack requires recovery from {stack['StackStatus']}; it was retained" + ) + wait_for_idle_functions(stack) manifest["owned"] = True save(ARTIFACTS / "manifest.json", manifest) - wait_stack(cfn, name, "CREATE_COMPLETE", 180) client("s3").put_object( Bucket=manifest["bucket"], Key=manifest["codeKey"], Body=data ) client("s3").put_object( - Bucket=manifest["bucket"], Key="control/release-all", Body=b"hold" + Bucket=manifest["bucket"], + Key=f"runs/{manifest['run']}/control/release-all", + Body=b"hold", ) - cfn.update_stack( - StackName=name, TemplateBody=json.dumps(template(manifest)), Tags=tags - ) - stack = wait_stack(cfn, name, "UPDATE_COMPLETE") + spec = template(manifest) + save(ARTIFACTS / "template.json", spec) + try: + cfn.update_stack( + StackName=manifest["stack"], TemplateBody=json.dumps(spec), Tags=tags + ) + except ClientError as error: + if "No updates are to be performed" not in error.response["Error"].get( + "Message", "" + ): + raise + stack = owned_stack(cfn, manifest) + else: + stack = wait_stack(cfn, manifest["stack"], "UPDATE_COMPLETE") + if {output["OutputKey"] for output in stack.get("Outputs", [])} != set( + CONCURRENCIES + ): + raise ProvisioningError( + "The persistent stack must expose both shared LMI functions" + ) for output in stack["Outputs"]: key, arn = output["OutputKey"], output["OutputValue"] manifest["functions"][key] = arn @@ -398,134 +441,97 @@ def owned_stack(cfn, manifest): try: stack = cfn.describe_stacks(StackName=manifest["stack"])["Stacks"][0] except ClientError as error: - if "does not exist" in str(error): + if error.response["Error"][ + "Code" + ] == "ValidationError" and "does not exist" in str(error): return None raise tags = {t["Key"]: t["Value"] for t in stack.get("Tags", [])} if ( tags.get("Suite") != OWNER - or tags.get("RunId") != manifest["run"] - or not manifest["stack"].startswith("py-lmi-") + or tags.get("Stack") != manifest["stack"] + or tags.get("Persistent") != "true" ): raise ProvisioningError( - "Refusing to delete resources not owned by this suite/run" + "Refusing to modify a stack not owned by this persistent suite" ) return stack +def wait_for_idle_functions(stack, seconds=270): + """Wait for prior runs without stopping them or updating active functions.""" + lam = client("lambda") + deadline = time.monotonic() + seconds + while True: + running = [] + for output in stack.get("Outputs", []): + if output["OutputKey"] not in CONCURRENCIES: + raise ProvisioningError( + "Unexpected function in persistent stack outputs" + ) + for page in lam.get_paginator( + "list_durable_executions_by_function" + ).paginate( + FunctionName=output["OutputValue"].rsplit(":", 1)[0], + Statuses=["RUNNING"], + ): + running.extend(page.get("DurableExecutions", [])) + if page.get("NextMarker") and time.monotonic() >= deadline: + raise ProvisioningError( + "Could not finish checking previous executions; code was not updated" + ) + save(ARTIFACTS / "pre-deploy-executions.json", running) + if not running: + return + if time.monotonic() >= deadline: + raise ProvisioningError( + "Previous durable executions are still running; persistent code was not updated" + ) + time.sleep(2) + + def cleanup(manifest): + """Release only this run's work; retain functions, stack, bucket, and code.""" if not manifest.get("owned"): - return # create failed (including a name collision): do not delete that stack - cfn, s3, lam = client("cloudformation"), client("s3"), client("lambda") - stack = owned_stack(cfn, manifest) - if stack is None: + return + cfn = client("cloudformation") + if owned_stack(cfn, manifest) is None: return resources = cfn.describe_stack_resources(StackName=manifest["stack"])[ "StackResources" ] - bucket_exists = any( + if any( r["ResourceType"] == "AWS::S3::Bucket" and r.get("PhysicalResourceId") == manifest["bucket"] and r["ResourceStatus"] != "DELETE_COMPLETE" for r in resources - ) - if bucket_exists: - s3.put_object( - Bucket=manifest["bucket"], Key="control/release-all", Body=b"release" + ): + client("s3").put_object( + Bucket=manifest["bucket"], + Key=f"runs/{manifest['run']}/control/release-all", + Body=b"release", ) - errors = [] - stop_errors = [] - for resource in resources: - if ( - resource["ResourceType"] != "AWS::Lambda::Function" - or resource["ResourceStatus"] == "DELETE_COMPLETE" - ): - continue - try: - for page in lam.get_paginator( - "list_durable_executions_by_function" - ).paginate( - FunctionName=resource["PhysicalResourceId"], Statuses=["RUNNING"] - ): - for execution in page.get("DurableExecutions", []): - lam.stop_durable_execution( - DurableExecutionArn=execution["DurableExecutionArn"] - ) - except ClientError as error: - stop_errors.append(str(error)) - try: - # Delete functions BEFORE emptying the bucket. This is the retirement - # path; stopping a logical execution alone cannot stop LMI Python code. - lam.delete_function(FunctionName=resource["PhysicalResourceId"]) - except ClientError as error: - if error.response["Error"]["Code"] != "ResourceNotFoundException": - errors.append(str(error)) - if errors: - raise ProvisioningError("Function retirement failed: " + "; ".join(errors)) - if stop_errors: - save(ARTIFACTS / "cleanup-stop-errors.json", stop_errors) - # Bounded retries account for already in-flight diagnostic PUTs during retirement. - for _attempt in range(6): - if bucket_exists: - for page in s3.get_paginator("list_objects_v2").paginate( - Bucket=manifest["bucket"] - ): - keys = [{"Key": o["Key"]} for o in page.get("Contents", [])] - if keys: - deleted = s3.delete_objects( - Bucket=manifest["bucket"], Delete={"Objects": keys} - ) - if deleted.get("Errors"): - raise ProvisioningError( - "Could not empty run-owned bucket: " - + str(deleted["Errors"]) - ) - cfn.delete_stack(StackName=manifest["stack"]) - try: - wait_stack(cfn, manifest["stack"], "DELETE_COMPLETE", 180) - save( - ARTIFACTS / "cleanup.json", - {"stack": manifest["stack"], "status": "DELETED"}, - ) - return - except ProvisioningError: - # A non-bucket failure stays visible after this finite retry budget. - time.sleep(2) - raise ProvisioningError( - "Run-owned stack retirement failed; use reconcile after investigating stack-events.json" - ) + from lmi_tests.cloud import Cloud - -def reconcile(): - """Only expired, suite-tagged stacks; never mutate/delete the shared provider.""" - cfn = client("cloudformation") - for page in cfn.get_paginator("describe_stacks").paginate(): - for stack in page["Stacks"]: - tags = {t["Key"]: t["Value"] for t in stack.get("Tags", [])} - if ( - tags.get("Suite") != OWNER - or int(tags.get("Expires", "0")) >= time.time() - ): - continue - if not tags.get("Expires") or not stack["StackName"].startswith("py-lmi-"): - continue - account = client("sts").get_caller_identity()["Account"] - cleanup( - { - "owned": True, - "stack": stack["StackName"], - "bucket": stack["StackName"] + "-" + account, - "run": tags["RunId"], - } - ) + driver = Cloud(manifest) + driver.invocations = driver.run_invocations() + driver.settle_case() + save( + ARTIFACTS / "cleanup.json", + { + "stack": manifest["stack"], + "run": manifest["run"], + "status": "RETAINED", + "message": "Run work released; persistent functions, bucket, and code retained", + }, + ) def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument( - "command", choices=["build", "deploy", "collect", "cleanup", "reconcile"] - ) + parser.add_argument("command", choices=["build", "deploy", "collect", "cleanup"]) parser.add_argument("--run-id") + parser.add_argument("--stack-name", default=DEFAULT_STACK) parser.add_argument("--runtime", choices=["python3.14"], default="python3.14") args = parser.parse_args() try: @@ -533,8 +539,6 @@ def main(): build() elif args.command == "deploy": deploy(args) - elif args.command == "reconcile": - reconcile() else: manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) if args.command == "cleanup": diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 3fc56caf5..d7e0ef1e3 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -97,18 +97,24 @@ def emit(self, phase, **fields): print("LMI_TEST " + json.dumps(event), flush=True) self.s3.put_object( Bucket=self.bucket, - Key=f"events/{self.identity['marker']}/{self.identity['request']}/{self.sequence:06d}.json", + Key=f"runs/{self.identity['run']}/events/{self.identity['marker']}/{self.identity['request']}/{self.sequence:06d}.json", Body=json.dumps(event).encode(), ContentType="application/json", ) return event def signal(self, name): - self.s3.put_object(Bucket=self.bucket, Key="control/" + name, Body=b"release") + self.s3.put_object( + Bucket=self.bucket, + Key=f"runs/{self.identity['run']}/control/{name}", + Body=b"release", + ) def released(self, name): try: - response = self.s3.get_object(Bucket=self.bucket, Key="control/" + name) + response = self.s3.get_object( + Bucket=self.bucket, Key=f"runs/{self.identity['run']}/control/{name}" + ) with response["Body"] as body: state = body.read() if state not in {b"hold", b"release"}: diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py index ae5b60a66..fcbcdbbb8 100644 --- a/lmi_tests/summary.py +++ b/lmi_tests/summary.py @@ -47,9 +47,9 @@ def summarize(directory): lines.append(f"- {path.stem}: {data['type']}") cleanup = directory / "cleanup.json" lines.append( - "Resources retired." + "Run work released; persistent stack, functions, bucket, and code retained." if cleanup.exists() - else "Resource retirement not confirmed; inspect cleanup diagnostics and run reconcile if needed." + else "Run cleanup not confirmed; persistent resources are retained. Inspect cleanup diagnostics before updating." ) return "\n\n".join(lines) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index 926b61936..8909008f8 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -220,14 +220,18 @@ def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset( ) item = driver.start(scenario) assert calls == [ - ("hold", "control/" + item["marker"] + "-loser", b"hold"), - ("hold", "control/" + item["marker"] + "-loser-started", b"hold"), + ("hold", "runs/unit/control/" + item["marker"] + "-loser", b"hold"), + ("hold", "runs/unit/control/" + item["marker"] + "-loser-started", b"hold"), ("invoke",), ] calls.clear() driver.start("barrier", gate="shared") driver.start("barrier", gate="shared") - assert calls == [("hold", "control/shared", b"hold"), ("invoke",), ("invoke",)] + assert calls == [ + ("hold", "runs/unit/control/shared", b"hold"), + ("invoke",), + ("invoke",), + ] assert "release-all" not in driver.gates @@ -244,11 +248,51 @@ def test_finally_control_is_initialized_before_invocation(driver): ) item = driver.start("suspend-cleanup") assert calls == [ - ("hold", "control/" + item["marker"] + "-cleanup", b"hold"), + ("hold", "runs/unit/control/" + item["marker"] + "-cleanup", b"hold"), ("invoke",), ] +def test_runs_share_functions_but_never_controls_or_event_prefixes(driver): + other = module.Cloud({**driver.manifest, "run": "other"}, "c1") + other.s3 = driver.s3 + objects = {} + driver.s3.put_object.side_effect = lambda **kw: objects.update( + {kw["Key"]: kw["Body"]} + ) + driver.hold("same-gate") + other.hold("same-gate") + driver.release("same-gate") + assert objects == { + "runs/unit/control/same-gate": b"release", + "runs/other/control/same-gate": b"hold", + } + driver.s3.get_paginator.return_value.paginate.return_value = [] + driver.refresh(markers=[]) + other.refresh(markers=[]) + assert [ + c.kwargs["Prefix"] + for c in driver.s3.get_paginator.return_value.paginate.call_args_list + ] == ["runs/unit/events/", "runs/other/events/"] + + +def test_cleanup_discovery_ignores_other_runs_and_rejects_foreign_functions( + driver, tmp_path +): + arn = driver.manifest["functions"]["c1"] + "/durable-execution/case/id" + driver.refresh = Mock(return_value=[{"execution": arn, "marker": "case"}]) + records = tmp_path / "invocations" + records.mkdir() + (records / "old.json").write_text( + json.dumps({"payload": {"run": "previous", "marker": "old"}, "arn": "foreign"}) + ) + assert driver.run_invocations() == [{"arn": arn, "marker": "case"}] + driver.refresh.assert_called_once_with(markers=[], validate_controls=False) + driver.refresh.return_value = [{"execution": "foreign", "marker": "bad"}] + with pytest.raises(CollectionError, match="outside this deployment"): + driver.run_invocations() + + def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): driver.invocations = [{"marker": "current"}] event = { @@ -262,7 +306,7 @@ def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): "gate": "g", "error": "403 Forbidden", } - key = "events/current/r/000001.json" + key = "runs/unit/events/current/r/000001.json" driver.s3.get_paginator.return_value.paginate.return_value = [ {"Contents": [{"Key": key}]} ] @@ -272,14 +316,14 @@ def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): with pytest.raises(CollectionError, match="403"): driver.refresh() driver.s3.get_paginator.return_value.paginate.assert_called_once_with( - Bucket="unit-bucket", Prefix="events/current/" + Bucket="unit-bucket", Prefix="runs/unit/events/current/" ) driver.s3.get_object.assert_called_once_with(Bucket="unit-bucket", Key=key) # Full diagnostic collection preserves errors without turning itself into a # second failing test or omitting evidence from earlier cases. assert driver.refresh(markers=[]) == [event] driver.s3.get_paginator.return_value.paginate.assert_called_with( - Bucket="unit-bucket", Prefix="events/" + Bucket="unit-bucket", Prefix="runs/unit/events/" ) diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py index 76a12e22f..5b6e61dfc 100644 --- a/lmi_tests/tests/unit/test_control.py +++ b/lmi_tests/tests/unit/test_control.py @@ -29,6 +29,7 @@ def trace(): aws_secret_access_key="test", ) value.bucket = "test-bucket" + value.identity = {"run": "unit"} value.emit = Mock() return value @@ -40,7 +41,7 @@ def test_control_reads_explicit_state_and_closes_body(trace, state, released): stub.add_response( "get_object", {"Body": body}, - {"Bucket": "test-bucket", "Key": "control/gate"}, + {"Bucket": "test-bucket", "Key": "runs/unit/control/gate"}, ) assert trace.released("gate") is released assert body.closed @@ -92,9 +93,9 @@ def test_fault_gate_outlives_invocation_deadline_but_remains_bounded( def test_winner_is_signalled_only_after_loser_actually_enters_held_io(trace): controls = { - "control/loser": b"hold", - "control/ready": b"hold", - "control/release-all": b"hold", + "runs/unit/control/loser": b"hold", + "runs/unit/control/ready": b"hold", + "runs/unit/control/release-all": b"hold", } trace.s3 = Mock() trace.s3.get_object.side_effect = lambda **kw: { @@ -102,10 +103,12 @@ def test_winner_is_signalled_only_after_loser_actually_enters_held_io(trace): } def signal(**kwargs): - assert kwargs["Key"] == "control/ready" and kwargs["Body"] == b"release" + assert ( + kwargs["Key"] == "runs/unit/control/ready" and kwargs["Body"] == b"release" + ) assert [c.args[0] for c in trace.emit.call_args_list] == ["BLOCKED", "EFFECT"] controls[kwargs["Key"]] = kwargs["Body"] - controls["control/loser"] = b"release" + controls["runs/unit/control/loser"] = b"release" trace.s3.put_object.side_effect = signal trace.gate("loser", effects=True, ready="ready") @@ -114,18 +117,19 @@ def signal(**kwargs): "EFFECT", "IO_EXIT", ] - assert controls["control/release-all"] == b"hold" + assert controls["runs/unit/control/release-all"] == b"hold" def test_emitted_records_are_partitioned_by_case_and_request(trace): trace.s3 = Mock() - trace.identity = {"marker": "case", "request": "request"} + trace.identity = {"run": "unit", "marker": "case", "request": "request"} trace.sequence = 0 trace.deadline = 10 trace.lock = threading.Lock() Trace.emit(trace, "BODY") assert ( - trace.s3.put_object.call_args.kwargs["Key"] == "events/case/request/000001.json" + trace.s3.put_object.call_args.kwargs["Key"] + == "runs/unit/events/case/request/000001.json" ) @@ -133,9 +137,9 @@ def test_emitted_records_are_partitioned_by_case_and_request(trace): def test_full_fixture_establishes_a_real_blocked_loser(scenario): """Exercise fixture wiring and SDK APIs; valid evidence survives future SDK fixes.""" objects = { - "control/release-all": b"hold", - "control/local-loser": b"hold", - "control/local-loser-started": b"hold", + "runs/unit/control/release-all": b"hold", + "runs/unit/control/local-loser": b"hold", + "runs/unit/control/local-loser-started": b"hold", } events = [] computed = threading.Event() @@ -144,7 +148,7 @@ def test_full_fixture_establishes_a_real_blocked_loser(scenario): def put(**kwargs): with lock: objects[kwargs["Key"]] = kwargs["Body"] - if kwargs["Key"].startswith("events/"): + if kwargs["Key"].startswith("runs/unit/events/"): event = json.loads(kwargs["Body"]) events.append(event) if event["phase"] == "WINNER_SELECTED": @@ -157,7 +161,7 @@ def get(**kwargs): trace = Trace.__new__(Trace) trace.s3, trace.bucket = Mock(), "test-bucket" trace.s3.put_object.side_effect, trace.s3.get_object.side_effect = put, get - trace.identity = {"marker": "local", "request": "request"} + trace.identity = {"run": "unit", "marker": "local", "request": "request"} trace.lock, trace.sequence, trace.deadline = threading.Lock(), 0, time.time() + 30 handler = durable_execution( lambda _e, c: workflow({"marker": "local", "scenario": scenario}, c, trace) @@ -173,5 +177,5 @@ def get(**kwargs): evidence.held_loser(snapshot) assert any(e["phase"] == "EFFECT" for e in snapshot) finally: - put(Key="control/local-loser", Body=b"release") + put(Key="runs/unit/control/local-loser", Body=b"release") assert future.result(timeout=3)["Status"] == "SUCCEEDED" diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index 950f66d9e..1845a7622 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -1,4 +1,7 @@ import copy +import json + +from botocore.exceptions import ClientError from unittest.mock import Mock import pytest @@ -11,12 +14,13 @@ def manifest(): return { "run": "unit", - "stack": "py-lmi-unit-314", + "stack": "python-lmi-e2e", "bucket": "test-bucket", "role": "role", "provider": "provider", "runtime": "python3.14", "concurrencies": deploy.CONCURRENCIES, + "functions": {}, "commit": "sha", "codeKey": "code/sha.zip", "codeSha256": "hash", @@ -106,75 +110,206 @@ def test_cleanup_refuses_foreign_owner_and_failed_create(manifest, monkeypatch): cfn.delete_stack.assert_not_called() -def test_cleanup_releases_stops_and_retires_before_bucket_deletion( - manifest, monkeypatch +def stack(manifest, status="UPDATE_COMPLETE"): + return { + "StackStatus": status, + "Tags": [ + {"Key": "Suite", "Value": deploy.OWNER}, + {"Key": "Stack", "Value": manifest["stack"]}, + {"Key": "Persistent", "Value": "true"}, + ], + "Outputs": [ + { + "OutputKey": key, + "OutputValue": f"arn:aws:lambda:us-west-2:123456789012:function:{manifest['stack']}-{key}:$LATEST.PUBLISHED", + } + for key in deploy.CONCURRENCIES + ], + } + + +@pytest.fixture +def apis(monkeypatch, tmp_path): + clients = {name: Mock() for name in ("cloudformation", "s3", "lambda")} + monkeypatch.setattr(deploy, "client", clients.__getitem__) + monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) + return clients + + +def test_later_run_updates_fixed_functions_without_recreating_resources( + manifest, apis, monkeypatch +): + cfn, lam = apis["cloudformation"], apis["lambda"] + monkeypatch.setattr( + deploy, "owned_stack", Mock(side_effect=[None, stack(manifest)]) + ) + monkeypatch.setattr(deploy, "wait_stack", Mock(return_value=stack(manifest))) + idle = Mock() + monkeypatch.setattr(deploy, "wait_for_idle_functions", idle) + verify = Mock() + monkeypatch.setattr(deploy, "verify", verify) + deploy.deploy_stack(manifest, b"first-code") + second = { + **manifest, + "run": "second", + "codeKey": "code/new-digest.zip", + "functions": {}, + } + deploy.deploy_stack(second, b"second-code") + cfn.create_stack.assert_called_once() + assert cfn.update_stack.call_count == 2 + templates = [ + json.loads(call.kwargs["TemplateBody"]) + for call in cfn.update_stack.call_args_list + ] + assert templates[0]["Outputs"] == templates[1]["Outputs"] + assert templates[0]["Resources"].keys() == templates[1]["Resources"].keys() + for key in deploy.CONCURRENCIES: + before = templates[0]["Resources"][key + "Function"]["Properties"] + after = templates[1]["Resources"][key + "Function"]["Properties"] + assert before["FunctionName"] == after["FunctionName"] + assert before["Code"]["S3Key"] != after["Code"]["S3Key"] + assert after["Environment"]["Variables"]["LMI_RUN_ID"] == "second" + idle.assert_called_once() + assert verify.call_count == 4 + assert lam.get_function_configuration.call_count == 4 + cfn.delete_stack.assert_not_called() + lam.delete_function.assert_not_called() + + +def test_unchanged_update_still_verifies_functions(manifest, apis, monkeypatch): + cfn = apis["cloudformation"] + cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} + cfn.update_stack.side_effect = ClientError( + { + "Error": { + "Code": "ValidationError", + "Message": "No updates are to be performed.", + } + }, + "UpdateStack", + ) + monkeypatch.setattr(deploy, "wait_for_idle_functions", Mock()) + wait = Mock() + monkeypatch.setattr(deploy, "wait_stack", wait) + verify = Mock() + monkeypatch.setattr(deploy, "verify", verify) + deploy.deploy_stack(manifest, b"code") + wait.assert_not_called() + assert verify.call_count == 2 + cfn.create_stack.assert_not_called() + + +@pytest.mark.parametrize( + "status", ["ROLLBACK_COMPLETE", "UPDATE_IN_PROGRESS", "UPDATE_ROLLBACK_FAILED"] +) +def test_failed_or_busy_persistent_stack_is_retained(manifest, apis, status): + cfn = apis["cloudformation"] + cfn.describe_stacks.return_value = {"Stacks": [stack(manifest, status)]} + with pytest.raises(ProvisioningError, match="retained"): + deploy.deploy_stack(manifest, b"code") + cfn.update_stack.assert_not_called() + cfn.delete_stack.assert_not_called() + apis["s3"].put_object.assert_not_called() + + +def test_foreign_stack_cannot_be_adopted(manifest, apis): + cfn = apis["cloudformation"] + cfn.describe_stacks.return_value = {"Stacks": [{"Tags": []}]} + with pytest.raises(ProvisioningError, match="not owned"): + deploy.deploy_stack(manifest, b"code") + cfn.update_stack.assert_not_called() + cfn.create_stack.assert_not_called() + + +def test_failed_creation_does_not_trigger_deletion(manifest, apis): + cfn = apis["cloudformation"] + cfn.describe_stacks.side_effect = ClientError( + {"Error": {"Code": "ValidationError", "Message": "Stack does not exist"}}, + "DescribeStacks", + ) + cfn.create_stack.side_effect = RuntimeError("create failed") + with pytest.raises(RuntimeError, match="create failed"): + deploy.deploy_stack(manifest, b"code") + cfn.delete_stack.assert_not_called() + + +def test_prior_executions_must_finish_before_code_updates(manifest, apis, monkeypatch): + lam = apis["lambda"] + one_function = {"Outputs": stack(manifest)["Outputs"][:1]} + lam.get_paginator.return_value.paginate.side_effect = [ + [{"DurableExecutions": [{"DurableExecutionArn": "old"}]}], + [{"DurableExecutions": []}], + ] + sleep = Mock() + monkeypatch.setattr(deploy.time, "sleep", sleep) + deploy.wait_for_idle_functions(one_function) + sleep.assert_called_once_with(2) + lam.stop_durable_execution.assert_not_called() + lam.get_paginator.return_value.paginate.assert_called_with( + FunctionName=one_function["Outputs"][0]["OutputValue"].rsplit(":", 1)[0], + Statuses=["RUNNING"], + ) + + +@pytest.mark.parametrize( + "page", + [ + {"DurableExecutions": [{"DurableExecutionArn": "old"}]}, + {"DurableExecutions": [], "NextMarker": "more"}, + ], +) +def test_busy_or_incompletely_listed_functions_block_updates( + manifest, apis, monkeypatch, page ): - calls = [] - cfn, s3, lam = Mock(), Mock(), Mock() + apis["lambda"].get_paginator.return_value.paginate.return_value = [page] + cfn = apis["cloudformation"] + cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} + wait = deploy.wait_for_idle_functions monkeypatch.setattr( - deploy, - "client", - lambda name: {"cloudformation": cfn, "s3": s3, "lambda": lam}[name], + deploy, "wait_for_idle_functions", lambda value: wait(value, seconds=0) ) - monkeypatch.setattr(deploy, "owned_stack", lambda *_: {}) - monkeypatch.setattr(deploy, "wait_stack", lambda *_: calls.append("stack-gone")) - monkeypatch.setattr(deploy, "save", lambda *_: None) + with pytest.raises(ProvisioningError, match="code was not updated"): + deploy.deploy_stack(manifest, b"code") + cfn.update_stack.assert_not_called() + apis["s3"].put_object.assert_not_called() + apis["lambda"].stop_durable_execution.assert_not_called() + + +def test_only_run_data_expires_and_code_is_retained(manifest): + rules = deploy.template(manifest)["Resources"]["Bucket"]["Properties"][ + "LifecycleConfiguration" + ]["Rules"] + assert all(rule.get("Prefix") == "runs/" for rule in rules) + assert all(rule["ExpirationInDays"] == 1 for rule in rules) + + +def test_cleanup_releases_only_current_run_and_keeps_infrastructure( + manifest, apis, monkeypatch, tmp_path +): + from lmi_tests import cloud + + cfn, s3, lam = apis["cloudformation"], apis["s3"], apis["lambda"] + cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} cfn.describe_stack_resources.return_value = { "StackResources": [ { "ResourceType": "AWS::S3::Bucket", "PhysicalResourceId": manifest["bucket"], "ResourceStatus": "CREATE_COMPLETE", - }, - { - "ResourceType": "AWS::Lambda::Function", - "PhysicalResourceId": "function", - "ResourceStatus": "CREATE_COMPLETE", - }, + } ] } - s3.put_object.side_effect = lambda **_: calls.append("release") - lam.get_paginator.return_value.paginate.return_value = [ - {"DurableExecutions": [{"DurableExecutionArn": "execution"}]} - ] - lam.stop_durable_execution.side_effect = lambda **_: calls.append("stop") - lam.delete_function.side_effect = lambda **_: calls.append("retire") - s3.get_paginator.return_value.paginate.return_value = [ - {"Contents": [{"Key": "evidence"}]} - ] - s3.delete_objects.side_effect = lambda **_: calls.append("empty") or {} - cfn.delete_stack.side_effect = lambda **_: calls.append("delete-stack") + driver = Mock() + driver.run_invocations.return_value = [{"arn": "this-run"}] + monkeypatch.setattr(cloud, "Cloud", Mock(return_value=driver)) deploy.cleanup(manifest) - assert calls == ["release", "stop", "retire", "empty", "delete-stack", "stack-gone"] - - -def test_reconcile_only_deletes_expired_suite_owned_runs(monkeypatch): - cfn = Mock() - monkeypatch.setattr(deploy, "client", lambda _: cfn) - cfn.get_caller_identity.return_value = {"Account": "123"} - - def stack(name, owner, expires): - return { - "StackName": name, - "Tags": [ - {"Key": "Suite", "Value": owner}, - {"Key": "RunId", "Value": "run"}, - {"Key": "Expires", "Value": expires}, - ], - } - - cfn.get_paginator.return_value.paginate.return_value = [ - { - "Stacks": [ - stack("py-lmi-old", deploy.OWNER, "1"), - stack("py-lmi-active", deploy.OWNER, "9999999999"), - stack("py-lmi-other", "other", "1"), - stack("unrelated", deploy.OWNER, "1"), - ] - } - ] - cleanup = Mock() - monkeypatch.setattr(deploy, "cleanup", cleanup) - deploy.reconcile() - assert cleanup.call_count == 1 - assert cleanup.call_args.args[0]["stack"] == "py-lmi-old" + s3.put_object.assert_called_once_with( + Bucket=manifest["bucket"], Key="runs/unit/control/release-all", Body=b"release" + ) + assert driver.invocations == [{"arn": "this-run"}] + driver.settle_case.assert_called_once() + cfn.delete_stack.assert_not_called() + lam.delete_function.assert_not_called() + s3.delete_objects.assert_not_called() + assert json.loads((tmp_path / "cleanup.json").read_text())["status"] == "RETAINED" diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py index 80ece91a5..68e6ed300 100644 --- a/lmi_tests/tests/unit/test_java_parity.py +++ b/lmi_tests/tests/unit/test_java_parity.py @@ -19,13 +19,13 @@ class Controls: def __init__(self): self.condition = threading.Condition() - self.objects = {"control/release-all": b"hold"} + self.objects = {"runs/unit/control/release-all": b"hold"} self.events = [] def put_object(self, *, Key, Body, **_kwargs): with self.condition: self.objects[Key] = Body - if Key.startswith("events/"): + if Key.startswith("runs/unit/events/"): self.events.append(json.loads(Body)) self.condition.notify_all() @@ -49,6 +49,7 @@ def trace(self, marker, scenario): trace.s3, trace.bucket = self, "unit" trace.payload = {"marker": marker, "scenario": scenario} trace.identity = { + "run": "unit", "marker": marker, "request": uuid.uuid4().hex, "environment": "local", @@ -65,7 +66,7 @@ def trace(self, marker, scenario): def test_pending_waits_for_finally_then_replays_without_repeating_body(): controls = Controls() - controls.put_object(Key="control/local-cleanup", Body=b"hold") + controls.put_object(Key="runs/unit/control/local-cleanup", Body=b"hold") def entry(event, context): trace = controls.trace("local", "suspend-cleanup") @@ -83,7 +84,7 @@ def entry(event, context): e for e in controls.snapshot() if e["phase"] == "WRAPPER_RETURN" ], "PENDING returned before finally completed" finally: - controls.put_object(Key="control/local-cleanup", Body=b"release") + controls.put_object(Key="runs/unit/control/local-cleanup", Body=b"release") result = runner.wait_for_result(arn, timeout=10) history = runner.get_execution_history(arn).to_dict()["Events"] assert result.status.value == "SUCCEEDED" @@ -143,7 +144,7 @@ def test_only_missed_admission_retries_environment_pair(monkeypatch): def run_inflight_case(scenario, status): controls = Controls() for suffix in ("loser", "loser-started"): - controls.put_object(Key="control/local-" + suffix, Body=b"hold") + controls.put_object(Key="runs/unit/control/local-" + suffix, Body=b"hold") def entry(event, context): trace = controls.trace("local", scenario) @@ -159,7 +160,7 @@ def entry(event, context): evidence.held_loser(controls.snapshot()) evidence.scope_exit(controls.snapshot(), "local-loser") finally: - controls.put_object(Key="control/local-loser", Body=b"release") + controls.put_object(Key="runs/unit/control/local-loser", Body=b"release") result = runner.wait_for_result(arn, timeout=5) history = runner.get_execution_history(arn).to_dict()["Events"] assert result.status.value == status From ab723eff39c0bcaa7dc80c009c5ecedfc63be55d Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 16 Oct 2026 22:16:54 +0000 Subject: [PATCH 11/32] fix: isolate LMI runs and gate cloud regressions Keep strict cloud regressions manual while the harness runs on PRs. Pin artifact uploads to the immutable revision used on current main. Invalidate old run metadata before build and deployment preflight, preserving the build manifest and preventing prior-run cleanup. --- .github/workflows/lmi-e2e-tests.yml | 14 ++- lmi_tests/README.md | 41 +++++---- lmi_tests/deploy.py | 21 +++++ lmi_tests/tests/unit/test_deploy.py | 130 ++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 24 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index a128f11c1..82d26b839 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -36,7 +36,7 @@ jobs: source = Path('packages/aws-durable-execution-sdk-python/src').resolve() assert Path(sdk.__file__).resolve().is_relative_to(source), sdk.__file__ PY - - uses: actions/upload-artifact@v7 + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 if: always() with: name: lmi-harness-tests @@ -45,12 +45,10 @@ jobs: cloud: name: cloud (Python 3.14, concurrency 1 and 2) needs: harness - # Run on every main push and trusted PR update, including drafts. - # Match the repository's cloud credential restrictions for forks/Dependabot. - if: >- - github.event_name != 'pull_request' || - (github.actor != 'dependabot[bot]' && - github.event.pull_request.head.repo.full_name == github.repository) + # The #741 regressions remain strict failures on current main. Keep the + # cloud suite opt-in until its implementation contract is satisfied; the + # non-cloud harness still runs on every PR and main push. + if: github.event_name == 'workflow_dispatch' && github.actor != 'dependabot[bot]' runs-on: ubuntu-latest timeout-minutes: 80 permissions: @@ -115,7 +113,7 @@ jobs: fi - name: Retain diagnostics if: always() - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: lmi-3.14-${{ github.run_id }}-${{ github.run_attempt }} retention-days: 7 diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 465e99671..4fc3d3337 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -8,9 +8,10 @@ It follows the separate cloud-suite approach in It changes no production SDK code. The deadline and early-completion regressions are **expected to fail on the current SDK**. They are ordinary failing assertions, with no `xfail`, swallowed failure, or expected-failure success status. CI runs the -cloud suite automatically on every trusted PR update and main push. Local cloud -runs and local red regressions require explicit commands; the ordinary local test -suite remains usable while the fix is designed. +non-cloud harness automatically on PR updates and main pushes. The cloud suite +requires `workflow_dispatch` until the #741 implementation satisfies its contract. +Local cloud runs and local red regressions also require explicit commands; the +ordinary local test suite remains usable while the fix is designed. ## Run locally @@ -76,7 +77,12 @@ python -m lmi_tests.summary For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls `collect` followed by `cleanup`; the workflow already has unconditional steps. -Use a fresh run ID for each run and reuse the same stack name. This deployment supplies both native +Use a fresh run ID for each run and reuse the same stack name. Before build or +deployment preflight, the driver removes prior run metadata and reports from +`lmi_tests/artifacts`, preserving `build.json` for artifact verification. Archive +any evidence you need before starting another run. The run manifest is invalidated +first, so a failed build, invalid configuration, or account/provider check cannot +make an exit trap collect or clean up the preceding run. This deployment supplies both native environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud command runs all 14 scenarios for each setting (28 cases), sequentially, without redeploying or changing function configuration between cases. Independent SDK @@ -158,7 +164,7 @@ and negative evidence controls without treating the SDK defect as a passing resu The current SDK has no public cooperative cancellation API. The fault fixture therefore deliberately remains blocked until interrupted/retired by the eventual -SDK/runtime policy or released externally. Its 75-second emergency limit always +SDK/runtime policy or released externally. Its 150-second emergency limit always invalidates a regression pass. It does not implement deadline cancellation in test code, use `shutdown(wait=False)` as a fix, kill the process itself, or manufacture successful checkpoints. Selecting a bounded retirement strategy belongs to #741. @@ -232,15 +238,17 @@ An unexecuted scenario is never summarized as passing. ## CI and resource ownership -The dedicated workflow runs the harness and full LMI suite automatically -when a same-repository PR is opened, updated, or reopened (including Draft PRs), -and on every push to `main`, including merged changes. There are no path filters, -label requirements, or ready-for-review requirements. `workflow_dispatch` remains -available for manual reruns; there are no scheduled jobs. - -Each workflow run has one harness job and one cloud job using the persistent resources. -All cloud jobs share the repository-wide `lmi-e2e-shared-capacity-provider` -concurrency group, across PRs, main pushes, and manual runs. Only one cloud job can +The dedicated workflow runs the non-cloud harness when a PR is opened, +updated, or reopened (including Draft PRs), and on every push to `main`. There +are no path filters or ready-for-review requirements for that harness. The cloud +job runs only with explicit `workflow_dispatch`; there are no scheduled jobs. +This keeps the deliberately failing #741 assertions from blocking every unrelated +PR/main build. Enable automatic cloud execution only after the implementation +satisfies those assertions; the manual cloud job still reports real failures. + +Each manual workflow run has one harness job and one cloud job using the persistent +resources. All cloud jobs share the repository-wide +`lmi-e2e-shared-capacity-provider` concurrency group. Only one cloud job can deploy, test, or clean up at a time; the slot is held until the entire job finishes. Harness jobs can run in parallel. The capacity provider's configuration is not changed. @@ -252,8 +260,9 @@ cancels additional arrivals when that queue is full. This repository-wide group does not coordinate manual local deployments or jobs in other repositories using the same provider; those consumers must also leave sufficient capacity available. -Privileged cloud jobs retain the repository's existing restrictions for forked PRs -and Dependabot; the harness still runs for those PRs. Cloud jobs reuse +PR events, including forks and Dependabot, run only the harness. A cloud run must +be dispatched explicitly by a trusted repository actor; Dependabot remains excluded. +Cloud jobs reuse `TEST_ROLE_ARN`, `TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. The #741 regression assertions remain visibly failing until its fix lands; they are not skipped or converted into expected-success results to keep CI green. diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index dac1f4860..f6d5c2f16 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -58,7 +58,27 @@ def save(path, data): path.write_text(json.dumps(scrub(data), indent=2, default=str) + "\n") +def clear_run_artifacts() -> None: + """Invalidate prior run evidence before fallible build/deployment work. + + Keep the build manifest, which deploy needs to verify the packaged wheel. + Remove the run manifest first: even if a later local deletion fails, an exit + trap cannot use that prior run's identity to collect, release, or stop work. + """ + if not ARTIFACTS.exists(): + return + (ARTIFACTS / "manifest.json").unlink(missing_ok=True) + for path in ARTIFACTS.iterdir(): + if path.name == "build.json": + continue + if path.is_dir() and not path.is_symlink(): + shutil.rmtree(path) + else: + path.unlink() + + def build(): + clear_run_artifacts() directory = ROOT / "lmi_tests/build" if directory.exists(): shutil.rmtree(directory) @@ -291,6 +311,7 @@ def verify(config, scaling, manifest, key): def deploy(args): + clear_run_artifacts() if not args.run_id or not re.fullmatch(r"[a-z0-9-]{1,24}", args.run_id): raise ProvisioningError( "run-id must be 1-24 lowercase letters, digits or hyphens" diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index 1845a7622..2ae939f0a 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -1,5 +1,6 @@ import copy import json +from pathlib import Path from botocore.exceptions import ClientError from unittest.mock import Mock @@ -313,3 +314,132 @@ def test_cleanup_releases_only_current_run_and_keeps_infrastructure( lam.delete_function.assert_not_called() s3.delete_objects.assert_not_called() assert json.loads((tmp_path / "cleanup.json").read_text())["status"] == "RETAINED" + + +@pytest.mark.parametrize( + "failure", + ["run-id", "stack-name", "environment", "account", "capacity", "build", "digest"], +) +def test_failed_preflight_cannot_reuse_prior_run_evidence( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: str +) -> None: + """An EXIT trap after any preflight error must not target the previous run.""" + import hashlib + from types import SimpleNamespace + + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + package = tmp_path / "lmi_tests/build" + package.mkdir(parents=True) + data = b"current-build" + (package / "function.zip").write_bytes(data) + build = {"commit": "current", "sha256": hashlib.sha256(data).hexdigest()} + if failure == "digest": + build["sha256"] = "wrong" + deploy.save(artifacts / "build.json", build) + build_bytes = (artifacts / "build.json").read_bytes() + deploy.save(artifacts / "manifest.json", {"run": "previous", "owned": True}) + deploy.save(artifacts / "cleanup.json", {"run": "previous", "status": "RETAINED"}) + deploy.save(artifacts / "configuration/c1.json", {"run": "previous"}) + (artifacts / "cloud.xml").write_text( + '' + ) + clients = {name: Mock() for name in ("sts", "lambda", "s3")} + clients["sts"].get_caller_identity.return_value = {"Account": "123456789012"} + clients["lambda"].get_capacity_provider.return_value = { + "CapacityProvider": {"CapacityProviderScalingConfig": {"MaxVCpuCount": 2}} + } + monkeypatch.setattr(deploy, "ROOT", tmp_path) + monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) + monkeypatch.setattr(deploy, "client", clients.__getitem__) + deploy_stack = Mock() + monkeypatch.setattr(deploy, "deploy_stack", deploy_stack) + monkeypatch.setenv("AWS_REGION", "us-west-2") + monkeypatch.setenv("TEST_ACCOUNT_ID", "123456789012") + monkeypatch.setenv( + "CAPACITY_PROVIDER_ARN", + "arn:aws:lambda:us-west-2:123456789012:capacity-provider/test", + ) + monkeypatch.setenv("TEST_LAMBDA_EXECUTION_ROLE_ARN", "test-role") + args = SimpleNamespace( + run_id="current", stack_name="python-lmi-e2e", runtime="python3.14" + ) + if failure == "run-id": + args.run_id = "INVALID" + elif failure == "stack-name": + args.stack_name = "INVALID" + elif failure == "environment": + monkeypatch.delenv("CAPACITY_PROVIDER_ARN") + elif failure == "account": + clients["sts"].get_caller_identity.return_value = {"Account": "other"} + elif failure == "capacity": + clients["lambda"].get_capacity_provider.side_effect = RuntimeError( + "capacity unavailable" + ) + elif failure == "build": + (package / "function.zip").unlink() + with pytest.raises((ProvisioningError, KeyError, RuntimeError, FileNotFoundError)): + deploy.deploy(args) + assert (artifacts / "build.json").read_bytes() == build_bytes + assert not (artifacts / "manifest.json").exists() + assert not (artifacts / "cleanup.json").exists() + assert not (artifacts / "cloud.xml").exists() + assert not (artifacts / "configuration").exists() + deploy_stack.assert_not_called() + for command in ("collect", "cleanup"): + monkeypatch.setattr(deploy.sys, "argv", ["deploy", command]) + with pytest.raises(FileNotFoundError): + deploy.main() + clients["s3"].put_object.assert_not_called() + from lmi_tests.summary import summarize + + summary = summarize(artifacts) + assert "old-case" not in summary + assert "Cloud scenarios did not run" in summary + assert "Run cleanup not confirmed" in summary + + +def test_failed_build_clears_prior_run_before_subprocess( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + artifacts = tmp_path / "lmi_tests/artifacts" + deploy.save(artifacts / "manifest.json", {"run": "previous", "owned": True}) + deploy.save(artifacts / "cleanup.json", {"run": "previous"}) + monkeypatch.setattr(deploy, "ROOT", tmp_path) + monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) + monkeypatch.setattr( + deploy.subprocess, "run", Mock(side_effect=RuntimeError("build failed")) + ) + with pytest.raises(RuntimeError, match="build failed"): + deploy.build() + assert not (artifacts / "manifest.json").exists() + assert not (artifacts / "cleanup.json").exists() + + +def test_artifact_cleanup_does_not_follow_symlinks( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + outside = tmp_path / "outside" + outside.mkdir() + (outside / "evidence.json").write_text("keep") + (artifacts / "old-output").symlink_to(outside, target_is_directory=True) + monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) + deploy.clear_run_artifacts() + assert (outside / "evidence.json").read_text() == "keep" + assert not (artifacts / "old-output").exists() + + +def test_manifest_is_invalidated_even_if_later_artifact_cleanup_fails( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + deploy.save(tmp_path / "manifest.json", {"run": "previous", "owned": True}) + deploy.save(tmp_path / "configuration/c1.json", {"run": "previous"}) + monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) + monkeypatch.setattr( + deploy.shutil, "rmtree", Mock(side_effect=PermissionError("blocked")) + ) + with pytest.raises(PermissionError, match="blocked"): + deploy.clear_run_artifacts() + assert not (tmp_path / "manifest.json").exists() From d2c52622826b9269750f4615fdac4aa800f9b82f Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 23 Oct 2026 13:37:41 -0700 Subject: [PATCH 12/32] test: add LMI observability coverage --- .github/workflows/lmi-e2e-tests.yml | 4 +- lmi_tests/README.md | 47 +++- lmi_tests/cloud.py | 43 +++- lmi_tests/deploy.py | 59 ++++- lmi_tests/fixture.py | 15 +- lmi_tests/telemetry.py | 174 +++++++++++++++ lmi_tests/telemetry_evidence.py | 134 ++++++++++++ lmi_tests/tests/e2e/test_observability.py | 140 ++++++++++++ lmi_tests/tests/unit/conftest.py | 29 +++ lmi_tests/tests/unit/test_telemetry.py | 255 ++++++++++++++++++++++ pyproject.toml | 6 +- 11 files changed, 885 insertions(+), 21 deletions(-) create mode 100644 lmi_tests/telemetry.py create mode 100644 lmi_tests/telemetry_evidence.py create mode 100644 lmi_tests/tests/e2e/test_observability.py create mode 100644 lmi_tests/tests/unit/conftest.py create mode 100644 lmi_tests/tests/unit/test_telemetry.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 82d26b839..95c76157a 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -50,7 +50,7 @@ jobs: # non-cloud harness still runs on every PR and main push. if: github.event_name == 'workflow_dispatch' && github.actor != 'dependabot[bot]' runs-on: ubuntu-latest - timeout-minutes: 80 + timeout-minutes: 120 permissions: contents: read id-token: write @@ -95,7 +95,7 @@ jobs: --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" - name: Run cloud regressions (red until issue 741 is fixed) - timeout-minutes: 20 + timeout-minutes: 60 run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml - name: Collect service history and runtime evidence if: always() diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 4fc3d3337..f50f1c08e 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -84,7 +84,7 @@ any evidence you need before starting another run. The run manifest is invalidat first, so a failed build, invalid configuration, or account/provider check cannot make an exit trap collect or clean up the preceding run. This deployment supplies both native environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud -command runs all 14 scenarios for each setting (28 cases), sequentially, without +command runs 14 lifecycle and 24 observability cases for each setting (76 cases), sequentially, without redeploying or changing function configuration between cases. Independent SDK branch-concurrency settings include 1, 2, and 3. @@ -224,9 +224,9 @@ transport timeout). Invocation and callback writes are not retried by this layer | Cleanup acceptance grace | 5 s | | Fault-fixture emergency I/O release | 150 s | | Per-case retirement after assertions | 30 s | -| Workflow cloud assertions (all 28 cases) | 20 min | +| Workflow cloud assertions (all 76 cases) | 60 min | | Waiting for prior executions before updating | 270 s | -| Single cloud job including deployment/run cleanup | 80 min | +| Single cloud job including deployment/run cleanup | 120 min | Artifacts contain JUnit, configuration/provider readbacks, qualified targets, commit/wheel/code hash, invocation inputs/ARNs, all lifecycle and side-effect records, @@ -299,3 +299,44 @@ and GitHub artifacts retain seven days. There is no automatic infrastructure del or janitor. The test-account owner manages final retirement of the persistent stack, functions, bucket, and provider. A forced runner termination can interrupt run cleanup; inspect artifacts and let prior executions settle before the next update. + +## OpenTelemetry acceptance + +The same two functions now also run the observability scope in #742. Both OTel +views use the real public factory registration and a process-owned provider. +The exporter writes the unmodified span identity, attributes, timing and status +to the existing private S3 evidence ledger. A separate invocation identity in +each record lets the assertions detect wrong attribution rather than deriving +the expected owner from the span itself. + +The cloud suite adds native X-Ray sampled/unsampled cases, sequential +success/failure/replay with proven process reuse, concurrent-worker attribution, +export failure/recovery, held export, and the original deadline, retry, PENDING, +late-operation and in-flight-return contracts with each OTel view enabled. +It still runs the original plugin-free lifecycle cases. The cloud assertion +budget is 60 minutes within a 120-minute job; provider capacity and the two +persistent functions are unchanged. + +An instrumented test caller supplies `X-Amzn-Trace-Id` on the real Invoke HTTP +request. The function records the actual runtime `_X_AMZN_TRACE_ID`; it never +substitutes a payload header into the runtime. Native acceptance requires the +observed runtime carrier and actual exported spans. Missing/invalid header +fallbacks are separately labeled local public-wrapper fixtures. OTel's explicit +`Sampled=0` decision can remain authoritative even when the trace ID is invalid. + +`hatch run lmi:unit` includes public-wrapper sampling, process-owned provider +reuse, real local suspension/replay, exporter failures, and negative evidence +controls for wrong execution/trace/parent attribution, missing/duplicate spans, +and leaked context. On non-Linux development hosts, unit-only resource snapshots +retain real thread identity while leaving RSS/FD measurement to the deployed +Linux fixture. + +`lmi:build` builds and installs both the checkout's core and OTel wheels with +the existing standalone OTel dependencies. `build.json` and the deployment +manifest retain both wheel hashes; `build/dependencies.json` records dependency +resolution. No published core/plugin substitutes for the checkout. + +The new tests do not close #742 without a fresh deployed acceptance run. +Plugin-enabled lifecycle cases still expose unresolved #741 behavior; local +green telemetry tests do not imply bounded non-cooperative cleanup or guaranteed +export after a runtime timeout. diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 0cbbe7ef9..1c3c505ca 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -22,8 +22,25 @@ class RunnerClient: """Share paced history reads with the public runner; other APIs are unchanged.""" - def __init__(self, cloud): + def __init__(self, cloud, trace_header=None): self.cloud = cloud + self.trace_header = trace_header + + def invoke(self, **kwargs): + if self.trace_header is None: + return self.cloud.lam.invoke(**kwargs) + # An instrumented caller's HTTP carrier goes through the real Lambda + # service. Never inject a synthetic header into the function payload. + isolated = client("lambda") + + def add_carrier(params, **_kwargs): + params["headers"]["X-Amzn-Trace-Id"] = self.trace_header + + isolated.meta.events.register("before-call.lambda.Invoke", add_carrier) + try: + return isolated.invoke(**kwargs) + finally: + isolated.close() def __getattr__(self, name): return getattr(self.cloud.lam, name) @@ -60,13 +77,31 @@ def verify(self): def concurrency(self): return self.manifest["concurrencies"][self.configuration] - def start(self, scenario, gate=None, target_environment=None): + def start( + self, + scenario, + gate=None, + target_environment=None, + *, + otel_view=None, + trace_header=None, + otel_export="success", + ): if self.configuration not in self.manifest["functions"]: raise ProvisioningError( "Select a deployed concurrency configuration before Invoke" ) marker = scenario + "-" + uuid.uuid4().hex[:12] payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} + if otel_view is not None: + if otel_view not in {"invocation", "execution"}: + raise ValueError("Unknown OTel view") + if otel_export not in {"success", "failure", "held"}: + raise ValueError("Unknown exporter fault") + payload.update(otel_view=otel_view, otel_export=otel_export) + if otel_export == "held": + payload["otel_export_gate"] = marker + "-export" + self.hold(payload["otel_export_gate"]) if target_environment is not None: payload["target_environment"] = target_environment if gate: @@ -90,7 +125,7 @@ def start(self, scenario, gate=None, target_environment=None): self.manifest["functions"][self.configuration], region=self.manifest["region"], ) - runner.lambda_client = RunnerClient(self) + runner.lambda_client = RunnerClient(self, trace_header) try: arn = runner.run_async(input=payload) if not arn: @@ -107,6 +142,7 @@ def start(self, scenario, gate=None, target_environment=None): "configuration": self.configuration, "runner": runner, "started": time.time(), + "trace_header": trace_header, } self.invocations.append(item) print(f"LMI {scenario}: marker={marker}, execution={arn}", flush=True) @@ -117,6 +153,7 @@ def start(self, scenario, gate=None, target_environment=None): "arn": arn, "function": runner.function_name, "started": item["started"], + "trace_header": trace_header, }, ) return item diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index f6d5c2f16..28038cc9e 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -83,14 +83,22 @@ def build(): if directory.exists(): shutil.rmtree(directory) directory.mkdir() - # Hatch builds this checkout's wheel; pip resolves only its dependencies. - subprocess.run( - ["hatch", "build", "-t", "wheel", str(directory)], - cwd=ROOT / "packages/aws-durable-execution-sdk-python", - env={k: v for k, v in os.environ.items() if k != "HATCH_ENV_ACTIVE"}, - check=True, - ) - wheel = next(directory.glob("*.whl")) + # Both SDK artifacts come from this checkout; pip resolves only their + # dependencies. A published core/plugin must not replace either artifact. + for package in ( + "aws-durable-execution-sdk-python", + "aws-durable-execution-sdk-python-otel", + ): + subprocess.run( + ["hatch", "build", "-t", "wheel", str(directory)], + cwd=ROOT / "packages" / package, + env={k: v for k, v in os.environ.items() if k != "HATCH_ENV_ACTIVE"}, + check=True, + ) + wheels = sorted(directory.glob("*.whl")) + if len(wheels) != 2: + raise ProvisioningError("Expected exactly the checked-out core and OTel wheels") + wheel = next(directory.glob("aws_durable_execution_sdk_python-*.whl")) target = directory / "package" subprocess.run( [ @@ -99,17 +107,32 @@ def build(): "pip", "install", "--only-binary=:all:", + "--platform", + "manylinux2014_aarch64", + "--platform", + "manylinux_2_28_aarch64", + "--python-version", + "3.14", + "--implementation", + "cp", + "--abi", + "cp314", "--no-compile", "--target", str(target), - str(wheel), + "--report", + str(directory / "dependencies.json"), + *map(str, wheels), + "opentelemetry-sdk>=1.20.0", + "opentelemetry-propagator-aws-xray", ], check=True, ) (target / "lmi_tests").mkdir() - for name in ("__init__.py", "fixture.py"): + for name in ("__init__.py", "fixture.py", "telemetry.py"): shutil.copy(ROOT / "lmi_tests" / name, target / "lmi_tests" / name) artifact = directory / "function.zip" + resolved = json.loads((directory / "dependencies.json").read_text()) with zipfile.ZipFile(artifact, "w", zipfile.ZIP_DEFLATED) as archive: for path in sorted(target.rglob("*")): if path.is_file(): @@ -123,6 +146,20 @@ def build(): ["git", "rev-parse", "HEAD"], cwd=ROOT, text=True ).strip(), "wheel": wheel.name, + "wheels": { + item.name: hashlib.sha256(item.read_bytes()).hexdigest() + for item in wheels + }, + "dependencies": [ + { + "name": item["metadata"]["name"], + "version": item["metadata"]["version"], + "hashes": item.get("download_info", {}) + .get("archive_info", {}) + .get("hashes", {}), + } + for item in resolved["install"] + ], "sha256": hashlib.sha256(artifact.read_bytes()).hexdigest(), }, ) @@ -365,6 +402,8 @@ def deploy(args): "runtime": args.runtime, "concurrencies": CONCURRENCIES, "commit": built["commit"], + "wheels": built.get("wheels", {}), + "dependencies": built.get("dependencies", []), "codeKey": "code/" + digest.hex() + ".zip", "codeSha256": base64.b64encode(digest).decode(), "invocationTimeout": 60, diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index d7e0ef1e3..ecb88c2b4 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -452,11 +452,22 @@ def handler(event, context): injected = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( invocation, client ) - decorated = durable_execution(lambda e, c: workflow(e, c, trace)) + + def invoke(plugins=None): + decorated = durable_execution( + lambda e, c: workflow(e, c, trace), plugins=plugins + ) + return decorated(injected, context) + before = resources() trace.emit("WRAPPER_ENTER", resources=before) try: - result = decorated(injected, context) + if payload.get("otel_view"): + from lmi_tests.telemetry import invoke as invoke_with_telemetry + + result = invoke_with_telemetry(trace, invoke) + else: + result = invoke() trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) return result except BaseException as error: diff --git a/lmi_tests/telemetry.py b/lmi_tests/telemetry.py new file mode 100644 index 000000000..3d4b7048a --- /dev/null +++ b/lmi_tests/telemetry.py @@ -0,0 +1,174 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real OTel export into the LMI evidence ledger; no runtime carrier injection.""" + +from __future__ import annotations + +from collections.abc import Callable, Sequence +from contextvars import ContextVar +from importlib.metadata import version +import os +import threading +import uuid +from weakref import WeakKeyDictionary +from typing import Any, TYPE_CHECKING + +from opentelemetry import context as otel_context +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider +from opentelemetry.sdk.trace.export import ( + SimpleSpanProcessor, + SpanExporter, + SpanExportResult, +) +from opentelemetry.sdk.trace.sampling import ALWAYS_ON + +from aws_durable_execution_sdk_python_otel import ( + ExecutionOtelPluginFactory, + InvocationOtelPluginFactory, + OtelPluginConfig, +) +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + DurableInstrumentationPluginFactory, + InvocationStartInfo, +) + +if TYPE_CHECKING: + from lmi_tests.fixture import Trace + +_CURRENT_TRACE: ContextVar[Trace | None] = ContextVar("lmi_otel_evidence", default=None) +_RUNTIMES: dict[str, Runtime] = {} +_RUNTIME_LOCK = threading.Lock() + + +class LedgerExporter(SpanExporter): + """Capture the actual exported data, independently of expected attribution.""" + + def __init__(self, view: str) -> None: + self.view = view + self.shutdown_calls = 0 + self.active_trace: Trace | None = None + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + # The checkpoint batcher may export outside the caller's ContextVar. + # Native Python LMI has one request per process. Keep its independently + # observed current invocation as a fallback; never infer ownership from + # the span's potentially corrupted execution attributes. + diagnostic = _CURRENT_TRACE.get() or self.active_trace + if diagnostic is None: + raise RuntimeError("OTel export lost the invocation evidence context") + mode = diagnostic.payload.get("otel_export", "success") + if mode == "held": + diagnostic.emit("OTEL_EXPORT_HELD", view=self.view) + diagnostic.gate(diagnostic.payload["otel_export_gate"]) + for span in spans: + context = span.context + if context is None: + raise ValueError("Exported span has no span context") + data = { + "name": span.name, + "trace_id": format(context.trace_id, "032x"), + "span_id": format(context.span_id, "016x"), + "parent_id": format(span.parent.span_id, "016x") + if span.parent + else None, + "sampled": context.trace_flags.sampled, + "start_ns": span.start_time, + "end_ns": span.end_time, + "status": span.status.status_code.name, + "attributes": dict(span.attributes or {}), + } + diagnostic.emit("OTEL_EXPORT", view=self.view, outcome=mode, span=data) + return ( + SpanExportResult.FAILURE if mode == "failure" else SpanExportResult.SUCCESS + ) + + def shutdown(self) -> None: + self.shutdown_calls += 1 + + +class EvidenceFactory: + """Observe factory ownership while returning the actual bundled OTel plugin.""" + + __durable_registration_api__ = 1 + exclusive_group = "aws-durable-execution-otel-view" + + def __init__(self, view: str, provider: TracerProvider) -> None: + kind = ( + InvocationOtelPluginFactory + if view == "invocation" + else ExecutionOtelPluginFactory + ) + self.delegate = kind( + OtelPluginConfig(tracer_provider=provider, enrich_logger=True) + ) + self.instances: WeakKeyDictionary[DurableInstrumentationPlugin, str] = ( + WeakKeyDictionary() + ) + self.lock = threading.Lock() + + def create_plugin(self, info: InvocationStartInfo) -> DurableInstrumentationPlugin: + trace = _CURRENT_TRACE.get() + if trace is None: + raise RuntimeError("Factory lost the invocation evidence context") + plugin = self.delegate.create_plugin(info) + with self.lock: + identity = self.instances.setdefault(plugin, uuid.uuid4().hex) + trace.emit( + "OTEL_FACTORY", + plugin_instance=identity, + plugin_execution=info.execution_arn, + ) + return plugin + + +class Runtime: + """A provider/exporter shared within one Python worker, across invocations.""" + + def __init__(self, view: str) -> None: + self.identity = uuid.uuid4().hex + self.exporter = LedgerExporter(view) + self.provider = TracerProvider(sampler=ALWAYS_ON, shutdown_on_exit=False) + self.provider.add_span_processor(SimpleSpanProcessor(self.exporter)) + self.factory = EvidenceFactory(view, self.provider) + + +def invoke( + trace: Trace, + invoke_handler: Callable[ + [list[DurableInstrumentationPluginFactory]], dict[str, Any] + ], +) -> dict[str, Any]: + """Enter before SDK dispatch so worker-context propagation is also exercised.""" + view = trace.payload["otel_view"] + if view not in {"invocation", "execution"}: + raise ValueError("Unknown OTel view") + with _RUNTIME_LOCK: + if view not in _RUNTIMES: + _RUNTIMES[view] = Runtime(view) + runtime = _RUNTIMES[view] + token = _CURRENT_TRACE.set(trace) + runtime.exporter.active_trace = trace + before = otel_context.get_current() + trace.emit( + "OTEL_ENTER", + view=view, + provider=runtime.identity, + carrier=os.environ.get("_X_AMZN_TRACE_ID"), + carrier_source="runtime_environment", + core_version=version("aws-durable-execution-sdk-python"), + otel_version=version("aws-durable-execution-sdk-python-otel"), + ) + try: + return invoke_handler([runtime.factory]) + finally: + try: + trace.emit( + "OTEL_EXIT", + view=view, + provider=runtime.identity, + context_restored=otel_context.get_current() == before, + exporter_shutdowns=runtime.exporter.shutdown_calls, + ) + finally: + _CURRENT_TRACE.reset(token) diff --git a/lmi_tests/telemetry_evidence.py b/lmi_tests/telemetry_evidence.py new file mode 100644 index 000000000..2d298cba5 --- /dev/null +++ b/lmi_tests/telemetry_evidence.py @@ -0,0 +1,134 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Independent assertions over recorded native headers and exported OTel spans.""" + +import re +from typing import Any + +from lmi_tests.evidence import CollectionError, PlacementError, select + + +def parse_header(header: object) -> dict[str, str | bool] | None: + if not isinstance(header, str): + return None + fields = dict( + piece.strip().split("=", 1) for piece in header.split(";") if "=" in piece + ) + root, parent, sampled = (fields.get(key) for key in ("Root", "Parent", "Sampled")) + if ( + not root + or not re.fullmatch(r"1-[0-9a-f]{8}-[0-9a-f]{24}", root) + or not parent + or not re.fullmatch(r"[0-9a-f]{16}", parent) + or sampled not in {"0", "1"} + ): + return None + return { + "trace_id": root[2:].replace("-", ""), + "parent_id": parent, + "sampled": sampled == "1", + } + + +def explicit_sampling(header: object) -> bool | None: + if not isinstance(header, str): + return None + values = [ + part.strip()[8:] + for part in header.split(";") + if part.strip().startswith("Sampled=") + ] + return None if not values or values[-1] not in {"0", "1"} else values[-1] == "1" + + +def invocation( + events: list[dict[str, Any]], + *, + native: bool = True, + expected_outcome: str = "success", +) -> None: + starts = select(events, "OTEL_ENTER") + if not starts: + raise CollectionError("Missing OTel invocation entry and carrier evidence") + for start in starts: + local = [event for event in events if event["request"] == start["request"]] + ends = select(local, "OTEL_EXIT") + if len(ends) != 1: + raise CollectionError("Missing or duplicate OTel invocation exit evidence") + assert ends[0]["context_restored"], "OTel context leaked after invocation End" + assert ends[0]["exporter_shutdowns"] == 0, ( + "Invocation shut down a shared exporter" + ) + factories = select(local, "OTEL_FACTORY") + assert len(factories) == 1, "Expected one plugin instance per invocation" + exports = select(local, "OTEL_EXPORT") + header = parse_header(start.get("carrier")) + if native and header is None: + raise CollectionError( + "Real LMI did not provide the required valid X-Ray carrier" + ) + if explicit_sampling(start.get("carrier")) is False: + assert not exports, "An explicit Sampled=0 invocation exported spans" + continue + roots = [ + event["span"] for event in exports if event["span"]["name"] == "Invocation" + ] + if len(roots) != 1: + raise CollectionError( + "Expected one exported Invocation span for a sampled request" + ) + for event in exports: + span = event["span"] + assert event["outcome"] == expected_outcome, "Unexpected export result" + assert ( + span["attributes"].get("durable.execution.arn") == start["execution"] + ), "Exported span belongs to another execution" + assert span["start_ns"] <= span["end_ns"], "Span duration is negative" + assert span["sampled"], "Recording export has an unsampled span context" + if header is not None: + assert span["trace_id"] == header["trace_id"], ( + "Runtime X-Ray Root was lost" + ) + if header is not None: + assert roots[0]["parent_id"] == header["parent_id"], ( + "Runtime X-Ray Parent was lost" + ) + instances = [event["plugin_instance"] for event in select(events, "OTEL_FACTORY")] + assert len(instances) == len(set(instances)), ( + "Plugin instance reused across invocations" + ) + + +def terminal(events: list[dict[str, Any]], *, failed: bool = False) -> None: + """Workflow is emitted once at the terminal invocation, not once per replay.""" + sampled = [ + entry + for entry in select(events, "OTEL_ENTER") + if explicit_sampling(entry.get("carrier")) is not False + ] + if not sampled: + return + spans = [event["span"] for event in select(events, "OTEL_EXPORT")] + workflows = [span for span in spans if span["name"] == "Workflow"] + assert len(workflows) == 1, "Workflow was missing or duplicated across replay" + assert workflows[0]["status"] == ("ERROR" if failed else "OK") + + +def warm_reuse(events: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + starts = select(events, "OTEL_ENTER") + cohorts = {} + for start in starts: + key = (start["environment"], start["process"], start["pid"], start["view"]) + cohorts.setdefault(key, []).append(start) + reused = [ + values + for values in cohorts.values() + if len({e["execution"] for e in values}) > 1 + ] + if not reused: + raise PlacementError("No observed sequential reuse of a Python worker process") + for values in reused: + assert len({entry["provider"] for entry in values}) == 1, ( + "Provider was rebuilt per invocation" + ) + return reused diff --git a/lmi_tests/tests/e2e/test_observability.py b/lmi_tests/tests/e2e/test_observability.py new file mode 100644 index 000000000..cdd8b57bf --- /dev/null +++ b/lmi_tests/tests/e2e/test_observability.py @@ -0,0 +1,140 @@ +"""Native LMI observability acceptance; controlled carriers stay in unit tests.""" + +import time +import uuid + +import pytest + +from lmi_tests import evidence, telemetry_evidence +from lmi_tests.tests.e2e import test_lifecycle as lifecycle + + +@pytest.fixture(params=["invocation", "execution"]) +def view(request): + return request.param + + +def header(sampled="1"): + return ( + f"Root=1-{int(time.time()):08x}-{uuid.uuid4().hex[:24]};" + f"Parent={uuid.uuid4().hex[:16]};Sampled={sampled}" + ) + + +def finish(cloud, item, *, scenario="success", export_outcome="success"): + if scenario == "replay": + callback = item["runner"].wait_for_callback( + item["arn"], name="completion", timeout=90 + ) + item["runner"].send_callback_success(callback, result=item["marker"].encode()) + history = cloud.finish(item, "FAILED" if scenario == "failure" else "SUCCEEDED") + events = cloud.for_item(item) + telemetry_evidence.invocation(events, expected_outcome=export_outcome) + telemetry_evidence.terminal(events, failed=scenario == "failure") + first = evidence.select(events, "OTEL_ENTER")[0] + actual = telemetry_evidence.parse_header(first["carrier"]) + sent = telemetry_evidence.parse_header(item["trace_header"]) + assert actual["trace_id"] == sent["trace_id"], ( + "Lambda did not preserve the caller's X-Ray Root" + ) + assert actual["sampled"] == sent["sampled"], ( + "Lambda did not preserve explicit sampling" + ) + if scenario == "replay": + evidence.replay(events, history, item["marker"]) + return events + + +@pytest.mark.parametrize("sampled", ["0", "1"]) +def test_native_runtime_carrier_and_sampling(cloud, view, sampled): + item = cloud.start("success", otel_view=view, trace_header=header(sampled)) + finish(cloud, item) + + +def test_warm_process_cleanup_and_replay(cloud, view): + events = [] + for scenario in ["success", "failure", "replay"] * 2: + item = cloud.start(scenario, otel_view=view, trace_header=header()) + events.extend(finish(cloud, item, scenario=scenario)) + telemetry_evidence.warm_reuse(events) + telemetry_evidence.invocation(events) + + +def test_concurrent_worker_attribution_and_peer_end(cloud, view): + items = [ + cloud.start( + "barrier", + gate="otel-" + uuid.uuid4().hex, + otel_view=view, + trace_header=header(), + ) + for _ in range(cloud.concurrency) + ] + lifecycle.wait_overlap(cloud, items, cloud.concurrency) + for index, item in enumerate(items): + blocked = cloud.phase(item, "BLOCKED")[0] + cloud.release(blocked["gate"]) + finish(cloud, item) + for peer in items[index + 1 :]: + peer_events = cloud.for_item(peer) + assert not evidence.select(peer_events, "OTEL_EXIT"), ( + "Ending A ended a held peer" + ) + assert evidence.select(peer_events, "ALIVE"), ( + "No healthy-peer progress evidence" + ) + + +def test_export_failure_then_healthy_invocation(cloud, view): + failed = cloud.start( + "success", otel_view=view, trace_header=header(), otel_export="failure" + ) + finish(cloud, failed, export_outcome="failure") + healthy = cloud.start("success", otel_view=view, trace_header=header()) + finish(cloud, healthy) + + +def test_held_export_must_settle_before_successful_cleanup(cloud, view): + item = cloud.start( + "success", otel_view=view, trace_header=header(), otel_export="held" + ) + cloud.phase(item, "OTEL_EXPORT_HELD") + assert not evidence.select(cloud.for_item(item), "OTEL_EXIT") + cloud.release(item["marker"] + "-export") + finish(cloud, item, export_outcome="held") + + +@pytest.mark.parametrize( + "case", + [ + "deadline", + "timeout-retry", + "pending-cleanup", + "late-operation", + "return-inflight", + "failure-inflight", + ], +) +def test_plugin_enabled_lifecycle_boundaries(cloud, view, case, monkeypatch): + # Run the original assertions with real OTel registration. Runtime defects + # remain failures; neither cleanup nor instrumentation turns them into passes. + original = cloud.start + + def start(*args, **kwargs): + return original(*args, **kwargs, otel_view=view, trace_header=header()) + + monkeypatch.setattr(cloud, "start", start) + if case == "deadline": + lifecycle.test_invocation_deadline_isolation_and_capacity_recovery(cloud) + elif case == "timeout-retry": + lifecycle.test_service_timeout_retry_does_not_repeat_completed_step(cloud) + elif case == "pending-cleanup": + lifecycle.test_pending_waits_for_root_finally_and_then_replays(cloud) + elif case == "late-operation": + lifecycle.test_abandoned_child_rejects_late_durable_operation(cloud) + else: + lifecycle.test_root_return_or_failure_settles_inflight_work( + cloud, case, "FAILED" if case == "failure-inflight" else "SUCCEEDED" + ) + for item in cloud.invocations: + telemetry_evidence.invocation(cloud.for_item(item)) diff --git a/lmi_tests/tests/unit/conftest.py b/lmi_tests/tests/unit/conftest.py new file mode 100644 index 000000000..cf4c61583 --- /dev/null +++ b/lmi_tests/tests/unit/conftest.py @@ -0,0 +1,29 @@ +"""Keep Linux-only resource sampling outside portable harness unit tests.""" + +import sys +import threading + +import pytest + +from lmi_tests import fixture + + +@pytest.fixture(autouse=True) +def portable_unit_resource_snapshot(monkeypatch): + if sys.platform == "linux": + return + + def snapshot(): + # Real thread lifetime remains observable. RSS/FD cloud assertions still + # use /proc in the deployed Linux fixture; these tests do not measure it. + return { + "threads": [ + {"id": thread.ident, "name": thread.name} + for thread in threading.enumerate() + ], + "fds": 0, + "rss": 0, + } + + monkeypatch.setattr(fixture, "resources", snapshot) + monkeypatch.setattr("lmi_tests.tests.unit.test_java_parity.resources", snapshot) diff --git a/lmi_tests/tests/unit/test_telemetry.py b/lmi_tests/tests/unit/test_telemetry.py new file mode 100644 index 000000000..469c8ea3b --- /dev/null +++ b/lmi_tests/tests/unit/test_telemetry.py @@ -0,0 +1,255 @@ +"""Public-wrapper OTel controls and negative evidence fixtures; no AWS calls.""" + +import copy +import json +import os +import time + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.config import Duration +from aws_durable_execution_sdk_python.execution import DurableExecutionInvocationInput +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture, telemetry, telemetry_evidence +from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context + + +HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1" + + +class MemoryTrace: + def __init__(self, payload, context, execution): + self.payload = payload + self.sequence = 0 + self.events = [] + self.identity = { + "request": context.aws_request_id, + "execution": execution, + "environment": "local", + "process": "local-worker", + "pid": os.getpid(), + "marker": payload["marker"], + } + + def emit(self, phase, **fields): + self.sequence += 1 + event = { + **self.identity, + "phase": phase, + "sequence": self.sequence, + "time": time.time(), + **fields, + } + self.events.append(event) + return event + + def body(self, name, value, step): + self.emit("BODY", operation=name, value=value, attempt=step.attempt) + return value + + +@pytest.fixture(autouse=True) +def isolated_runtimes(monkeypatch): + monkeypatch.setenv("LMI_RUN_ID", "unit") + monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) + monkeypatch.setattr( + fixture, "resources", lambda: {"threads": [], "fds": 0, "rss": 0} + ) + telemetry._RUNTIMES.clear() + yield + for runtime in telemetry._RUNTIMES.values(): + runtime.provider.shutdown() + telemetry._RUNTIMES.clear() + + +def run( + monkeypatch, view, marker, *, header=HEADER, scenario="success", mode="success" +): + if header is None: + monkeypatch.delenv("_X_AMZN_TRACE_ID", raising=False) + else: + monkeypatch.setenv("_X_AMZN_TRACE_ID", header) + original = os.environ.get("_X_AMZN_TRACE_ID") + source = invocation() + event = source.to_json_dict() + event["DurableExecutionArn"] = ( + "arn:aws:lambda:us-west-2:123456789012:function:lmi:1/durable-execution/" + + marker + + "/local" + ) + event["InitialExecutionState"]["Operations"][0]["StartTimestamp"] = int( + time.time() * 1000 + ) + payload = { + "run": "unit", + "marker": marker, + "scenario": scenario, + "otel_view": view, + "otel_export": mode, + } + event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps(payload) + context = lambda_context(30000) + context.aws_request_id = "request-" + marker + trace = MemoryTrace(payload, context, event["DurableExecutionArn"]) + monkeypatch.setattr(fixture, "Trace", lambda *_: trace) + monkeypatch.setattr( + fixture.LambdaClient, "initialize_client", lambda: source.service_client + ) + result = fixture.handler(event, context) + assert os.environ.get("_X_AMZN_TRACE_ID") == original + assert result["Status"] == ("FAILED" if scenario == "failure" else "SUCCEEDED") + telemetry_evidence.invocation( + trace.events, native=header == HEADER, expected_outcome=mode + ) + return trace.events + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("sampled", ["0", "1"]) +@pytest.mark.parametrize("scenario", ["success", "failure"]) +def test_real_public_wrapper_exports_and_respects_sampling( + monkeypatch, view, sampled, scenario +): + events = run( + monkeypatch, view, "first", header=HEADER[:-1] + sampled, scenario=scenario + ) + telemetry_evidence.terminal(events, failed=scenario == "failure") + exports = [e for e in events if e["phase"] == "OTEL_EXPORT"] + assert bool(exports) == (sampled == "1") + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("header", [None, "", "malformed", "Root=invalid;Sampled=0"]) +def test_warm_reuse_does_not_borrow_previous_header(monkeypatch, view, header): + first = run(monkeypatch, view, "first") + second = run(monkeypatch, view, "second", header=header) + telemetry_evidence.warm_reuse(first + second) + first_root = next( + e["span"]["trace_id"] for e in first if e["phase"] == "OTEL_EXPORT" + ) + second_roots = { + e["span"]["trace_id"] for e in second if e["phase"] == "OTEL_EXPORT" + } + assert first_root not in second_roots + assert bool(second_roots) == ( + telemetry_evidence.explicit_sampling(header) is not False + ) + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +def test_failed_export_does_not_close_provider_or_corrupt_next_invocation( + monkeypatch, view +): + first = run(monkeypatch, view, "failed-export", mode="failure") + second = run(monkeypatch, view, "healthy") + telemetry_evidence.warm_reuse(first + second) + assert all(e["outcome"] == "success" for e in second if e["phase"] == "OTEL_EXPORT") + + +@pytest.mark.parametrize( + "mutation", ["execution", "trace", "parent", "cleanup", "missing", "duplicate"] +) +def test_evidence_rejects_corrupt_or_incomplete_telemetry(monkeypatch, mutation): + events = copy.deepcopy(run(monkeypatch, "invocation", "control")) + spans = [e for e in events if e["phase"] == "OTEL_EXPORT"] + root = next(e for e in spans if e["span"]["name"] == "Invocation") + if mutation == "execution": + root["span"]["attributes"]["durable.execution.arn"] = "other" + elif mutation == "trace": + root["span"]["trace_id"] = "0" * 32 + elif mutation == "parent": + root["span"]["parent_id"] = "0" * 16 + elif mutation == "cleanup": + next(e for e in events if e["phase"] == "OTEL_EXIT")["context_restored"] = False + elif mutation == "missing": + events.remove(root) + else: + events.append(copy.deepcopy(root)) + with pytest.raises((AssertionError, telemetry_evidence.CollectionError)): + telemetry_evidence.invocation(events) + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("failed", [False, True]) +def test_public_runner_suspends_replays_and_finalizes_once(monkeypatch, view, failed): + monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) + traces = [] + bodies = [] + + def business(event, context): + value = context.step( + lambda _: bodies.append("saved") or event["marker"], name="saved" + ) + context.wait(Duration.from_seconds(1), name="pause") + if failed: + raise ValueError("expected failure") + return value + + def entry(event, context): + invocation_input = ( + event + if hasattr(event, "durable_execution_arn") + else DurableExecutionInvocationInput.from_json_dict(event) + ) + diagnostic = MemoryTrace( + {"marker": "replay", "otel_view": view}, + context, + invocation_input.durable_execution_arn, + ) + # The local runner uses a fixed Lambda-context test double. Give each + # controlled attempt a diagnostic ID; native tests use real request IDs. + diagnostic.identity["request"] = "controlled-" + str(len(traces)) + traces.append(diagnostic) + return telemetry.invoke( + diagnostic, + lambda plugins: durable_execution(business, plugins=plugins)( + event, context + ), + ) + + with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: + result = runner.run(input={"marker": "replay"}, execution_timeout=15) + assert result.status.value == ("FAILED" if failed else "SUCCEEDED") + assert bodies == ["saved"] + assert len(traces) >= 2 + events = [event for diagnostic in traces for event in diagnostic.events] + telemetry_evidence.invocation(events) + telemetry_evidence.terminal(events, failed=failed) + + +@pytest.mark.parametrize("raises", [False, True]) +def test_instrumented_invoke_owns_header_client_and_closes_it(monkeypatch, raises): + from types import SimpleNamespace + from unittest.mock import Mock + from lmi_tests import cloud as module + + isolated = Mock() + owner = SimpleNamespace(lam=Mock()) + monkeypatch.setattr(module, "client", lambda name: isolated) + expected = ( + "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=0" + ) + + def send(**kwargs): + event, hook = isolated.meta.events.register.call_args.args + assert event == "before-call.lambda.Invoke" + params = {"headers": {}} + hook(params=params) + assert params["headers"]["X-Amzn-Trace-Id"] == expected + assert kwargs["Payload"] == b"payload" + if raises: + raise RuntimeError("transport failed") + return {"DurableExecutionArn": "execution"} + + isolated.invoke.side_effect = send + caller = module.RunnerClient(owner, expected) + if raises: + with pytest.raises(RuntimeError, match="transport failed"): + caller.invoke(Payload=b"payload") + else: + assert caller.invoke(Payload=b"payload")["DurableExecutionArn"] == "execution" + isolated.close.assert_called_once() + owner.lam.invoke.assert_not_called() diff --git a/pyproject.toml b/pyproject.toml index fa7272aa0..6fed26f94 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,8 +33,12 @@ python = "3.14" workspace.members = [ "packages/aws-durable-execution-sdk-python", "packages/aws-durable-execution-sdk-python-testing", + "packages/aws-durable-execution-sdk-python-otel", +] +dependencies = [ + "boto3>=1.43.0", "pytest", "ruff", "build", + "opentelemetry-sdk>=1.20.0", "opentelemetry-propagator-aws-xray", ] -dependencies = ["boto3>=1.43.0", "pytest", "ruff", "build"] [tool.hatch.envs.lmi.scripts] unit = "pytest lmi_tests/tests/unit {args}" From de4588d1bfb2c2c956fb866a3a2e3225565452bc Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:32:57 -0700 Subject: [PATCH 13/32] ci: refresh credentials for LMI test phases --- .github/workflows/lmi-e2e-tests.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 95c76157a..33c0a95ca 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -94,9 +94,24 @@ jobs: hatch run lmi:python -m lmi_tests.deploy deploy \ --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" + - name: Refresh credentials for cloud assertions + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ secrets.TEST_ROLE_ARN }} + role-session-name: python-lmi-assertions + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - name: Run cloud regressions (red until issue 741 is fixed) timeout-minutes: 60 run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml + - name: Refresh credentials for evidence and cleanup + if: always() && hashFiles('lmi_tests/artifacts/manifest.json') != '' + uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 + with: + role-to-assume: ${{ secrets.TEST_ROLE_ARN }} + role-session-name: python-lmi-cleanup + aws-region: ${{ env.AWS_REGION }} + allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - name: Collect service history and runtime evidence if: always() timeout-minutes: 5 From b6915853ce1541edd0f66bc4be752f3288e0db65 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 23 Oct 2026 14:49:46 -0700 Subject: [PATCH 14/32] test: preserve the observed SDK return boundary --- lmi_tests/fixture.py | 23 ++++++++++++++++++++--- lmi_tests/tests/unit/test_telemetry.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index ecb88c2b4..de73c9e2f 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -68,6 +68,8 @@ def __init__(self, payload, context, execution_arn): self.bucket = os.environ["LMI_BUCKET"] self.lock = threading.Lock() self.sequence = 0 + self.sdk_completed_at = None + self.sdk_resources = None self.identity = { "run": os.environ["LMI_RUN_ID"], "commit": os.environ["LMI_COMMIT"], @@ -457,7 +459,12 @@ def invoke(plugins=None): decorated = durable_execution( lambda e, c: workflow(e, c, trace), plugins=plugins ) - return decorated(injected, context) + try: + return decorated(injected, context) + finally: + # Preserve the actual SDK boundary before post-call evidence I/O. + trace.sdk_completed_at = time.time() + trace.sdk_resources = resources() before = resources() trace.emit("WRAPPER_ENTER", resources=before) @@ -468,8 +475,18 @@ def invoke(plugins=None): result = invoke_with_telemetry(trace, invoke) else: result = invoke() - trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) + trace.emit( + "WRAPPER_RETURN", + status=result["Status"], + resources=trace.sdk_resources, + time=trace.sdk_completed_at, + ) return result except BaseException as error: - trace.emit("WRAPPER_RAISE", error=type(error).__name__, resources=resources()) + trace.emit( + "WRAPPER_RAISE", + error=type(error).__name__, + resources=getattr(trace, "sdk_resources", None) or resources(), + time=getattr(trace, "sdk_completed_at", None) or time.time(), + ) raise diff --git a/lmi_tests/tests/unit/test_telemetry.py b/lmi_tests/tests/unit/test_telemetry.py index 469c8ea3b..6ad509c1c 100644 --- a/lmi_tests/tests/unit/test_telemetry.py +++ b/lmi_tests/tests/unit/test_telemetry.py @@ -253,3 +253,19 @@ def send(**kwargs): assert caller.invoke(Payload=b"payload")["DurableExecutionArn"] == "execution" isolated.close.assert_called_once() owner.lam.invoke.assert_not_called() + + +def test_post_call_evidence_does_not_move_the_sdk_boundary(monkeypatch): + original = MemoryTrace.emit + + def delayed(self, phase, **fields): + record = original(self, phase, **fields) + if phase == "OTEL_EXIT": + time.sleep(0.02) # Controlled diagnostic I/O latency, not a durable wait. + return record + + monkeypatch.setattr(MemoryTrace, "emit", delayed) + events = run(monkeypatch, "invocation", "boundary") + boundary = next(event for event in events if event["phase"] == "WRAPPER_RETURN") + evidence_exit = next(event for event in events if event["phase"] == "OTEL_EXIT") + assert boundary["time"] <= evidence_exit["time"] From aaaf2b602c48366097c0c91949bd6466ecf51fcb Mon Sep 17 00:00:00 2001 From: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:52:23 -0700 Subject: [PATCH 15/32] docs(lmi): describe shared conformance adapter design --- lmi_tests/shared-suite-design.md | 76 ++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 lmi_tests/shared-suite-design.md diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md new file mode 100644 index 000000000..8cdbdc495 --- /dev/null +++ b/lmi_tests/shared-suite-design.md @@ -0,0 +1,76 @@ +# Shared LMI conformance adapter + +## Problem + +Python's LMI regression harness and the shared conformance runner currently use +independent drivers, evidence formats and acceptance reports. Merging both test +PRs does not make the Python fixtures satisfy shared coverage or ownership rules. + +## Cause + +The SDK harness owns the actual public-API functions, deployment, S3 ledger and +service calls. The shared runner owns the multi-invocation scenario matrix, +resource leases, validators and strict reports. The existing ledger does not yet +record every operation, export and lifecycle boundary the shared schema requires. + +## Options considered + +1. Keep invoking pytest from the shared adapter. This reuses the old command but + does not expose placement, collection and retirement as separate phases, or + bind every native invocation to the shared ownership journal. +2. Copy the shared validators into this repository. This preserves two competing + acceptance implementations and makes version/replay comparisons unreliable. +3. Adapt the existing Cloud/fixture implementation and add the missing public-API + observations, while delegating scheduling and acceptance to a pinned shared + runner. This reuses the actual SDK behavior and eliminates the independent CI + acceptance path without fabricating missing evidence. + +## Chosen design + +Use option 3, following the requested adoption of conformance PR #186. Pin the +shared runner revision. Generate its `python-lifecycle` and `export-lifecycle` +matrices for the two persistent c1/c2 functions: 76 mapped lifecycle/OTel cells +plus 24 exporter-mode cells. Keep exporter mode independent of language and choose +immutable provider variants per invocation. The shared conformance package runs +on the driver only, not inside the deployed function bundle. + +The SDK-owned adapter exposes prepare/start/place/execute/collect/release/retire +and finish. It keeps a distinct Cloud/controller state for every case, journals +all direct launches and observed service resumes, and records exact artifact, +configuration and original request identities. Public plugin hooks and the +existing forwarding Lambda client provide actual operation/checkpoint metadata. +Missing observations remain incomplete; no timestamps, completion events or SDK +capabilities are synthesized to force shared validation to pass. + +Lifecycle and telemetry records preserve actual boundary timestamps separately +from later diagnostic I/O. Normalization retains raw source evidence. Synchronous +exports settle before wrapper return; asynchronous exports and flush work retain +ownership after a bounded wait until actual completion. Case cleanup releases only +its own controls and executions. Persistent ownership uses conditional S3 writes +outside the expiring run prefix and is retained after unproven retirement. S3 +stays private and uses native SigV4/IAM authentication. + +CI retains the existing opt-in cloud policy and SDK-local unit/regression tests, +but invokes the shared runner for cloud acceptance. Reports and all attempted +case evidence are retained. The implementation and tests use the real public SDK +handler/plugin paths against the local service where practical; real-LMI +placement/recovery and deployed acceptance remain explicitly unverified until a +cloud run is recorded. + +## Removed or superseded work + +The old pytest cloud command is no longer the primary CI acceptance path. Its +SDK-specific fixture/regression tests remain available during migration. Shared +requirements and validators are not duplicated here, and this change does not +claim to fix the open runtime deadline/cleanup defects tracked by Python #741. + +## Reviewable decisions and limits + +- This PR is stacked on Python #743 and does not modify that PR's branch. +- Conformance is pinned to PR #186's exact revision, not a moving branch. +- The default target is the Python-specific 76-cell matrix plus the portable + 24-cell exporter matrix; it does not imply ordinary SDK/OTel suite acceptance + on LMI or acceptance against the separate standard profile. +- Fixture budgets are declared before a run and kept distinct from product SLAs. +- SDK bugs stay failures. No cloud success or bounded arbitrary user-I/O + termination is inferred from local harness tests. From 8d1308ca28579939fa887ad278ccf35e0c765bbf Mon Sep 17 00:00:00 2001 From: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:39:34 -0700 Subject: [PATCH 16/32] test: adapt Python LMI fixtures to shared conformance --- .github/workflows/lmi-e2e-tests.yml | 32 +- .gitignore | 2 + lmi_tests/README.md | 64 +- lmi_tests/cloud.py | 339 +++++++- lmi_tests/deploy.py | 50 +- lmi_tests/fixture.py | 400 ++++++++-- lmi_tests/shared-suite-design.md | 20 +- lmi_tests/shared-telemetry.md | 80 ++ lmi_tests/shared.py | 409 ++++++++++ lmi_tests/shared_adapter.py | 229 ++++++ lmi_tests/shared_contract.py | 40 + lmi_tests/shared_evidence.py | 687 ++++++++++++++++ lmi_tests/shared_exporter.py | 489 ++++++++++++ lmi_tests/shared_lock.py | 239 ++++++ lmi_tests/shared_observations.py | 348 ++++++++ lmi_tests/shared_scenarios.py | 603 ++++++++++++++ lmi_tests/shared_telemetry.py | 207 +++++ lmi_tests/telemetry.py | 242 ++++-- .../tests/e2e/test_shared_driver_local.py | 141 ++++ .../tests/e2e/test_shared_evidence_local.py | 100 +++ .../tests/e2e/test_shared_exports_local.py | 239 ++++++ .../e2e/test_shared_observations_local.py | 171 ++++ .../tests/e2e/test_shared_telemetry_local.py | 112 +++ lmi_tests/tests/unit/test_control.py | 14 +- lmi_tests/tests/unit/test_deploy.py | 46 +- lmi_tests/tests/unit/test_java_parity.py | 6 + lmi_tests/tests/unit/test_shared_adapter.py | 752 ++++++++++++++++++ lmi_tests/tests/unit/test_shared_cleanup.py | 94 +++ .../unit/test_shared_control_receipts.py | 46 ++ .../unit/test_shared_deadline_measurement.py | 263 ++++++ lmi_tests/tests/unit/test_shared_lock.py | 110 +++ lmi_tests/tests/unit/test_shared_manifest.py | 98 +++ .../tests/unit/test_shared_observations.py | 396 +++++++++ lmi_tests/tests/unit/test_shared_telemetry.py | 646 +++++++++++++++ lmi_tests/tests/unit/test_shared_workflow.py | 46 ++ pyproject.toml | 4 + 36 files changed, 7575 insertions(+), 189 deletions(-) create mode 100644 lmi_tests/shared-telemetry.md create mode 100644 lmi_tests/shared.py create mode 100644 lmi_tests/shared_adapter.py create mode 100644 lmi_tests/shared_contract.py create mode 100644 lmi_tests/shared_evidence.py create mode 100644 lmi_tests/shared_exporter.py create mode 100644 lmi_tests/shared_lock.py create mode 100644 lmi_tests/shared_observations.py create mode 100644 lmi_tests/shared_scenarios.py create mode 100644 lmi_tests/shared_telemetry.py create mode 100644 lmi_tests/tests/e2e/test_shared_driver_local.py create mode 100644 lmi_tests/tests/e2e/test_shared_evidence_local.py create mode 100644 lmi_tests/tests/e2e/test_shared_exports_local.py create mode 100644 lmi_tests/tests/e2e/test_shared_observations_local.py create mode 100644 lmi_tests/tests/e2e/test_shared_telemetry_local.py create mode 100644 lmi_tests/tests/unit/test_shared_adapter.py create mode 100644 lmi_tests/tests/unit/test_shared_cleanup.py create mode 100644 lmi_tests/tests/unit/test_shared_control_receipts.py create mode 100644 lmi_tests/tests/unit/test_shared_deadline_measurement.py create mode 100644 lmi_tests/tests/unit/test_shared_lock.py create mode 100644 lmi_tests/tests/unit/test_shared_manifest.py create mode 100644 lmi_tests/tests/unit/test_shared_observations.py create mode 100644 lmi_tests/tests/unit/test_shared_telemetry.py create mode 100644 lmi_tests/tests/unit/test_shared_workflow.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 33c0a95ca..f403da925 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -15,18 +15,30 @@ on: permissions: contents: read +env: + CONFORMANCE_REVISION: 6f899679c7eb6ede836c737acb9816e6fd7072fd + LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} + jobs: harness: runs-on: ubuntu-latest timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out pinned shared conformance + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: aws/aws-durable-execution-conformance-tests + ref: ${{ env.CONFORMANCE_REVISION }} + path: .conformance + persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.14' - run: pip install hatch==1.16.5 + - run: hatch run lmi:shared-setup - run: hatch run lmi:lint - - run: hatch run lmi:unit --junitxml=lmi_tests/artifacts/unit.xml + - run: hatch run lmi:shared-tests --junitxml=lmi_tests/artifacts/unit.xml - name: Verify examples workspace initialization run: | hatch run examples:python - <<'PY' @@ -68,6 +80,13 @@ jobs: TEST_LAMBDA_EXECUTION_ROLE_ARN: ${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Check out pinned shared conformance + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: aws/aws-durable-execution-conformance-tests + ref: ${{ env.CONFORMANCE_REVISION }} + path: .conformance + persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: '3.14' @@ -83,6 +102,7 @@ jobs: test -n "$TEST_LAMBDA_EXECUTION_ROLE_ARN" test "$(aws sts get-caller-identity --query Account --output text)" = "$TEST_ACCOUNT_ID" - run: pip install hatch==1.16.5 + - run: hatch run lmi:shared-setup - name: Build SDK checkout timeout-minutes: 5 run: hatch run lmi:build @@ -101,9 +121,9 @@ jobs: role-session-name: python-lmi-assertions aws-region: ${{ env.AWS_REGION }} allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Run cloud regressions (red until issue 741 is fixed) + - name: Run shared lifecycle and exporter conformance timeout-minutes: 60 - run: hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml + run: hatch run lmi:shared - name: Refresh credentials for evidence and cleanup if: always() && hashFiles('lmi_tests/artifacts/manifest.json') != '' uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 @@ -124,7 +144,7 @@ jobs: timeout-minutes: 5 run: | if test -f lmi_tests/artifacts/manifest.json; then - hatch run lmi:python -m lmi_tests.deploy cleanup + hatch run lmi:shared-cleanup fi - name: Retain diagnostics if: always() @@ -135,4 +155,6 @@ jobs: path: lmi_tests/artifacts/ - name: Summarize outcomes if: always() - run: python -m lmi_tests.summary >> "$GITHUB_STEP_SUMMARY" + run: | + python -m lmi_tests.summary >> "$GITHUB_STEP_SUMMARY" + find lmi_tests/artifacts/shared -name report.json -print 2>/dev/null >> "$GITHUB_STEP_SUMMARY" || true diff --git a/.gitignore b/.gitignore index 13ccb7ad3..516a14a52 100644 --- a/.gitignore +++ b/.gitignore @@ -38,3 +38,5 @@ dist/ **/template.generated.json .env + +/.conformance/ diff --git a/lmi_tests/README.md b/lmi_tests/README.md index f50f1c08e..4450a7796 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -1,4 +1,13 @@ -# Python LMI end-to-end tests +# Python LMI shared conformance adapter + +The cloud entry point is now the shared runner from +[conformance PR #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186), +pinned to `6f899679c7eb6ede836c737acb9816e6fd7072fd`. The Python adapter runs +`python-lifecycle` (76 cells) and `export-lifecycle` (24 cells) against the same +persistent c1/c2 functions. The latter covers both OTel views with synchronous +export and asynchronous export with a bounded flush wait. Shared requirements, +validation, JSON/JUnit reports and ownership journals come from that pinned +checkout; the conformance packages are installed on the driver only. This suite implements [#742](https://github.com/aws/aws-durable-execution-sdk-python/issues/742) and asserts the invocation-lifetime behavior requested by @@ -18,7 +27,11 @@ ordinary local test suite remains usable while the fix is designed. Use Python 3.14 and Hatch from the repository root: ```sh -hatch run lmi:unit +# Check out the exact shared runner revision first. +gh repo clone aws/aws-durable-execution-conformance-tests .conformance +git -C .conformance checkout 6f899679c7eb6ede836c737acb9816e6fd7072fd +hatch run lmi:shared-setup +hatch run lmi:shared-tests hatch run lmi:lint hatch run lmi:regressions --junitxml=lmi_tests/artifacts/regressions.xml ``` @@ -68,15 +81,15 @@ is substituted. The artifact hash, SDK wheel version, and commit are retained. hatch run lmi:build hatch run lmi:python -m lmi_tests.deploy deploy \ --stack-name python-lmi-e2e --run-id local-20260923-a --runtime python3.14 -hatch run lmi:cloud --junitxml=lmi_tests/artifacts/cloud.xml +hatch run lmi:shared # Run both commands below even when a scenario fails: hatch run lmi:python -m lmi_tests.deploy collect -hatch run lmi:python -m lmi_tests.deploy cleanup +hatch run lmi:shared-cleanup python -m lmi_tests.summary ``` For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls -`collect` followed by `cleanup`; the workflow already has unconditional steps. +`collect` followed by `shared-cleanup`; the workflow already has unconditional steps. Use a fresh run ID for each run and reuse the same stack name. Before build or deployment preflight, the driver removes prior run metadata and reports from `lmi_tests/artifacts`, preserving `build.json` for artifact verification. Archive @@ -84,10 +97,40 @@ any evidence you need before starting another run. The run manifest is invalidat first, so a failed build, invalid configuration, or account/provider check cannot make an exit trap collect or clean up the preceding run. This deployment supplies both native environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud -command runs 14 lifecycle and 24 observability cases for each setting (76 cases), sequentially, without -redeploying or changing function configuration between cases. Independent SDK +command runs 100 shared coverage cells. The shared coordinator reserves native +capacity for each case and serializes exclusive cases, without redeploying or +changing function configuration between cases. Independent SDK branch-concurrency settings include 1, 2, and 3. +To launch the same cloud workflow from an SDK branch: + +```sh +gh workflow run lmi-e2e-tests.yml --repo aws/aws-durable-execution-sdk-python \ + --ref YOUR_BRANCH -f region=us-west-2 +``` + +CI first runs the local harness, then assumes the repository's configured test +role and deploys the checked-out SDK and OTel wheels. It uploads the raw S3/service +records, shared manifests, attempt journals, coverage inventories, and JSON/JUnit +reports under `lmi_tests/artifacts/shared/`. Missing placement or source evidence +is a failure, not an accepted skip. Local integration checks exercise the real +public SDK and local checkpoint service; they do not establish LMI recovery. + +A non-expiring conditional S3 ownership record under `ownership//` fences +updates to the shared functions. Case controls live under separate run/case +prefixes. Normal completion releases ownership only when both profiles prove +quiescence. Interrupted or unproven retirement retains ownership for reconciliation; +`shared-cleanup` releases only that run's recorded controls and executions and +cannot turn an unknown retirement into success. Do not delete the record merely +to bypass a blocked deployment. + +The fixture's asynchronous processor waits up to five seconds through the real +provider flush API; shared acceptance allows ten seconds for that wait boundary. +Export completion is tracked separately and remains owned after wrapper return. +These are declared test budgets, not SDK SLAs. See +[shared telemetry observations](shared-telemetry.md) for capture/publication +ordering and [the adapter design](shared-suite-design.md) for ownership details. + The first run creates the persistent `python-lmi-e2e` stack, one private bucket, and two functions/log groups (`python-lmi-e2e-c1`, `python-lmi-e2e-c2`). Later runs update these same resources and preserve their names/ARNs. Select another dedicated @@ -114,7 +157,7 @@ The handler mappings are saved under `c1` and `c2` in `DurableFunctionCloudTestRunner` for async invocation, callback completion, and waiting for results, with finite boto transport timeouts. -## Evidence and regression contracts +## SDK fixture and regression contracts | Scenario | Evidence required | | --- | --- | @@ -126,6 +169,11 @@ waiting for results, with finite boto transport timeouts. | Invocation deadline (#741) | A real short function timeout, service invocation-completed error for the exact request, external side effects, full original-environment worker capacity recovery, and healthy concurrent work on another process for c2 | | Timeout/retry | A second service invocation of the **same** durable execution, completed step skipped, original attempt effects cease; repeated interrupted at-least-once work is recorded separately and is not called exactly-once | +The following sections describe the retained SDK-local fixtures and legacy +pytest regression entry points. Cloud acceptance uses the shared manifests and +validators described above. Java-specific executor, handler-owner and Context/MDC +assertions remain the Java SDK's responsibility. + ### Java lifecycle coverage in Python The analogous cases from Java PR #728 are mapped to Python's public APIs: diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 1c3c505ca..09a76d030 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -3,7 +3,12 @@ """Bounded real-service driver using the repository's cloud runner.""" import json +import hashlib +from pathlib import Path +from threading import Event, Lock +from typing import Any from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy import time import uuid @@ -22,11 +27,14 @@ class RunnerClient: """Share paced history reads with the public runner; other APIs are unchanged.""" - def __init__(self, cloud, trace_header=None): + def __init__(self, cloud, trace_header=None, *, before_call=None): self.cloud = cloud self.trace_header = trace_header + self.before_call = before_call def invoke(self, **kwargs): + if self.before_call is not None: + self.before_call() if self.trace_header is None: return self.cloud.lam.invoke(**kwargs) # An instrumented caller's HTTP carrier goes through the real Lambda @@ -43,21 +51,138 @@ def add_carrier(params, **_kwargs): isolated.close() def __getattr__(self, name): - return getattr(self.cloud.lam, name) + operation = getattr(self.cloud.lam, name) + if not callable(operation): + return operation + + def guarded(**kwargs): + if self.before_call is not None: + self.before_call() + return operation(**kwargs) + + return guarded def get_durable_execution_history(self, **kwargs): return self.cloud.history_page(**kwargs) class Cloud: - def __init__(self, manifest, configuration=None): + def __init__( + self, manifest, configuration=None, *, artifacts_dir=None, case_context=None + ): self.manifest = manifest self.configuration = configuration self.lam, self.s3, self.logs = client("lambda"), client("s3"), client("logs") self.events = {} self.invocations = [] self.gates = set() + self.released_gates: set[str] = set() self._next_history_read = 0.0 + self.artifacts_dir = ( + Path(artifacts_dir) if artifacts_dir is not None else ARTIFACTS + ) + self.case_context = case_context + self.cancel_event = Event() + self._history_lock = Lock() + self._retiring = False + self.request_roles: dict[str, str] = {} + self.request_launches: dict[str, str] = {} + self.raw_histories: dict[str, list[dict[str, Any]]] = {} + self.execution_results: dict[str, dict[str, Any]] = {} + self.control_records: list[dict[str, Any]] = [] + self.source_uris: set[str] = set() + self.observation_ended_at: float | None = None + self.deadline_measurement: dict[str, Any] | None = None + self.retired_at: float | None = None + self.callback_records: list[dict[str, Any]] = [] + self.control_prefix = f"runs/{manifest['run']}/control/" + if case_context is not None: + owner = hashlib.sha256( + f"{case_context.scenario.case_id}:{case_context.scenario.cohort_id}".encode() + ).hexdigest()[:20] + self.control_prefix = f"runs/{manifest['run']}/cases/{owner}/control/" + + def check_active(self) -> None: + if self.cancel_event.is_set(): + raise CollectionError("Case driver was cancelled") + + def cancel(self) -> None: + """Request cooperative stop before joining this case's blocking worker.""" + self.cancel_event.set() + + def begin_retirement(self) -> None: + """Called only after the cancelled phase worker has actually exited.""" + self._retiring = True + self.cancel_event.clear() + + def wait(self, seconds: float) -> None: + self.check_active() + if self.case_context is None: + time.sleep(max(0, seconds)) + elif self.cancel_event.wait(max(0, seconds)): + self.check_active() + + def _save(self, relative: str, data: Any) -> None: + save(self.artifacts_dir / relative, data) + self.source_uris.add((self.artifacts_dir / relative).resolve().as_uri()) + + def _require_execution_target(self, arn: str) -> None: + if self.case_context is not None and not arn.startswith( + self.manifest["functions"][self.configuration] + "/durable-execution/" + ): + raise CollectionError("Execution is outside the case-owned function") + + def _bind_requests(self, events: list[dict[str, Any]]) -> None: + if self.case_context is None: + return + by_marker = {item["marker"]: item for item in self.invocations} + for event in sorted( + events, key=lambda item: (item["time"], item["request"], item["sequence"]) + ): + if ( + event["phase"] != "WRAPPER_ENTER" + or event["request"] in self.request_roles + ): + continue + item = by_marker.get(event["marker"]) + if item is None: + raise CollectionError( + "Observed an invocation outside this case's launch journal" + ) + if item.get("arn") is not None and item["arn"] != event["execution"]: + raise CollectionError( + "An invocation changed its original durable execution" + ) + self._require_execution_target(event["execution"]) + item["arn"] = event["execution"] + ordinal = len(item["native_requests"]) + role = item["role"] + if ordinal and item["resume_roles"]: + role = item["resume_roles"][ + min(ordinal - 1, len(item["resume_roles"]) - 1) + ] + launch_id = ( + item["launch_id"] + if ordinal == 0 + else f"{item['launch_id']}:native:{event['request']}" + ) + if ordinal == 0: + self.case_context.update_attempt( + launch_id, + requestId=event["request"], + executionArn=event["execution"], + ) + else: + self.case_context.record_attempt( + launch_id, + role=role, + requestId=event["request"], + executionArn=event["execution"], + source="service-observed", + ) + item["native_requests"].append(event["request"]) + self.request_roles[event["request"]] = role + self.request_launches[event["request"]] = launch_id def verify(self): if self.manifest.get("concurrencies") != CONCURRENCIES or set( @@ -86,13 +211,34 @@ def start( otel_view=None, trace_header=None, otel_export="success", + role=None, + resume_roles=(), + dispatcher_payload=None, + marker=None, ): + self.check_active() + if self._retiring: + raise CollectionError("A retiring case cannot launch new executions") if self.configuration not in self.manifest["functions"]: raise ProvisioningError( "Select a deployed concurrency configuration before Invoke" ) - marker = scenario + "-" + uuid.uuid4().hex[:12] - payload = {"scenario": scenario, "marker": marker, "run": self.manifest["run"]} + marker = marker or scenario + "-" + uuid.uuid4().hex[:12] + payload = dict(dispatcher_payload or {}) + if dispatcher_payload: + payload["sharedScenario"] = dispatcher_payload["scenario"] + payload.update(scenario=scenario, marker=marker, run=self.manifest["run"]) + if self.case_context is not None: + payload["controlPrefix"] = self.control_prefix + if "release-all" not in self.gates: + self.hold("release-all") + if role not in self.case_context.scenario.roles: + raise ValueError("Launch role is not part of the admitted scenario") + if any( + resume not in self.case_context.scenario.roles + for resume in resume_roles + ): + raise ValueError("Resume role is not part of the admitted scenario") if otel_view is not None: if otel_view not in {"invocation", "execution"}: raise ValueError("Unknown OTel view") @@ -125,29 +271,50 @@ def start( self.manifest["functions"][self.configuration], region=self.manifest["region"], ) - runner.lambda_client = RunnerClient(self, trace_header) - try: - arn = runner.run_async(input=payload) - if not arn: - raise CollectionError("Invoke did not return a durable execution ARN") - except Exception as error: - save( - ARTIFACTS / f"invocations/{marker}.json", - {"payload": payload, "error": str(error)}, - ) - raise CollectionError(f"{marker}: invocation API failed") from error + runner.lambda_client = RunnerClient( + self, trace_header, before_call=self.check_active + ) item = { "marker": marker, - "arn": arn, + "arn": None, "configuration": self.configuration, "runner": runner, "started": time.time(), "trace_header": trace_header, + "payload": payload, + "role": role, + "resume_roles": tuple(resume_roles), + "native_requests": [], + "launch_id": marker, } - self.invocations.append(item) + if self.case_context is not None: + # Retain attempted writes before the transport, even if its response + # is lost. Later ledger observations can recover their native IDs. + self.invocations.append(item) + self.case_context.record_attempt( + marker, role=role, marker=marker, source="driver-invoke" + ) + self._save( + f"invocations/{marker}.json", + {"payload": payload, "launched": item["started"]}, + ) + self.check_active() + try: + arn = runner.run_async(input=payload) + if not arn: + raise CollectionError("Invoke did not return a durable execution ARN") + self._require_execution_target(arn) + except Exception as error: + self._save( + f"invocations/{marker}.json", {"payload": payload, "error": str(error)} + ) + raise CollectionError(f"{marker}: invocation API failed") from error + item["arn"] = arn + if self.case_context is None: + self.invocations.append(item) print(f"LMI {scenario}: marker={marker}, execution={arn}", flush=True) - save( - ARTIFACTS / f"invocations/{marker}.json", + self._save( + f"invocations/{marker}.json", { "payload": payload, "arn": arn, @@ -159,12 +326,13 @@ def start( return item def hold(self, gate): + self.check_active() if gate in self.gates: return # another invocation in this case uses the same barrier try: self.s3.put_object( Bucket=self.manifest["bucket"], - Key=f"runs/{self.manifest['run']}/control/{gate}", + Key=f"{self.control_prefix}{gate}", Body=b"hold", IfNoneMatch="*", ) @@ -174,19 +342,50 @@ def hold(self, gate): f"Cannot initialize control {gate}: {error}" ) from error self.gates.add(gate) + self.control_records.append( + {"gate": gate, "action": "hold", "time": time.time()} + ) def release(self, gate): + self.check_active() + if gate in self.released_gates: + return + if self.case_context is not None and gate not in self.gates: + raise CollectionError( + "Refusing to release a control not owned by this case" + ) + intent = { + "gate": gate, + "action": "release", + "started": time.time(), + "state": "PENDING", + } + self.control_records.append(intent) + self._save("control-records.json", self.control_records) self.s3.put_object( Bucket=self.manifest["bucket"], - Key=f"runs/{self.manifest['run']}/control/{gate}", + Key=f"{self.control_prefix}{gate}", Body=b"release", ) + intent.update(time=time.time(), state="ACKNOWLEDGED") + self.released_gates.add(gate) + self._save("control-records.json", self.control_records) def refresh(self, markers=None, *, validate_controls=True): + self.check_active() + if self.case_context is not None and markers == []: + raise CollectionError( + "A shared case cannot collect another case's run prefix" + ) # A new pytest fixture must not scan every earlier case before it can # observe its first event. Explicit [] requests the complete run for collection. if markers is None: markers = [item["marker"] for item in self.invocations] + if self.case_context is not None: + if not markers: + return [] + if not set(markers) <= {item["marker"] for item in self.invocations}: + raise CollectionError("A shared case cannot read another case's marker") prefix = f"runs/{self.manifest['run']}/events/" prefixes = [f"{prefix}{marker}/" for marker in markers] if markers else [prefix] missing = [] @@ -202,6 +401,7 @@ def refresh(self, markers=None, *, validate_controls=True): ) def read(key): + self.check_active() response = self.s3.get_object(Bucket=self.manifest["bucket"], Key=key) with response["Body"] as stream: event = json.loads(stream.read()) @@ -210,6 +410,7 @@ def read(key): or event["commit"] != self.manifest["commit"] ): raise CollectionError("Event came from another build/run") + self.check_active() return key, event if missing: @@ -222,7 +423,12 @@ def read(key): events = sorted( self.events.values(), key=lambda e: (e["time"], e["request"], e["sequence"]) ) - save(ARTIFACTS / "events.json", events) + self.check_active() + self._bind_requests(events) + self.source_uris.update( + f"s3://{self.manifest['bucket']}/{key}" for key in self.events + ) + self._save("events.json", events) selected = [e for e in events if not markers or e["marker"] in markers] if markers and validate_controls: check_controls(selected) @@ -240,16 +446,65 @@ def poll( ): end = time.monotonic() + seconds while True: + self.check_active() value = predicate() if value: return value if time.monotonic() >= end: raise category(message) - time.sleep(0.5) + self.wait(min(0.5, max(0, end - time.monotonic()))) def phase(self, item, phase, **kwargs): return self.poll(lambda: select(self.for_item(item), phase), **kwargs) + def record_deadline_measurement(self, snapshot: dict[str, Any]) -> None: + """Persist the completed measurement before any victim-fault release.""" + self.check_active() + recorded = deepcopy(snapshot) + self._save("measurements/deadline.json", recorded) + self.deadline_measurement = recorded + + def wait_terminal(self, item: dict[str, Any]) -> dict[str, Any]: + """Observe actual terminal state and wrapper exits without asserting success. + + Deadline fixtures can legitimately finish with several service outcomes. + Their declared conformance expectations decide acceptance; this method + only observes completion and does not stop or retry an execution. + """ + self._require_execution_target(item["arn"]) + + def terminal() -> dict[str, Any] | None: + result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) + if result["Status"] in {"SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"}: + return result + return None + + result = self.poll( + terminal, + seconds=self.manifest["driverTimeout"], + message="Released victim execution did not reach an actual terminal state", + ) + self.execution_results[item["arn"]] = result + self._save(f"executions/{item['marker']}.json", result) + + def wrappers_closed() -> bool: + events = self.for_item(item) + entered = {event["request"] for event in select(events, "WRAPPER_ENTER")} + exited = { + event["request"] + for event in events + if event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + } + return bool(entered) and entered <= exited + + self.poll( + wrappers_closed, + seconds=self.manifest["driverTimeout"], + message="Released victim has native invocations without actual wrapper exits", + ) + self.history(item) + return result + def finish(self, item, status="SUCCEEDED"): try: item["runner"].wait_for_result( @@ -260,7 +515,8 @@ def finish(self, item, status="SUCCEEDED"): "Driver timeout, not Lambda invocation timeout evidence" ) from error result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) - save(ARTIFACTS / f"executions/{item['marker']}.json", result) + self.execution_results[item["arn"]] = result + self._save(f"executions/{item['marker']}.json", result) assert result["Status"] == status, ( f"{item['marker']}: {result['Status']} (expected {status})" ) @@ -272,15 +528,21 @@ def finish(self, item, status="SUCCEEDED"): return self.history(item) def history_page(self, **request): + """Serialize paced reads within one case, without shared-case throttling state.""" + with self._history_lock: + return self._history_page(**request) + + def _history_page(self, **request): """Bound read-only throttling retries without replaying invocation writes.""" deadline = time.monotonic() + 20 last_error = None for attempt in range(5): + self.check_active() delay = max(0, self._next_history_read - time.monotonic()) if time.monotonic() + delay >= deadline: break if delay: - time.sleep(delay) + self.wait(delay) try: return self.lam.get_durable_execution_history(**request) except ClientError as error: @@ -308,7 +570,8 @@ def history(self, item): marker = result.get("NextMarker") if not marker: break - save(ARTIFACTS / f"histories/{item['marker']}.json", events) + self.raw_histories[item["arn"]] = events + self._save(f"histories/{item['marker']}.json", events) return events def platform_timeout(self, item, request): @@ -331,6 +594,9 @@ def observed(): ) def collect(self, *, full_run=True): + self.check_active() + if self.case_context is not None and full_run: + raise CollectionError("A shared case may collect only its own invocations") errors = [] if not full_run and not self.invocations: return @@ -349,14 +615,13 @@ def collect(self, *, full_run=True): errors.append(str(error)) items = {} if not full_run: - items.update({i["arn"]: i for i in self.invocations}) + items.update({i["arn"]: i for i in self.invocations if i.get("arn")}) for item in items.values(): try: self.history(item) - save( - ARTIFACTS / f"executions/{item['marker']}.json", - self.lam.get_durable_execution(DurableExecutionArn=item["arn"]), - ) + result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) + self.execution_results[item["arn"]] = result + self._save(f"executions/{item['marker']}.json", result) except Exception as error: errors.append(str(error)) for key, arn in self.manifest["functions"].items() if full_run else []: @@ -368,15 +633,17 @@ def collect(self, *, full_run=True): startTime=int(self.manifest["created"] * 1000), ): logs.extend(page.get("events", [])) - save(ARTIFACTS / f"logs/{key}.json", logs) + self._save(f"logs/{key}.json", logs) except Exception as error: errors.append(str(error)) if errors: - save(ARTIFACTS / "collection-errors.json", errors) + self._save("collection-errors.json", errors) raise CollectionError("; ".join(errors)) def release_all(self): - for gate in self.gates: + for gate in sorted( + self.gates - self.released_gates, key=lambda value: value != "release-all" + ): self.release(gate) self.gates.clear() @@ -387,7 +654,7 @@ def run_invocations(self): e["execution"]: {"arn": e["execution"], "marker": e["marker"]} for e in events } - for path in (ARTIFACTS / "invocations").glob("*.json"): + for path in (self.artifacts_dir / "invocations").glob("*.json"): record = json.loads(path.read_text()) if ( record.get("arn") diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 28038cc9e..cb82f7461 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -20,6 +20,7 @@ from botocore.exceptions import ClientError from lmi_tests.evidence import ProvisioningError +from lmi_tests.shared_lock import RunLease ROOT = Path(__file__).resolve().parents[1] @@ -129,7 +130,13 @@ def build(): check=True, ) (target / "lmi_tests").mkdir() - for name in ("__init__.py", "fixture.py", "telemetry.py"): + for name in ( + "__init__.py", + "fixture.py", + "telemetry.py", + "shared_observations.py", + "shared_exporter.py", + ): shutil.copy(ROOT / "lmi_tests" / name, target / "lmi_tests" / name) artifact = directory / "function.zip" resolved = json.loads((directory / "dependencies.json").read_text()) @@ -327,6 +334,11 @@ def verify(config, scaling, manifest, key): actual.get("PerExecutionEnvironmentMaxConcurrency") == manifest["concurrencies"][key], config.get("Runtime") == manifest["runtime"], + config.get("Role") == manifest["role"], + config.get("Handler") == "lmi_tests.fixture.handler", + not config.get("Layers", []), + config.get("Environment", {}).get("Variables", {}).get("LMI_BUCKET") + == manifest["bucket"], config.get("Architectures") == ["arm64"], config.get("Version") == QUALIFIER, config.get("CodeSha256") == manifest["codeSha256"], @@ -444,6 +456,42 @@ def deploy_stack(manifest, data): raise ProvisioningError( f"Persistent stack requires recovery from {stack['StackStatus']}; it was retained" ) + storage = client("s3") + storage.put_public_access_block( + Bucket=manifest["bucket"], + PublicAccessBlockConfiguration={ + "BlockPublicAcls": True, + "IgnorePublicAcls": True, + "BlockPublicPolicy": True, + "RestrictPublicBuckets": True, + }, + ) + protection = storage.get_public_access_block(Bucket=manifest["bucket"])[ + "PublicAccessBlockConfiguration" + ] + if not all( + protection.get(name) is True + for name in ( + "BlockPublicAcls", + "IgnorePublicAcls", + "BlockPublicPolicy", + "RestrictPublicBuckets", + ) + ): + raise ProvisioningError( + "Evidence bucket must be private before uploading the tested artifact" + ) + partition = manifest["provider"].split(":")[1] + targets = tuple( + f"arn:{partition}:lambda:{manifest['region']}:{manifest['account']}:function:{manifest['stack']}-{key}:{QUALIFIER}" + for key in CONCURRENCIES + ) + lease = RunLease.acquire( + manifest["bucket"], manifest["stack"], manifest["run"], targets, s3=storage + ) + manifest["sharedLease"] = lease.to_manifest() + save(ARTIFACTS / "manifest.json", manifest) + if stack is not None: wait_for_idle_functions(stack) manifest["owned"] = True save(ARTIFACTS / "manifest.json", manifest) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index de73c9e2f..25d46ea19 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """Fault-injection handlers for real LMI. Never install these as SDK examples.""" +from contextlib import nullcontext import fcntl import json import os @@ -29,6 +30,11 @@ from aws_durable_execution_sdk_python.lambda_service import LambdaClient from aws_durable_execution_sdk_python.retries import RetryDecision from aws_durable_execution_sdk_python.exceptions import OrphanedChildException +from lmi_tests.shared_observations import ( + GateRelease, + LifecycleObservationFactory, + LifecycleObservations, +) PROCESS_ID = uuid.uuid4().hex @@ -60,34 +66,100 @@ def resources(): } +def observe(trace, kind, **fields): + """Preserve the legacy ledger when this invocation is not a shared case.""" + if trace.payload.get("coverageProfile"): + return trace.observe(kind, **fields) + return None + + +def configuration(trace, name, max_concurrency): + if trace.payload.get("coverageProfile"): + return trace.observer.configuration(name, max_concurrency) + return nullcontext() + + class Trace: def __init__(self, payload, context, execution_arn): - self.payload = payload + self.payload = dict(payload) + if "runId" in self.payload: + self.payload.setdefault("run", self.payload["runId"]) + if "otelView" in self.payload: + self.payload.setdefault("otel_view", self.payload["otelView"]) + self.shared = bool(self.payload.get("coverageProfile")) + if self.shared: + for key in ( + "runId", + "caseId", + "cohortId", + "role", + "sharedScenario", + "controlPrefix", + ): + if not isinstance(self.payload.get(key), str) or not self.payload[key]: + raise ValueError(f"Shared invocation requires {key}") + if not self.payload["controlPrefix"].startswith( + f"runs/{self.payload['run']}/cases/" + ) or not self.payload["controlPrefix"].endswith("/"): + raise ValueError("Shared controls must use a case-owned run prefix") self.context = context self.s3 = boto3.client("s3", config=IO_CONFIG) self.bucket = os.environ["LMI_BUCKET"] self.lock = threading.Lock() + # Never make observation capture wait for the S3 publication lock. + self.capture_lock = threading.Lock() + self.capture_sequence = 0 self.sequence = 0 + self.call_sequence = 0 self.sdk_completed_at = None + self.sdk_observation = None self.sdk_resources = None self.identity = { "run": os.environ["LMI_RUN_ID"], "commit": os.environ["LMI_COMMIT"], - "marker": payload["marker"], - "scenario": payload["scenario"], + "marker": self.payload["marker"], + "scenario": self.payload["scenario"], "execution": execution_arn, "request": context.aws_request_id, "environment": environment_id(), "process": PROCESS_ID, "pid": os.getpid(), } - self.deadline = time.time() + context.get_remaining_time_in_millis() / 1000 + if self.shared: + self.identity.update( + { + key: self.payload[key] + for key in ( + "runId", + "caseId", + "cohortId", + "role", + "coverageProfile", + "sharedScenario", + "otelView", + "exportMode", + "behaviorBudgets", + ) + if key in self.payload + } + ) + self.control_prefix = self.payload.get( + "controlPrefix", f"runs/{self.identity['run']}/control/" + ) + self.deadline_observed_at_ms = time.time() * 1000 + self.remaining_ms = context.get_remaining_time_in_millis() + self.deadline = (self.deadline_observed_at_ms + self.remaining_ms) / 1000 + self.observer = LifecycleObservations(self) if self.shared else None def emit(self, phase, **fields): - # Diagnostic I/O is outside orchestration decisions. The ledger survives a - # stuck wrapper and does not rely on the invocation response or log delivery. + # Keep original evidence order independently of retrospective boundaries. with self.lock: self.sequence += 1 + if phase == "CONFORMANCE": + fields["observation"] = { + **fields["observation"], + "sourceSequence": self.sequence, + } event = { **self.identity, "phase": phase, @@ -105,17 +177,37 @@ def emit(self, phase, **fields): ) return event + def capture_observation(self, kind, **fields): + with self.capture_lock: + self.capture_sequence += 1 + at_ms = fields.pop("atMs", time.time_ns() / 1_000_000) + return { + "kind": kind, + "atMs": at_ms, + **fields, + "captureSequence": self.capture_sequence, + } + + def publish_observation(self, observation): + return self.emit("CONFORMANCE", observation=observation) + + def observe(self, kind, **fields): + return self.publish_observation(self.capture_observation(kind, **fields)) + + def next_call_id(self, kind): + with self.lock: + self.call_sequence += 1 + return f"{self.identity['request']}:{kind}:{self.call_sequence}" + def signal(self, name): self.s3.put_object( - Bucket=self.bucket, - Key=f"runs/{self.identity['run']}/control/{name}", - Body=b"release", + Bucket=self.bucket, Key=self.control_prefix + name, Body=b"release" ) def released(self, name): try: response = self.s3.get_object( - Bucket=self.bucket, Key=f"runs/{self.identity['run']}/control/{name}" + Bucket=self.bucket, Key=self.control_prefix + name ) with response["Body"] as body: state = body.read() @@ -123,48 +215,117 @@ def released(self, name): raise ValueError(f"Invalid control state for {name}") return state == b"release" except (ClientError, ValueError) as error: - # Every key must exist before Invoke. Missing/forbidden objects are - # fixture errors, never an implicit hold or release signal. self.emit("CONTROL_ERROR", gate=name, error=str(error)) + observe(self, "control_error", gateId=name, error=str(error)) raise def gate(self, name, *, effects=False, attempt=1, ready=None): - # Leave time to observe the real 60-second invocation timeout and - # recovery. This safety escape must never masquerade as SDK cancellation. + # This return value is checkpointed by existing controlled-io steps. + self.gate_release(name, effects=effects, attempt=attempt, ready=ready) + + def gate_release( + self, + name, + *, + effects=False, + attempt=1, + ready=None, + on_hold=None, + on_release=None, + ): + """Observe normal versus emergency release without changing step results.""" end = time.monotonic() + 150 entered = False + held_at_ms = None + task_id = self.next_call_id("io") if self.shared else None + observe(self, "io_start", taskId=task_id, gateId=name) try: - while not self.released(name) and not self.released("release-all"): + while True: + # Shared cleanup releases the case abort control first. It must + # not masquerade as a normal measurement release of the target. + if self.shared and self.released("release-all"): + mode = "emergency" + release_observed_at_ms = time.time_ns() / 1_000_000 + else: + specific_release = self.released(name) + target_observed_at_ms = time.time_ns() / 1_000_000 + mode = ( + "normal" + if specific_release + else "emergency" + if not self.shared and self.released("release-all") + else None + ) + release_observed_at_ms = ( + target_observed_at_ms + if specific_release + else time.time_ns() / 1_000_000 + ) + if mode is not None: + released = GateRelease( + name, release_observed_at_ms, mode, held_at_ms + ) + if on_release is not None: + on_release(released) + observe( + self, + "fault_released", + gateId=name, + mode=mode, + atMs=released.observed_at_ms, + source="control-poll", + ) + return released if not entered: + held_at_ms = target_observed_at_ms + if on_hold is not None: + on_hold(held_at_ms) self.emit("BLOCKED", gate=name) + observe(self, "blocked", gateId=name, atMs=held_at_ms) if time.monotonic() >= end: self.emit("ESCAPE", gate=name) + observe(self, "escape", gateId=name) raise TimeoutError( "Test emergency release; never a passing regression" ) if effects: - # Each record is an independently identifiable external side effect. - # Do not stop it using the test's own Lambda deadline: that would - # mask the missing SDK cancellation/worker-retirement behavior. self.emit( "EFFECT", operation="blocked-io", gate=name, attempt=attempt ) + observe( + self, + "effect", + operation="blocked-io", + gateId=name, + attempt=attempt, + ) else: self.emit("ALIVE", gate=name) + observe(self, "alive", gateId=name) if not entered and ready: self.signal(ready) entered = True time.sleep(0.2) # fault-injected blocking I/O, not a durable delay except BaseException as error: self.emit("IO_INTERRUPTED", gate=name, error=type(error).__name__) + observe( + self, + "io_interrupted", + taskId=task_id, + gateId=name, + error=type(error).__name__, + ) raise finally: + exited_at = time.time() * 1000 self.emit("IO_EXIT", gate=name) + observe(self, "io_exit", taskId=task_id, gateId=name, atMs=exited_at) def body(self, name, value, step=None): - self.emit( - "BODY", operation=name, value=value, attempt=getattr(step, "attempt", 1) - ) + attempt = getattr(step, "attempt", 1) + self.emit("BODY", operation=name, value=value, attempt=attempt) + if self.shared: + self.observer.body(name, value, attempt) return value @@ -175,11 +336,19 @@ def __init__(self, client, trace): self.client, self.trace = client, trace def get_execution_state(self, **kwargs): + call_id = ( + self.trace.next_call_id("poll") + if self.trace.payload.get("coverageProfile") + else None + ) self.trace.emit("POLL_CALL") + observe(self.trace, "poll_call", callId=call_id) try: return self.client.get_execution_state(**kwargs) finally: + exited_at = time.time() * 1000 self.trace.emit("POLL_EXIT") + observe(self.trace, "poll_exit", callId=call_id, atMs=exited_at) def checkpoint(self, **kwargs): updates = [ @@ -191,10 +360,25 @@ def checkpoint(self, **kwargs): } for u in kwargs["updates"] ] + call_id = ( + self.trace.next_call_id("checkpoint") + if self.trace.payload.get("coverageProfile") + else None + ) self.trace.emit("CHECKPOINT_CALL", updates=updates) + observe(self.trace, "checkpoint_call", callId=call_id, updates=updates) try: response = self.client.checkpoint(**kwargs) + acknowledged_at = time.time() * 1000 self.trace.emit("CHECKPOINT_ACK", updates=updates) + observe( + self.trace, + "checkpoint_ack", + callId=call_id, + operationIds=[item["id"] for item in updates], + source="service", + atMs=acknowledged_at, + ) if self.trace.payload["scenario"] == "checkpoint" and any( u["name"] == "checkpoint-loser" and u["action"] == "SUCCEED" for u in updates @@ -202,54 +386,90 @@ def checkpoint(self, **kwargs): # Real service response, withheld while a synchronous waiter owns a # branch. This tests settlement/join ordering without a fake service. self.trace.emit("ACK_HELD") + observed_operation = next( + item["id"] + for item in updates + if item["name"] == "checkpoint-loser" + and item["action"] == "SUCCEED" + ) + observe( + self.trace, + "ack_held", + callId=call_id, + operationId=observed_operation, + gateId=self.trace.payload["marker"] + "-checkpoint", + ) self.trace.gate(self.trace.payload["marker"] + "-checkpoint") return response except BaseException as error: self.trace.emit("CHECKPOINT_ERROR", error=type(error).__name__) + observe( + self.trace, + "checkpoint_error", + callId=call_id, + error=type(error).__name__, + ) raise finally: + exited_at = time.time() * 1000 self.trace.emit("CHECKPOINT_EXIT") + observe(self.trace, "checkpoint_exit", callId=call_id, atMs=exited_at) def nested_progress(context, trace, marker): - """Nested public operations must progress even when each branch pool has one lane.""" + """Record public one-lane configurations without changing nested work.""" trace.emit("PROGRESS_BEGIN") + observe(trace, "progress_begin") def children(child): def mapped(branch): - return branch.map( - [0, 1], - lambda inner, item, _index, _all: inner.run_in_child_context( - lambda leaf: leaf.step( - lambda step: trace.body("leaf", f"{marker}:{item}", step), - name="leaf", + config = MapConfig(max_concurrency=1) + with configuration(trace, "inner-map", config.max_concurrency): + return branch.map( + [0, 1], + lambda inner, item, _index, _all: inner.run_in_child_context( + lambda leaf: leaf.step( + lambda step: trace.body("leaf", f"{marker}:{item}", step), + name="leaf", + ), + name="grandchild", ), - name="grandchild", - ), - name="inner-map", - config=MapConfig(max_concurrency=1), - ).get_results() + name="inner-map", + config=config, + ).get_results() - return child.parallel( - [ - mapped, - lambda branch: branch.run_in_child_context( - lambda leaf: leaf.step( - lambda step: trace.body("sibling", marker, step), name="sibling" + config = ParallelConfig(max_concurrency=1) + with configuration(trace, "outer-parallel", config.max_concurrency): + return child.parallel( + [ + mapped, + lambda branch: branch.run_in_child_context( + lambda leaf: leaf.step( + lambda step: trace.body("sibling", marker, step), + name="sibling", + ), + name="sibling-child", ), - name="sibling-child", - ), - ], - name="outer-parallel", - config=ParallelConfig(max_concurrency=1), - ).get_results() + ], + name="outer-parallel", + config=config, + ).get_results() result = context.run_in_child_context(children, name="progress-child") assert result == [[marker + ":0", marker + ":1"], marker] trace.emit("PROGRESS", value=result) + observe(trace, "progress", value=result) def workflow(event, context, trace): + scope = ( + trace.observer.root_scope() if event.get("coverageProfile") else nullcontext() + ) + with scope: + return run_workflow(event, context, trace) + + +def run_workflow(event, context, trace): marker, scenario = event["marker"], event["scenario"] admitted = False try: @@ -310,6 +530,12 @@ def io(step): # rejected after the parent finishes. Still test the next # SDK boundary, without swallowing unrelated failures. trace.emit("LATE_ATTEMPT") + scope_id = ( + trace.observer.closed_ancestor() + if event.get("coverageProfile") + else None + ) + observe(trace, "late_attempt", scopeId=scope_id) try: child.step( lambda step: trace.body("late-work", marker, step), @@ -318,13 +544,21 @@ def io(step): ) except OrphanedChildException as error: trace.emit("LATE_REJECTED", error=type(error).__name__) + observe( + trace, + "late_rejected", + scopeId=scope_id, + error={"type": type(error).__name__}, + ) else: trace.emit("LATE_ACCEPTED") + observe(trace, "late_accepted", scopeId=scope_id) def winning(child): def io(_): trace.gate(marker + "-loser-started") trace.emit("WINNER_READY") + observe(trace, "winner_ready") return marker return child.step(io, name="winner", config=NO_RETRY) @@ -363,6 +597,7 @@ def race(child): else: race(context) trace.emit("WINNER_SELECTED") + observe(trace, "winner_selected") if scenario == "failure-inflight": trace.emit("ROOT_FAILURE_READY") raise ValueError("expected:" + marker) @@ -375,6 +610,9 @@ def race(child): name="checkpoint-pool", config=ParallelConfig(max_concurrency=2), ) + if event.get("coverageProfile"): + scope_id = trace.observer.completed_context("checkpoint-pool") + trace.observe("branch_join", scopeId=scope_id) elif scenario == "replay": def fail(step): @@ -387,6 +625,11 @@ def fail(step): trace.emit( "STORED_FAILURE", message=error.message, error_type=error.error_type ) + observe( + trace, + "stored_failure", + error={"type": error.error_type, "message": error.message}, + ) def retry(step): trace.body("retry", marker, step) @@ -433,41 +676,79 @@ def retry(step): # Test-only root-finally diagnostics. This gate changes only exit # timing; it introduces no durable operations during suspension. trace.emit("CLEANUP_ENTER") + observe(trace, "cleanup_enter") try: trace.gate(marker + "-cleanup") finally: trace.emit("CLEANUP_EXIT") + observe(trace, "cleanup_exit") + exited_at = time.time() * 1000 trace.emit("USER_EXIT") + observe(trace, "user_exit", atMs=exited_at) def handler(event, context): - invocation = DurableExecutionInvocationInput.from_json_dict(event) + invocation = ( + event + if isinstance(event, DurableExecutionInvocationInputWithClient) + else DurableExecutionInvocationInput.from_json_dict(event) + ) payload = next( json.loads(op.execution_details.input_payload) for op in invocation.initial_execution_state.operations if op.execution_details and op.execution_details.input_payload ) + payload = dict(payload) + if "runId" in payload: + payload.setdefault("run", payload["runId"]) + if "otelView" in payload: + payload.setdefault("otel_view", payload["otelView"]) if payload["run"] != os.environ["LMI_RUN_ID"]: raise ValueError("Input belongs to a different deployment") trace = Trace(payload, context, invocation.durable_execution_arn) - client = ObservedClient(LambdaClient.initialize_client(), trace) + source_client = ( + invocation.service_client + if isinstance(invocation, DurableExecutionInvocationInputWithClient) + else LambdaClient.initialize_client() + ) + client = ObservedClient(source_client, trace) injected = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( invocation, client ) def invoke(plugins=None): + configured_plugins = ( + [LifecycleObservationFactory(trace.observer), *(plugins or [])] + if payload.get("coverageProfile") + else plugins + ) decorated = durable_execution( - lambda e, c: workflow(e, c, trace), plugins=plugins + lambda e, c: workflow(e, c, trace), plugins=configured_plugins ) try: return decorated(injected, context) finally: # Preserve the actual SDK boundary before post-call evidence I/O. - trace.sdk_completed_at = time.time() + completed_at_ns = time.time_ns() + trace.sdk_completed_at = completed_at_ns / 1_000_000_000 + if payload.get("coverageProfile"): + trace.sdk_observation = trace.capture_observation( + "sdk_completion", atMs=completed_at_ns / 1_000_000 + ) trace.sdk_resources = resources() before = resources() + entered_at = time.time() * 1000 trace.emit("WRAPPER_ENTER", resources=before) + observe(trace, "wrapper_enter", atMs=entered_at, resources=before) + if payload.get("coverageProfile"): + trace.observe( + "runtime_deadline", + atMs=trace.deadline_observed_at_ms, + remainingTimeMs=trace.remaining_ms, + deadlineAtMs=trace.deadline * 1000, + source="runtime", + ) try: if payload.get("otel_view"): from lmi_tests.telemetry import invoke as invoke_with_telemetry @@ -481,12 +762,33 @@ def invoke(plugins=None): resources=trace.sdk_resources, time=trace.sdk_completed_at, ) + if payload.get("coverageProfile"): + trace.publish_observation( + { + **trace.sdk_observation, + "kind": "wrapper_return", + "outcome": result["Status"], + "resources": trace.sdk_resources, + } + ) return result except BaseException as error: + if payload.get("coverageProfile") and trace.sdk_observation is None: + trace.sdk_observation = trace.capture_observation("sdk_completion") trace.emit( "WRAPPER_RAISE", error=type(error).__name__, resources=getattr(trace, "sdk_resources", None) or resources(), time=getattr(trace, "sdk_completed_at", None) or time.time(), ) + if payload.get("coverageProfile"): + trace.publish_observation( + { + **trace.sdk_observation, + "kind": "wrapper_raise", + "outcome": "THREW", + "error": {"type": type(error).__name__, "message": str(error)}, + "resources": getattr(trace, "sdk_resources", None) or resources(), + } + ) raise diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index 8cdbdc495..ad25c66b9 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -66,11 +66,27 @@ claim to fix the open runtime deadline/cleanup defects tracked by Python #741. ## Reviewable decisions and limits -- This PR is stacked on Python #743 and does not modify that PR's branch. -- Conformance is pinned to PR #186's exact revision, not a moving branch. +- This PR targets current main, reuses the Python #743 harness, and does not modify that PR's branch. +- Conformance is pinned to PR #186 revision `6f899679c7eb6ede836c737acb9816e6fd7072fd`. - The default target is the Python-specific 76-cell matrix plus the portable 24-cell exporter matrix; it does not imply ordinary SDK/OTel suite acceptance on LMI or acceptance against the separate standard profile. - Fixture budgets are declared before a run and kept distinct from product SLAs. - SDK bugs stay failures. No cloud success or bounded arbitrary user-I/O termination is inferred from local harness tests. + +## Observation boundaries + +The public SDK does not emit a root user-function hook. The fixture observes its +actual root entry/finally using the root operation ID supplied by public invocation +metadata. Operation names may be null. A Workflow span may start at the real +service execution start, before the current native invocation begins; history +retains that timestamp separately from invocation admission. + +The source ledger reserves capture order at each actual SDK boundary independently +of later S3 publication. Export release receipts record when the SDK observed the +control; driver release intent and acknowledgement remain separate observations. +Native timeout and wrapper return are independent outcomes. Deadline measurements +are frozen before cleanup releases the held step; cleanup cannot repair failed +behavior assertions. The shared error expectations explicitly enumerate allowed +service statuses where a measurement may precede execution termination. diff --git a/lmi_tests/shared-telemetry.md b/lmi_tests/shared-telemetry.md new file mode 100644 index 000000000..355c6f502 --- /dev/null +++ b/lmi_tests/shared-telemetry.md @@ -0,0 +1,80 @@ +# Shared telemetry observations + +The LMI fixture uses the real public invocation/execution OTel factories and +providers. `ObservedPlugin` forwards all seven lifecycle hooks and propagation +metadata, binding the invocation owner captured when that actual plugin was +created. An export snapshots that owner and the actual completed span data when +the configured processor accepts the work. Background export never falls back +to whichever invocation happens to be current later. + +Process-owned runtimes are keyed by `(otelView, exportMode)`. Selecting a variant +does not mutate environment variables, a shared plugin configuration or another +invocation's client. Providers remain alive across invocation completion and warm +reuse. Completed owners, export records and flush records are pruned after their +observations are published; the process registry does not retain past traces or +their S3 clients. + +## Export modes + +`sync` uses the standard `SimpleSpanProcessor`. Its exporter runs on the caller +of `span.end()`, so a held export prevents that call and the SDK wrapper from +returning until the gate is released. + +`async-bounded-flush` queues the same exporter work with its captured owner. The +processor's configured wait is `DEFAULT_FLUSH_TIMEOUT_MS = 5000`, capped by the +timeout supplied through the real `TracerProvider.force_flush()` API. The shared +manifest separately declares a 10-second acceptance budget, allowing observation +and scheduling overhead. Neither value is a product SLA. Tests can inject a +shorter processor wait through the private runtime constructor; invocation input +cannot mutate the process-owned variant's timeout. + +A bounded wait returning does not settle its exports. The runtime records +`flush_started`, `flush_wait_completed` and actual `flush_completed` independently, +including the actual configured/effective timeout and pending export IDs. The +original invocation continues to own asynchronous work after wrapper return. +Success and failure are observed from actual exporter results, not inferred from +an elapsed timeout. + +`Trace.gate()` still returns `None`, preserving existing checkpoint serialization. +Its private `gate_release()` observer reports the first actual held poll and the +normal or emergency release receipt. Only calls that actually waited emit +canonical `export_held`/`export_released` pairs. Later exports after an open gate +remain in the ledger without being falsely labeled as held. Shared case aborts +take precedence over individual gate releases; legacy control order is unchanged. + +## Capture, publication and retirement + +`Trace.capture_observation()` reserves a per-request `captureSequence` and actual +boundary timestamp using a small lock independent of S3 publication. The SDK +return boundary is captured before resource sampling and post-call diagnostics. +`publish_observation()` retains that capture while the original `sourceSequence` +records publication order. The shared assembler sorts by actual time and capture +sequence; it does not turn delayed log arrival into delayed SDK completion. + +Canonical telemetry observations are published on a process-owned worker so +diagnostic S3 latency cannot extend the SDK's bounded flush wait. Publication is +still owned work: failures remain outstanding and cannot produce quiescence. +Legacy uppercase OTel records remain available to the existing SDK-local tests. + +`telemetry.active_work(trace)` and `runtime.pending_for_case(case_id)` report actual +pending export/flush IDs, publication count and publication errors. A canonical +`export_quiescence` signal is emitted only after invocation closure, actual work +completion and publication of all earlier observations. The controller must also +observe the actual wrapper and lifecycle retirement; the telemetry signal alone +does not settle a durable execution or authorize a deployment update. + +## Driver conversion + +`shared_telemetry.collect_telemetry(raw_events, attempts, *, view, +expected_sampled, caller_headers, observed_until_ms, export_mode=None)` converts +actual ledger spans with the pinned shared canonical model. Caller headers and +sampling expectations come from the admitted driver's Invoke configuration; +runtime carriers come from the observed process environment. Source sequence +gaps, missing wrapper/quiescence observations and unknown export owners remain +incomplete. The converter never fills in a missing export or completion. + +Only `telemetry.py`, `shared_exporter.py` and the lifecycle observation helpers run +inside Lambda. `shared_telemetry.py`, the shared conformance packages and the +scenario coordinator run on the driver. Local tests exercise public SDK handlers, +the real local durable service, both OTel views and both export modes. They do not +establish real-LMI placement or claim to fix the runtime defects tracked by #741. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py new file mode 100644 index 000000000..97a2c6610 --- /dev/null +++ b/lmi_tests/shared.py @@ -0,0 +1,409 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run the SDK-owned fixtures through the pinned shared conformance package.""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +from copy import deepcopy +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Any, Mapping + +from lmi_tests.deploy import ARTIFACTS, ROOT, client, save +from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity +from lmi_tests.shared_lock import OwnershipError, RunLease + + +COVERAGE_PROFILES = ("python-lifecycle", "export-lifecycle") +BEHAVIOR_BUDGETS = { + "observationGraceMs": 1000, + "cleanupMs": 5000, + "recoveryMs": 5000, + "retryMs": 40000, + "progressMs": 15000, + "maxThreadGrowth": 1, + "maxFdGrowth": 8, + "maxRssGrowthBytes": 32 * 1024 * 1024, + "exportMs": 30000, + "exportCompletionMs": 30000, + "flushWaitMs": 10000, +} + + +def verify_checkout(checkout: Path) -> None: + revision = subprocess.check_output( + ["git", "-C", str(checkout), "rev-parse", "HEAD"], text=True + ).strip() + if revision != CONFORMANCE_REVISION: + raise ValueError( + f"Conformance checkout must be pinned to {CONFORMANCE_REVISION}" + ) + changed = subprocess.check_output( + ["git", "-C", str(checkout), "status", "--porcelain", "--", "packages"], + text=True, + ) + if changed: + raise ValueError("Pinned conformance package sources have local changes") + + +def setup(checkout: Path) -> None: + verify_checkout(checkout) + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "--editable", + str(checkout / "packages/aws-durable-execution-conformance-tests"), + "--editable", + str(checkout / "packages/aws-durable-execution-conformance-tests-otel"), + ], + check=True, + ) + + +def verify_install(checkout: Path) -> None: + verify_checkout(checkout) + import aws_durable_execution_conformance_tests as core + import aws_durable_execution_conformance_tests_otel as otel + + for module in (core, otel): + if not Path(module.__file__).resolve().is_relative_to(checkout.resolve()): + raise ValueError( + "Installed conformance packages are not from the pinned checkout; run shared setup" + ) + + +def make_manifest( + deployment: Mapping[str, Any], coverage: str, directory: Path +) -> dict[str, Any]: + from aws_durable_execution_conformance_tests.lmi.matrix import required_cases + + if coverage not in COVERAGE_PROFILES: + raise ValueError("Unknown Python adapter coverage profile") + if "sharedLease" not in deployment: + raise ValueError( + "Deployment was not prepared with persistent shared-run ownership" + ) + actual = deepcopy(dict(deployment)) + actual["conformanceRevision"] = CONFORMANCE_REVISION + core_wheels = { + name: digest + for name, digest in actual["wheels"].items() + if name.startswith("aws_durable_execution_sdk_python-") + } + if len(core_wheels) != 1: + raise ValueError("Deployment does not retain an exact SDK wheel identity") + artifact = "sha256:" + next(iter(core_wheels.values())) + profiles = [ + { + "profile_id": key, + "resource_id": actual["functions"][key], + "native_concurrency": actual["concurrencies"][key], + "capacity": actual["concurrencies"][key], + "artifact_id": artifact, + "configuration_id": configuration_identity(actual, key), + "recovery_policy": "reuse-worker", + } + for key in ("c1", "c2") + ] + scenarios = [] + for required in required_cases("python", "lifecycle-otel", coverage): + case_id = f"{coverage}-{required.scenario}-c{required.native_concurrency}-{required.otel_view or 'baseline'}-{required.export_mode or 'sdk-default'}" + scenarios.append( + { + "case_id": case_id, + "cohort_id": "cohort-1", + "requirement_id": required.requirement_id, + "profile_id": f"c{required.native_concurrency}", + "scenario": required.scenario, + "otel_view": required.otel_view, + "export_mode": required.export_mode, + "roles": list(required.roles), + "slots": required.minimum_native_concurrency, + "exclusive": required.exclusive, + "behavior_budgets": { + name: BEHAVIOR_BUDGETS[name] for name in required.budget_names + }, + } + ) + return { + "schemaVersion": 1, + "runId": actual["run"], + "runtime": "python", + "region": actual["region"], + "coverageProfile": coverage, + "capabilityProfile": "lifecycle-otel", + "scenarioConcurrency": 8, + "profiles": profiles, + "scenarios": scenarios, + "budgets": { + "provisioning": 120, + "lease": 1800, + "placement": 120, + "execution": 240, + "collection": 180, + "release": 60, + "retirement": 180, + }, + "sdk": { + "deployment": actual, + "artifacts_dir": str(directory.resolve() / "source"), + "lease": actual["sharedLease"], + }, + } + + +def run(deployment: Mapping[str, Any], directory: Path) -> int: + from aws_durable_execution_conformance_tests.lmi.app import ( + build_report, + coverage_inventory, + live_results, + load_manifest, + ) + from aws_durable_execution_conformance_tests.reporters import ( + render_console, + write_report, + ) + + directory.mkdir(parents=True, exist_ok=True) + if any(directory.iterdir()): + raise ValueError( + "Shared run output must be empty; preserve prior attempts and failure evidence" + ) + lease = RunLease.from_manifest( + {"deployment": deployment, "lease": deployment["sharedLease"]} + ) + exit_code = 0 + completed = False + try: + for coverage in COVERAGE_PROFILES: + output = directory / coverage + output.mkdir() + plan = make_manifest(deployment, coverage, output) + path = output / "manifest.json" + save(path, plan) + validated = load_manifest(path) + results = asyncio.run( + live_results( + validated, + "lmi_tests.shared_adapter:create_adapter", + audit_dir=output, + ) + ) + report = build_report(validated, results) + save(output / "evidence.json", results) + save(output / "coverage.json", coverage_inventory(validated)) + print(render_console(report)) + for format_name in ("json", "junit"): + write_report(report, format_name, str(output / "report")) + exit_code = max(exit_code, report.exit_code()) + completed = True + finally: + if completed: + try: + lease.release() + save( + directory / "ownership-released.json", + {"runId": deployment["run"], "lease": lease.to_manifest()}, + ) + except OwnershipError as error: + save( + directory / "ownership-retained.json", + {"error": str(error), "lease": lease.to_manifest()}, + ) + exit_code = 1 + else: + save( + directory / "ownership-retained.json", + { + "reason": "Interrupted or incomplete shared run", + "lease": lease.to_manifest(), + }, + ) + return exit_code + + +def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: + """Release only persisted case controls; unknown retirement retains ownership. + + This is the final CI recovery path after the runner process exits. It never + rewrites assertion results or clears quarantine based on service status alone. + """ + receipt = directory / "ownership-released.json" + lease = RunLease.from_manifest( + {"deployment": deployment, "lease": deployment["sharedLease"]} + ) + if receipt.exists() and json.loads(receipt.read_text()) == { + "runId": deployment["run"], + "lease": lease.to_manifest(), + }: + return 0 + record, _ = lease._read() + record = lease._owned(record) + if all(value == "quiescent" for value in record["profiles"].values()): + lease.release() + save(receipt, {"runId": deployment["run"], "lease": lease.to_manifest()}) + return 0 + storage, service = ( + client("s3", deployment["region"]), + client("lambda", deployment["region"]), + ) + errors: list[str] = [] + for coverage in COVERAGE_PROFILES: + plan_path = directory / coverage / "manifest.json" + if not plan_path.exists(): + continue + plan = json.loads(plan_path.read_text()) + if plan.get("runId") != deployment["run"]: + raise OwnershipError("Cleanup manifest belongs to another run") + profiles = {profile["profile_id"]: profile for profile in plan["profiles"]} + for case in plan["scenarios"]: + target = profiles[case["profile_id"]]["resource_id"] + if target not in deployment["functions"].values(): + raise OwnershipError("Cleanup target is outside the owned deployment") + owner = hashlib.sha256( + f"{case['case_id']}:{case.get('cohort_id', 'cohort-1')}".encode() + ).hexdigest()[:20] + prefix = f"runs/{deployment['run']}/cases/{owner}/control/" + case_dir = directory / coverage / "source" / "cases" / owner + try: + objects = [ + item["Key"] + for page in storage.get_paginator("list_objects_v2").paginate( + Bucket=deployment["bucket"], Prefix=prefix + ) + for item in page.get("Contents", []) + ] + if any(not key.startswith(prefix) for key in objects): + raise OwnershipError( + "Control listing escaped the case-owned prefix" + ) + executions: set[str] = set() + for path in (case_dir / "invocations").glob("*.json"): + invocation = json.loads(path.read_text()) + payload = invocation.get("payload", {}) + if ( + payload.get("run") != deployment["run"] + or payload.get("caseId") != case["case_id"] + ): + raise OwnershipError( + "Invocation intent belongs to another case" + ) + if invocation.get("arn"): + executions.add(invocation["arn"]) + else: + marker = payload["marker"] + event_prefix = f"runs/{deployment['run']}/events/{marker}/" + for page in storage.get_paginator("list_objects_v2").paginate( + Bucket=deployment["bucket"], Prefix=event_prefix + ): + for obj in page.get("Contents", []): + with storage.get_object( + Bucket=deployment["bucket"], Key=obj["Key"] + )["Body"] as stream: + event = json.loads(stream.read()) + if ( + event.get("run") != deployment["run"] + or event.get("caseId") != case["case_id"] + ): + raise OwnershipError( + "Recovered invocation evidence belongs to another case" + ) + executions.add(event["execution"]) + if not executions: + errors.append( + f"Unreconciled Invoke: {case['case_id']}/{marker}" + ) + if any( + not arn.startswith(target + "/durable-execution/") + for arn in executions + ): + raise OwnershipError("Refusing to stop a foreign execution") + for key in sorted( + objects, key=lambda value: value != prefix + "release-all" + ): + with storage.get_object(Bucket=deployment["bucket"], Key=key)[ + "Body" + ] as stream: + state = stream.read() + if state not in {b"hold", b"release"}: + raise OwnershipError( + "Unknown control content; refusing to overwrite it" + ) + if state == b"hold": + storage.put_object( + Bucket=deployment["bucket"], Key=key, Body=b"release" + ) + for arn in executions: + if ( + service.get_durable_execution(DurableExecutionArn=arn)["Status"] + == "RUNNING" + ): + try: + service.stop_durable_execution(DurableExecutionArn=arn) + except Exception: + if ( + service.get_durable_execution(DurableExecutionArn=arn)[ + "Status" + ] + == "RUNNING" + ): + raise + except Exception as error: + errors.append(f"{case['case_id']}: {type(error).__name__}: {error}") + for target, status in record["profiles"].items(): + if status != "quiescent": + lease.finish_profile(target, False) + save( + directory / "cleanup.json", + { + "runId": deployment["run"], + "errors": errors, + "status": "OWNERSHIP_RETAINED", + "reason": "Fault release and Stop do not prove task/export quiescence; retained run requires reconciliation", + }, + ) + return 1 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("setup", "plan", "run", "cleanup")) + parser.add_argument( + "--checkout", + type=Path, + default=Path( + os.environ.get("LMI_CONFORMANCE_CHECKOUT", str(ROOT / ".conformance")) + ), + ) + parser.add_argument("--manifest", type=Path, default=ARTIFACTS / "manifest.json") + parser.add_argument("--output", type=Path, default=ARTIFACTS / "shared") + args = parser.parse_args() + if args.command == "setup": + setup(args.checkout) + return + verify_install(args.checkout) + deployment = json.loads(args.manifest.read_text()) + expected_run = os.environ.get("LMI_SHARED_RUN_ID") + if expected_run and deployment.get("run") != expected_run: + raise OwnershipError("Deployment manifest belongs to a different CI run") + if args.command == "cleanup": + raise SystemExit(cleanup(deployment, args.output)) + if args.command == "plan": + for coverage in COVERAGE_PROFILES: + output = args.output / coverage + save(output / "manifest.json", make_manifest(deployment, coverage, output)) + return + raise SystemExit(run(deployment, args.output)) + + +if __name__ == "__main__": + main() diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py new file mode 100644 index 000000000..6177ec6f6 --- /dev/null +++ b/lmi_tests/shared_adapter.py @@ -0,0 +1,229 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""SDK-owned bridge to the pinned shared LMI scenario coordinator.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Mapping +from concurrent.futures import TimeoutError as FutureTimeout +from copy import deepcopy +import hashlib +from pathlib import Path +from typing import Any, Protocol + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + ScenarioContext, +) +from lmi_tests.cloud import Cloud +from lmi_tests.evidence import CollectionError, ProvisioningError +from lmi_tests.shared_scenarios import ScenarioDriver +from lmi_tests.shared_contract import configuration_identity + + +class RunOwnership(Protocol): + def assert_owned(self, resource_id: str) -> None: ... + def finish_profile(self, resource_id: str, quiescent: bool) -> None: ... + + +class EvidenceBuilder(Protocol): + def __call__( + self, context: ScenarioContext, cloud: Cloud, *, phase: str + ) -> dict[str, Any]: ... + + +class _Case: + def __init__(self, driver: ScenarioDriver) -> None: + self.driver = driver + self.worker: asyncio.Task[Any] | None = None + + async def call(self, operation: Callable[[], Any]) -> Any: + if self.worker is not None and not self.worker.done(): + raise CollectionError( + "A previous case worker still owns blocking service calls" + ) + self.worker = asyncio.create_task(asyncio.to_thread(operation)) + try: + # Keep the worker task alive on outer cancellation so retirement can + # join the actual thread before issuing cleanup writes. + return await asyncio.shield(self.worker) + except asyncio.CancelledError: + self.driver.cloud.cancel() + raise + + async def prepare_retirement(self) -> None: + if self.worker is not None: + try: + await asyncio.shield(self.worker) + except Exception: + # The coordinator retains the original phase error; this only + # confirms its service-call worker has really stopped. + pass + finally: + if self.worker.done(): + self.worker = None + self.driver.cloud.begin_retirement() + + +class SharedAdapter: + """Each shared case has one Cloud, worker, control namespace and evidence sink.""" + + def __init__( + self, + manifest: Mapping[str, Any], + *, + ownership: RunOwnership, + evidence_builder: EvidenceBuilder, + cloud_factory: Callable[..., Cloud] = Cloud, + ) -> None: + self.manifest = deepcopy(dict(manifest)) + self.sdk = self.manifest["sdk"] + self.deployment = self.sdk["deployment"] + self.artifacts_dir = Path(self.sdk["artifacts_dir"]) + self.ownership = ownership + self.evidence_builder = evidence_builder + self.cloud_factory = cloud_factory + self.cases: dict[str, _Case] = {} + + async def prepare(self, profile: Profile) -> None: + def verify() -> None: + self.ownership.assert_owned(profile.resource_id) + if profile.resource_id not in self.deployment["functions"].values(): + raise ProvisioningError( + "Shared profile is not part of the owned SDK deployment" + ) + key = next( + key + for key, arn in self.deployment["functions"].items() + if arn == profile.resource_id + ) + if self.deployment["concurrencies"][key] != profile.native_concurrency: + raise ProvisioningError( + "Native concurrency differs from the shared profile" + ) + if profile.configuration_id != configuration_identity(self.deployment, key): + raise ProvisioningError( + "Shared configuration identity differs from the deployed fixture" + ) + cloud = self.cloud_factory( + self.deployment, key, artifacts_dir=self.artifacts_dir / "configuration" + ) + cloud.verify() + + await asyncio.to_thread(verify) + + def _case(self, context: ScenarioContext) -> _Case: + try: + return self.cases[context.scenario.case_id] + except KeyError as error: + raise CollectionError( + "Case was not initialized by the shared adapter" + ) from error + + async def start(self, context: ScenarioContext) -> None: + if context.scenario.case_id in self.cases: + raise CollectionError("A later scenario cannot replace an admitted case") + await asyncio.to_thread( + self.ownership.assert_owned, context.profile.resource_id + ) + key = next( + ( + key + for key, arn in self.deployment["functions"].items() + if arn == context.profile.resource_id + ), + None, + ) + if key is None: + raise ProvisioningError("Case target is outside the owned deployment") + owner = hashlib.sha256( + f"{context.scenario.case_id}:{context.scenario.cohort_id}".encode() + ).hexdigest()[:20] + cloud = await asyncio.to_thread( + self.cloud_factory, + self.deployment, + key, + artifacts_dir=self.artifacts_dir / "cases" / owner, + case_context=context, + ) + loop = asyncio.get_running_loop() + + def capacity(operation: Callable[[], Any]) -> None: + cloud.check_active() + pending = asyncio.run_coroutine_threadsafe(operation(), loop) + try: + while True: + try: + pending.result(timeout=0.2) + return + except FutureTimeout: + cloud.check_active() + finally: + if not pending.done(): + pending.cancel() + + driver = ScenarioDriver( + context, + cloud, + release_capacity=lambda: capacity(context.release_capacity), + reserve_capacity=lambda: capacity(context.reserve_capacity), + ) + case = self.cases[context.scenario.case_id] = _Case(driver) + await case.call(driver.start) + + async def place(self, context: ScenarioContext) -> None: + case = self._case(context) + await case.call(case.driver.place) + + async def execute(self, context: ScenarioContext) -> None: + case = self._case(context) + await case.call(case.driver.execute) + + async def collect(self, context: ScenarioContext) -> dict[str, Any]: + case = self._case(context) + + def collect() -> dict[str, Any]: + case.driver.collect() + return self.evidence_builder( + context, case.driver.cloud, phase="observations" + ) + + return await case.call(collect) + + async def release(self, context: ScenarioContext) -> None: + case = self.cases.get(context.scenario.case_id) + if case is None: + return + await case.prepare_retirement() + await case.call(case.driver.cloud.release_all) + + async def retire(self, context: ScenarioContext) -> bool: + case = self.cases.get(context.scenario.case_id) + if case is None: + return True + await case.prepare_retirement() + + def retire() -> bool: + complete = case.driver.retire() + final = self.evidence_builder(context, case.driver.cloud, phase="final") + # The coordinator freezes observation evidence before fault release. + # Only actual retirement facts may be appended by this phase. + context.evidence["retirement"] = final.get("retirement", {}) + return complete + + return await case.call(retire) + + async def finish(self, profile: Profile, *, quiescent: bool) -> None: + await asyncio.to_thread( + self.ownership.finish_profile, profile.resource_id, quiescent + ) + + +def create_adapter(manifest: Mapping[str, Any]) -> SharedAdapter: + """Factory used by the shared CLI; persistent ownership is mandatory.""" + from lmi_tests.shared_evidence import build_evidence + from lmi_tests.shared_lock import RunLease + + ownership = RunLease.from_manifest(manifest["sdk"]) + return SharedAdapter(manifest, ownership=ownership, evidence_builder=build_evidence) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py new file mode 100644 index 000000000..d8d273e7e --- /dev/null +++ b/lmi_tests/shared_contract.py @@ -0,0 +1,40 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Pinned runner and verified deployment identities for the SDK adapter.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any, Mapping + +CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" +# Updated only with the matching reviewed shared evidence contract. +CONFORMANCE_REVISION = "6f899679c7eb6ede836c737acb9816e6fd7072fd" + + +def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: + fields = ( + "runtime", + "role", + "bucket", + "commit", + "run", + "provider", + "codeSha256", + "invocationTimeout", + "executionTimeout", + ) + configuration = {name: deployment[name] for name in fields} + configuration.update( + function=deployment["functions"][key], + nativeConcurrency=deployment["concurrencies"][key], + architecture="arm64", + memorySize=2048, + handler="lmi_tests.fixture.handler", + scaling={"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1}, + ) + return ( + "sha256:" + + hashlib.sha256(json.dumps(configuration, sort_keys=True).encode()).hexdigest() + ) diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py new file mode 100644 index 000000000..f7f3034d2 --- /dev/null +++ b/lmi_tests/shared_evidence.py @@ -0,0 +1,687 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Normalize observed SDK/service evidence for the pinned shared validator. + +This module runs only in the driver. It never fills in a missing SDK boundary or +uses a fixture's expected answer as an observed result. Raw source objects remain +in the case artifact directory and S3 ledger. +""" + +from __future__ import annotations + +from collections import Counter, defaultdict +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import datetime +import json +from typing import Any + +from aws_durable_execution_conformance_tests.lmi.coordinator import ScenarioContext +from lmi_tests.cloud import Cloud + + +TERMINAL = {"SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"} +_ACTIVE_FAULTS = {"deadline-recovery", "timeout-retry"} +_OPERATION_TYPES = { + "Step": "STEP", + "Wait": "WAIT", + "Callback": "CALLBACK", + "Context": "CONTEXT", + "ChainedInvoke": "CHAINED_INVOKE", +} + + +def _milliseconds(value: Any) -> float: + if isinstance(value, datetime): + return value.timestamp() * 1000 + if isinstance(value, str): + return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000 + if type(value) in (int, float): + return float(value) * 1000 + raise ValueError("Service evidence is missing its actual timestamp") + + +def _decoded(value: Any) -> Any: + if isinstance(value, str): + return json.loads(value) + return value + + +def _error(value: Any) -> dict[str, Any] | None: + if not isinstance(value, Mapping): + return None + if "Payload" in value: + value = value["Payload"] + if not isinstance(value, Mapping): + return None + return { + "type": value.get("ErrorType", value.get("type")), + "message": value.get("ErrorMessage", value.get("message")), + } + + +def _resources(value: Mapping[str, Any]) -> dict[str, Any]: + result: dict[str, Any] = {} + if "threads" in value: + result["threads"] = [ + {**thread, "id": str(thread["id"])} for thread in value["threads"] + ] + if "fds" in value: + result["fds"] = value["fds"] + if "rssBytes" in value or "rss" in value: + result["rssBytes"] = value.get("rssBytes", value.get("rss")) + return result + + +@dataclass(frozen=True) +class CaseSources: + context: ScenarioContext + cloud: Cloud + raw: tuple[Mapping[str, Any], ...] + + @classmethod + def capture(cls, context: ScenarioContext, cloud: Cloud) -> CaseSources: + return cls(context, cloud, tuple(cloud.events.values())) + + def observations(self) -> tuple[list[dict[str, Any]], list[str]]: + events: list[dict[str, Any]] = [] + errors: list[str] = [] + by_request: dict[str, list[int]] = defaultdict(list) + for raw in self.raw: + request = raw.get("request") + if ( + raw.get("run") != self.context.run_id + or raw.get("commit") != self.cloud.manifest["commit"] + or raw.get("caseId") != self.context.scenario.case_id + or raw.get("cohortId") != self.context.scenario.cohort_id + ): + errors.append( + "Raw evidence does not belong to the admitted run/artifact/case" + ) + continue + if not isinstance(request, str) or type(raw.get("sequence")) is not int: + errors.append( + "Raw evidence has no native request identity or source sequence" + ) + continue + by_request[request].append(raw["sequence"]) + observation = raw.get("observation") + if raw.get("phase") != "CONFORMANCE" or not isinstance( + observation, Mapping + ): + continue + event = { + "requestId": request, + "ownerRequestId": request, + "executionArn": raw["execution"], + **observation, + "sourceSequence": raw["sequence"], + "observedAtMs": raw["time"] * 1000, + } + if "resources" in event: + event["resources"] = _resources(event["resources"]) + if event.get("kind") == "wrapper_raise" and event.get("outcome") == "THREW": + event["sourceOutcome"] = "THREW" + event["outcome"] = ( + "FAILED" # Actual exception, not an inferred platform timeout. + ) + if event.get("kind") in { + "observation_error", + "control_error", + "export_publication_error", + }: + errors.append( + str(event.get("message", event.get("error", event["kind"]))) + ) + events.append(event) + for request, sequences in by_request.items(): + if len(set(sequences)) != len(sequences) or sorted(sequences) != list( + range(1, max(sequences) + 1) + ): + errors.append(f"Raw source publication is incomplete for {request}") + return events, errors + + def operation_snapshots( + self, events: list[dict[str, Any]] + ) -> dict[str, dict[str, Mapping[str, Any]]]: + snapshots: dict[str, dict[str, Mapping[str, Any]]] = defaultdict(dict) + for event in events: + records = list(event.get("operations", [])) + if event.get("operationId") and event.get("operationType"): + records.append(event) + for record in records: + snapshots[event["executionArn"]][record["operationId"]] = record + return snapshots + + def histories( + self, events: list[dict[str, Any]] + ) -> tuple[list[dict[str, Any]], dict[str, Any], list[str]]: + snapshots = self.operation_snapshots(events) + histories: list[dict[str, Any]] = [] + expected: dict[str, Any] = {} + errors: list[str] = [] + for item in self.cloud.invocations: + arn = item.get("arn") + if ( + not arn + or arn not in self.cloud.raw_histories + or arn not in self.cloud.execution_results + ): + errors.append( + "An admitted execution is missing actual service history or state" + ) + continue + operations: dict[str, dict[str, Any]] = {} + for raw in self.cloud.raw_histories[arn]: + event_type = raw.get("EventType", "") + kind = next( + ( + value + for prefix, value in _OPERATION_TYPES.items() + if event_type.startswith(prefix) + ), + None, + ) + identity = raw.get("Id") + if kind is None or not isinstance(identity, str): + continue + info = snapshots.get(arn, {}).get(identity, {}) + operation = operations.setdefault( + identity, + { + "id": identity, + "type": kind, + "name": raw.get("Name", info.get("name")), + "parentId": raw.get("ParentId", info.get("parentId")), + "status": "STARTED", + }, + ) + if "Name" in raw: + operation["name"] = raw["Name"] + if "ParentId" in raw: + operation["parentId"] = raw["ParentId"] + details = raw.get(event_type + "Details", {}) + if event_type.endswith("Succeeded"): + operation["status"] = "SUCCEEDED" + result = details.get("Result", info.get("serializedResult")) + if result is not None: + operation["result"] = _decoded(result) + elif event_type.endswith("Failed"): + operation["status"] = "FAILED" + operation["error"] = _error(details.get("Error")) or info.get( + "error" + ) + elif event_type.endswith("TimedOut"): + operation["status"] = "TIMED_OUT" + result = self.cloud.execution_results[arn] + started = result.get("StartTimestamp") + if started is None: + started = next( + ( + event.get("EventTimestamp") + for event in self.cloud.raw_histories[arn] + if event.get("EventType") == "ExecutionStarted" + ), + None, + ) + history: dict[str, Any] = { + "executionArn": arn, + "ownerCaseId": self.context.scenario.case_id, + "status": result.get("Status"), + "operations": list(operations.values()), + } + if started is not None: + history["startedAtMs"] = _milliseconds(started) + if "Result" in result: + history["result"] = _decoded(result["Result"]) + if "Error" in result: + history["error"] = _error(result["Error"]) + histories.append(history) + fixture = item.get("payload", {}).get("scenario") + if fixture in {"failure", "failure-inflight"}: + expected[arn] = { + "status": "FAILED", + "error": { + "type": "ValueError", + "message": "expected:" + item["marker"], + }, + } + elif ( + fixture == "deadline" + and self.context.scenario.scenario in _ACTIVE_FAULTS + ): + # This fixture asserts native deadline/retry behavior. Its logical + # execution may still be active or may finish after normal release. + expected[arn] = { + "allowedStatuses": [ + "RUNNING", + "SUCCEEDED", + "FAILED", + "TIMED_OUT", + "STOPPED", + ] + } + else: + expected[arn] = {"status": "SUCCEEDED", "result": item["marker"]} + return histories, expected, errors + + def attempts( + self, events: list[dict[str, Any]] + ) -> tuple[list[dict[str, Any]], dict[str, str], list[str]]: + attempts: list[dict[str, Any]] = [] + selected: dict[str, str] = {} + errors: list[str] = [] + journal = {item.get("requestId"): item for item in self.context.attempts} + raw_by_request: dict[str, Mapping[str, Any]] = {} + for raw in self.raw: + if raw.get("phase") == "WRAPPER_ENTER": + raw_by_request[raw["request"]] = raw + for request, raw in raw_by_request.items(): + item = next( + ( + item + for item in self.cloud.invocations + if item["marker"] == raw["marker"] + ), + None, + ) + if ( + item is None + or request not in journal + or request not in self.cloud.request_roles + ): + errors.append( + f"Native request {request} was not bound to an original launch" + ) + continue + owned = [event for event in events if event.get("requestId") == request] + boundaries = [ + event + for event in owned + if event.get("kind") in {"wrapper_return", "wrapper_raise"} + ] + deadline = [ + event for event in owned if event.get("kind") == "runtime_deadline" + ] + timeout = None + for service_event in self.cloud.raw_histories.get(raw["execution"], []): + details = service_event.get("InvocationCompletedDetails", {}) + message = json.dumps(details.get("Error", {})).lower().replace(" ", "") + if details.get("RequestId") == request and any( + word in message for word in ("timeout", "timedout") + ): + timeout = { + "kind": "platform_timeout", + "requestId": request, + "ownerRequestId": request, + "executionArn": raw["execution"], + "atMs": _milliseconds(service_event.get("EventTimestamp")), + "source": "platform", + "sourceEventId": service_event.get("EventId"), + } + if timeout is not None: + events.append(timeout) + role = self.cloud.request_roles[request] + attempt: dict[str, Any] = { + "requestId": request, + "executionArn": raw["execution"], + "environmentId": raw["environment"], + "processId": raw["process"], + "workerId": raw["process"], + "pid": raw.get("pid"), + "role": role, + "attempt": journal[request]["attempt"], + "marker": raw["marker"], + "fixture": item.get("payload", {}).get("scenario"), + "startedAtMs": min( + [raw["time"] * 1000, *(event["atMs"] for event in owned)] + ), + } + if boundaries: + attempt["outcome"] = boundaries[-1]["outcome"] + attempt["endedAtMs"] = boundaries[-1]["atMs"] + if timeout is not None: + attempt.update(outcome="TIMED_OUT", endedAtMs=timeout["atMs"]) + if deadline: + attempt["deadlineAtMs"] = deadline[0]["deadlineAtMs"] + if "outcome" not in attempt: + errors.append( + f"Native request {request} has no observed completion boundary" + ) + if journal[request]["attempt"] == 1: + selected[role] = request + attempts.append(attempt) + if ( + self.context.scenario.scenario == "timeout-retry" + and "victim" in selected + and "retry" in selected + ): + original = next( + item for item in attempts if item["requestId"] == selected["victim"] + ) + retry = next( + item for item in attempts if item["requestId"] == selected["retry"] + ) + if ( + original.get("outcome") == "TIMED_OUT" + and retry["executionArn"] == original["executionArn"] + ): + events.append( + { + "kind": "service_retry", + "requestId": retry["requestId"], + "ownerRequestId": retry["requestId"], + "executionArn": retry["executionArn"], + "atMs": retry["startedAtMs"], + "source": "platform", + "previousRequestId": original["requestId"], + "sourceEvidence": "single-original-Invoke-and-native-service-continuation", + } + ) + return attempts, selected, errors + + +def _causal_order(events: list[dict[str, Any]]) -> None: + requests = {event["requestId"] for event in events} + for request in requests: + owned = sorted( + (event for event in events if event["requestId"] == request), + key=lambda event: ( + event["atMs"], + event.get("captureSequence", event.get("sourceSequence", -1)), + ), + ) + for sequence, event in enumerate(owned, 1): + event["sequence"] = sequence + + +def _quiescent(request: str, events: list[dict[str, Any]], *, telemetry: bool) -> bool: + local = [event for event in events if event.get("requestId") == request] + if not any( + event.get("kind") in {"wrapper_return", "wrapper_raise"} for event in local + ): + return False + if not any( + event.get("kind") == "task_exit" and event.get("taskId") == "root" + for event in local + ): + return False + for start, end, key in ( + ("task_start", "task_exit", "taskId"), + ("io_start", "io_exit", "taskId"), + ("checkpoint_call", "checkpoint_exit", "callId"), + ("poll_call", "poll_exit", "callId"), + ("export_started", "export_completed", "exportId"), + ("flush_started", "flush_completed", "flushId"), + ): + begun = Counter(event.get(key) for event in local if event.get("kind") == start) + ended = Counter(event.get(key) for event in local if event.get("kind") == end) + if begun - ended: + return False + if telemetry and not any( + event.get("kind") == "export_quiescence" + and event.get("pendingExportIds") == [] + and event.get("pendingFlushIds") == [] + and event.get("publicationCount") == 0 + for event in local + ): + return False + return not any( + event.get("kind") in {"observation_error", "control_error"} for event in local + ) + + +def build_evidence( + context: ScenarioContext, cloud: Cloud, *, phase: str = "observations" +) -> dict[str, Any]: + sources = CaseSources.capture(context, cloud) + events, errors = sources.observations() + histories, expected, history_errors = sources.histories(events) + attempts, selected, attempt_errors = sources.attempts(events) + errors.extend(history_errors + attempt_errors) + _causal_order(events) + ended = (cloud.observation_ended_at or 0) * 1000 + complete = bool(attempts) and all( + _quiescent( + item["requestId"], events, telemetry=context.scenario.otel_view is not None + ) + for item in attempts + ) + if not complete: + errors.append( + "Actual wrapper/task/export publication has not established a complete observation window" + ) + latest = max((event["atMs"] for event in events), default=0) + if ended < latest: + errors.append("Collection predates a retained observation") + watermarks = ( + {name: ended for name in ("lifecycle", "sdkRequests", "history")} + if not errors + else {} + ) + if context.scenario.export_mode and not errors: + watermarks["exportLifecycle"] = ended + artifact = { + "sdk": context.profile.artifact_id, + "conformance": cloud.manifest.get("conformanceRevision", "unverified"), + } + if context.scenario.otel_view: + artifact["plugin"] = next( + ( + f"sha256:{digest}" + for name, digest in cloud.manifest.get("wheels", {}).items() + if "_otel-" in name + ), + "", + ) + profile = { + "functionArn": context.profile.resource_id, + "configurationDigest": context.profile.configuration_id, + "nativeConcurrency": context.profile.native_concurrency, + "timeoutMs": cloud.manifest["invocationTimeout"] * 1000, + } + if context.profile.recovery_policy: + profile["recoveryPolicy"] = context.profile.recovery_policy + evidence: dict[str, Any] = { + "schemaVersion": 1, + "runId": context.run_id, + "caseId": context.scenario.case_id, + "cohortId": context.scenario.cohort_id, + "coverageProfile": context.scenario.coverage_profile, + "scenario": context.scenario.scenario, + "requirementId": context.scenario.requirement_id, + "runtime": "python", + "artifact": artifact, + "profile": profile, + "attempts": attempts, + "selectedRequests": selected, + "events": events, + "histories": histories, + "expectedHistories": expected, + "observationEndedAtMs": ended, + "sdkRequests": [], + "collection": { + "watermarks": watermarks, + "evidenceUris": sorted(cloud.source_uris), + }, + "collectionErrors": errors, + } + if context.scenario.otel_view: + evidence["otelView"] = context.scenario.otel_view + if context.scenario.export_mode: + evidence["exportMode"] = context.scenario.export_mode + evidence["exportLifecycle"] = { + "budgets": dict(context.scenario.behavior_budgets) + } + else: + evidence["regression"] = {"budgets": dict(context.scenario.behavior_budgets)} + victim = next( + (item for item in cloud.invocations if item.get("role") == "victim"), None + ) + if victim: + fixture = victim.get("payload", {}).get("scenario") + suffix = ( + "-checkpoint" + if fixture == "checkpoint" + else "-cleanup" + if fixture == "suspend-cleanup" + else "-loser" + ) + if fixture in { + "checkpoint", + "suspend-cleanup", + "parallel", + "map", + "nested", + "return-inflight", + "failure-inflight", + "late-operation", + }: + evidence["regression"]["gateId"] = victim["marker"] + suffix + if context.scenario.scenario in {"warm-lifecycle", "otel-warm-lifecycle"}: + evidence["regression"]["warmSequence"] = [ + {"executionArn": item["arn"], "fixture": item["payload"]["scenario"]} + for item in cloud.invocations + ] + for event in events: + if event.get("kind") in {"checkpoint_call", "poll_call"}: + evidence["sdkRequests"].append( + { + **event, + "method": "CheckpointDurableExecution" + if event["kind"] == "checkpoint_call" + else "GetDurableExecutionState", + "outcome": "DISPATCHED", + } + ) + created = [ + { + "kind": "invocation", + "id": item["requestId"], + "ownerCaseId": context.scenario.case_id, + } + for item in attempts + ] + created.extend( + {"kind": "execution", "id": arn, "ownerCaseId": context.scenario.case_id} + for arn in dict.fromkeys(item["executionArn"] for item in attempts) + ) + for kind in ("export", "flush"): + created.extend( + { + "kind": kind, + "id": event[f"{kind}Id"], + "ownerCaseId": context.scenario.case_id, + } + for event in events + if event.get("kind") == f"{kind}_started" + ) + evidence["resourcesCreated"] = created + faults = [] + for gate in dict.fromkeys( + record["gate"] for record in cloud.control_records if record["action"] == "hold" + ): + fault: dict[str, Any] = {"id": gate, "ownerCaseId": context.scenario.case_id} + receipts = [ + event["atMs"] + for event in events + if event.get("gateId") == gate + and event.get("kind") in {"fault_released", "export_released"} + and event.get("mode") == "normal" + ] + releases = [ + record + for record in cloud.control_records + if record["gate"] == gate + and record["action"] == "release" + and record.get("state", "ACKNOWLEDGED") == "ACKNOWLEDGED" + ] + if receipts: + fault["releasedAtMs"] = min(receipts) + if releases: + release = releases[0] + requested = release.get("started", release["time"]) * 1000 + if not receipts or requested <= min(receipts): + fault.update( + releaseRequestedAtMs=requested, + releaseAcknowledgedAtMs=release["time"] * 1000, + ) + fault.setdefault("releasedAtMs", release["time"] * 1000) + faults.append(fault) + evidence["faultsCreated"] = faults + if context.scenario.otel_view: + from lmi_tests.shared_telemetry import collect_telemetry + + headers = { + request: item["trace_header"] + for item in cloud.invocations + for request in item.get("native_requests", []) + if item.get("trace_header") + } + evidence["telemetry"] = collect_telemetry( + list(sources.raw), + attempts, + view=context.scenario.otel_view, + expected_sampled=context.scenario.scenario != "runtime-carrier-unsampled", + caller_headers=headers, + observed_until_ms=ended, + export_mode=context.scenario.export_mode or "sync", + ) + if phase == "final": + retired = (cloud.retired_at or 0) * 1000 + settled = [] + for resource in created: + item = { + **resource, + "settledAtMs": retired, + "evidenceUri": (cloud.artifacts_dir / "events.json").resolve().as_uri(), + } + if resource["kind"] == "invocation": + item["outcome"] = ( + "QUIESCENT" + if _quiescent( + resource["id"], + events, + telemetry=context.scenario.otel_view is not None, + ) + else "UNKNOWN" + ) + elif resource["kind"] == "execution": + item["outcome"] = cloud.execution_results.get(resource["id"], {}).get( + "Status", "UNKNOWN" + ) + else: + completion = [ + event + for event in events + if event.get("kind") == resource["kind"] + "_completed" + and event.get(resource["kind"] + "Id") == resource["id"] + ] + item["outcome"] = ( + completion[0].get("outcome") if len(completion) == 1 else "UNKNOWN" + ) + settled.append(item) + evidence["retirement"] = { + "retiredAtMs": retired, + "resources": settled, + "faults": [ + { + "id": fault["id"], + "ownerCaseId": context.scenario.case_id, + "releasedAtMs": fault.get( + "releaseAcknowledgedAtMs", fault.get("releasedAtMs", 0) + ), + "mode": "emergency" + if any( + event.get("gateId") == fault["id"] + and event.get("kind") in {"fault_released", "export_released"} + and event.get("mode") == "emergency" + for event in events + ) + else "normal", + } + for fault in faults + ], + } + cloud._save(f"shared-{phase}-evidence.json", evidence) + return evidence diff --git a/lmi_tests/shared_exporter.py b/lmi_tests/shared_exporter.py new file mode 100644 index 000000000..f0740e222 --- /dev/null +++ b/lmi_tests/shared_exporter.py @@ -0,0 +1,489 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real processor/exporter fixtures with immutable invocation ownership. + +Only diagnostic publication is deferred in synchronous mode. Export itself +remains on SimpleSpanProcessor's caller. The asynchronous mode queues the same +exporter work and bounds the SDK's flush wait independently of its completion. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +import queue +import threading +import time +from typing import Any, Protocol +import uuid + +from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor +from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult + + +class Trace(Protocol): + payload: Mapping[str, Any] + identity: Mapping[str, Any] + + def emit(self, phase: str, **fields: Any) -> Any: ... + + def observe(self, kind: str, **fields: Any) -> Any: ... + + def capture_observation(self, kind: str, **fields: Any) -> dict[str, Any]: ... + + def publish_observation(self, observation: Mapping[str, Any]) -> Any: ... + + def gate(self, name: str, **fields: Any) -> Any: ... + + def gate_release(self, name: str, **fields: Any) -> Any: ... + + +@dataclass(eq=False) +class Owner: + trace: Trace + tracker: Tracker + export_behavior: str + gate: str | None + shared: bool + case_id: str + request_id: str + exports: list[Export] = field(default_factory=list) + publications: int = 0 + closed: bool = False + quiescent: bool = False + publication_errors: list[str] = field(default_factory=list) + + +CURRENT_OWNER: ContextVar[Owner | None] = ContextVar("lmi_export_owner", default=None) + + +@contextmanager +def owned(owner: Owner): + token = CURRENT_OWNER.set(owner) + try: + yield + finally: + CURRENT_OWNER.reset(token) + + +@dataclass(eq=False) +class Export: + owner: Owner + identity: str + spans: tuple[dict[str, Any], ...] + result: SpanExportResult | None = None + + +@dataclass(eq=False) +class Flush: + owner: Owner + identity: str + exports: tuple[Export, ...] + result: SpanExportResult | None = None + + +def snapshot(span: ReadableSpan) -> dict[str, Any]: + context = span.context + if context is None: + raise ValueError("Exported span has no span context") + return { + "name": span.name, + "trace_id": format(context.trace_id, "032x"), + "span_id": format(context.span_id, "016x"), + "parent_id": format(span.parent.span_id, "016x") if span.parent else None, + "sampled": context.trace_flags.sampled, + "start_ns": span.start_time, + "end_ns": span.end_time, + "status": span.status.status_code.name, + "kind": span.kind.name, + "attributes": dict(span.attributes or {}), + "service_name": span.resource.attributes.get("service.name"), + "links": [ + { + "trace_id": format(link.context.trace_id, "032x"), + "span_id": format(link.context.span_id, "016x"), + } + for link in span.links + ], + "exception_event_count": ( + None + if span.dropped_events + else sum(event.name == "exception" for event in span.events) + ), + } + + +# Test-processor configuration, separate from the admitted acceptance budget. +DEFAULT_FLUSH_TIMEOUT_MS = 5000 + + +class Tracker: + """Keep SDK work and evidence publication owned until both actually settle.""" + + def __init__( + self, + provider_id: str, + exporter_id: str, + view: str, + export_mode: str, + flush_timeout_ms: float = DEFAULT_FLUSH_TIMEOUT_MS, + ) -> None: + self.provider_id, self.exporter_id, self.view = provider_id, exporter_id, view + self.export_mode = export_mode + self.flush_timeout_ms = flush_timeout_ms + self.lock = threading.RLock() + self.changed = threading.Condition(self.lock) + self.exports: dict[str, Export] = {} + self.flushes: dict[str, Flush] = {} + self.owners: dict[str, Owner] = {} + self.publications: queue.Queue[tuple[Owner, str, dict[str, Any]] | None] = ( + queue.Queue() + ) + self.publisher = threading.Thread( + target=self._publish, name="lmi-otel-evidence", daemon=True + ) + self.publisher.start() + + def owner(self, trace: Trace) -> Owner: + payload = trace.payload + result = Owner( + trace=trace, + tracker=self, + export_behavior=str(payload.get("otel_export", "success")), + gate=payload.get("otel_export_gate"), + shared=bool(payload.get("coverageProfile")), + case_id=str(payload.get("caseId", trace.identity.get("marker", ""))), + request_id=str(trace.identity["request"]), + ) + with self.lock: + self.owners[result.request_id] = result + return result + + def observe(self, owner: Owner, kind: str, **fields: Any) -> None: + if not owner.shared: + return + with self.lock: + captured = time.time_ns() + data = { + "atMs": captured / 1_000_000, + "capturedAtNs": captured, + "otelView": self.view, + "exportMode": self.export_mode, + "providerId": self.provider_id, + "exporterId": self.exporter_id, + **fields, + } + observation = owner.trace.capture_observation(kind, **data) + owner.publications += 1 + owner.quiescent = False + self.owners[owner.request_id] = owner + self.publications.put((owner, kind, observation)) + + def _publish(self) -> None: + while (item := self.publications.get()) is not None: + owner, kind, data = item + try: + owner.trace.publish_observation(data) + except Exception as error: + # Unknown publication remains owned, never a fabricated success. + with self.lock: + owner.publication_errors.append(repr(error)) + else: + with self.lock: + owner.publications -= 1 + if ( + kind == "export_quiescence" + and owner.publications == 0 + and owner.quiescent + ): + self.owners.pop(owner.request_id, None) + else: + self._quiescence(owner) + finally: + self.publications.task_done() + del item, owner, data + self.publications.task_done() + + def pending(self, owner: Owner) -> list[str]: + return [ + identity + for identity, work in self.exports.items() + if work.owner.case_id == owner.case_id and work.result is None + ] + + def active(self, case_id: str) -> dict[str, Any]: + with self.lock: + owners = [ + owner for owner in self.owners.values() if owner.case_id == case_id + ] + return { + "pendingExportIds": [ + key + for key, work in self.exports.items() + if work.owner.case_id == case_id + ], + "pendingFlushIds": [ + key + for key, work in self.flushes.items() + if work.owner.case_id == case_id + ], + "publicationCount": sum(owner.publications for owner in owners), + "publicationErrors": [ + error for owner in owners for error in owner.publication_errors + ], + } + + def close_invocation(self, owner: Owner) -> None: + with self.lock: + owner.closed = True + owner.exports.clear() + self._quiescence(owner) + + def _quiescence(self, owner: Owner) -> None: + if ( + not owner.closed + or owner.publications + or owner.quiescent + or any(work.owner is owner for work in self.exports.values()) + or any(work.owner is owner for work in self.flushes.values()) + ): + return + owner.quiescent = True + if owner.shared: + # This signal follows publication of every earlier observation. + # Do not count the signal itself as outstanding prior publication. + owner.publications += 1 + self.publications.put( + ( + owner, + "export_quiescence", + owner.trace.capture_observation( + "export_quiescence", + pendingExportIds=[], + pendingFlushIds=[], + publicationCount=0, + ), + ) + ) + else: + self.owners.pop(owner.request_id, None) + + def accept(self, spans: Sequence[ReadableSpan]) -> Export: + owner = CURRENT_OWNER.get() + if owner is None or owner.tracker is not self: + raise RuntimeError( + "Export lost its original public-plugin invocation owner" + ) + work = Export(owner, uuid.uuid4().hex, tuple(snapshot(span) for span in spans)) + with self.lock: + self.exports[work.identity] = work + owner.exports.append(work) + self.observe(owner, "export_started", exportId=work.identity) + for span in work.spans: + self.observe( + owner, + "otel_export_started", + exportId=work.identity, + traceId=span["trace_id"], + spanId=span["span_id"], + ) + return work + + def complete( + self, work: Export, result: SpanExportResult, error: str | None + ) -> None: + with self.changed: + work.result = result + self.observe( + work.owner, + "export_completed", + exportId=work.identity, + outcome="succeeded" if result is SpanExportResult.SUCCESS else "failed", + **( + {"fixture": "bounded-export-error"} + if work.owner.export_behavior == "failure" + else {} + ), + **( + {"error": error or "SpanExportResult.FAILURE"} + if result is not SpanExportResult.SUCCESS + else {} + ), + ) + self.exports.pop(work.identity) + for flush in tuple(self.flushes.values()): + self._complete_flush(flush) + self.changed.notify_all() + self._quiescence(work.owner) + + def _complete_flush(self, flush: Flush) -> None: + if flush.result is not None or any( + work.result is None for work in flush.exports + ): + return + flush.result = ( + SpanExportResult.SUCCESS + if all(work.result is SpanExportResult.SUCCESS for work in flush.exports) + else SpanExportResult.FAILURE + ) + self.observe( + flush.owner, + "flush_completed", + flushId=flush.identity, + outcome="succeeded" + if flush.result is SpanExportResult.SUCCESS + else "failed", + pendingExportIds=self.pending(flush.owner), + ) + self.flushes.pop(flush.identity) + self.changed.notify_all() + + def force_flush(self, timeout_millis: float) -> bool: + owner = CURRENT_OWNER.get() + if owner is None or owner.tracker is not self: + raise RuntimeError("Flush lost its public-plugin invocation owner") + allowed = min(float(timeout_millis), self.flush_timeout_ms) + if allowed <= 0: + raise ValueError( + "Flush wait must have a positive provider and scenario budget" + ) + deadline = time.monotonic() + allowed / 1000 + with self.changed: + flush = Flush(owner, uuid.uuid4().hex, tuple(owner.exports)) + owner.exports.clear() + self.flushes[flush.identity] = flush + self.observe( + owner, + "flush_started", + flushId=flush.identity, + timeoutMs=allowed, + configuredTimeoutMs=self.flush_timeout_ms, + pendingExportIds=self.pending(owner), + ) + self._complete_flush(flush) + while ( + flush.result is None and (remaining := deadline - time.monotonic()) > 0 + ): + self.changed.wait(remaining) + outcome = ( + "timeout" + if flush.result is None + else "completed" + if flush.result is SpanExportResult.SUCCESS + else "error" + ) + self.observe( + owner, + "flush_wait_completed", + flushId=flush.identity, + outcome=outcome, + pendingExportIds=self.pending(owner), + **({"error": "Observed export failure"} if outcome == "error" else {}), + ) + return flush.result is SpanExportResult.SUCCESS + + def shutdown(self) -> None: + self.publications.join() + self.publications.put(None) + self.publisher.join() + + +class LedgerExporter(SpanExporter): + def __init__(self, tracker: Tracker) -> None: + self.tracker = tracker + self.shutdown_calls = 0 + + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + return self.export_owned(self.tracker.accept(spans)) + + def export_owned(self, work: Export) -> SpanExportResult: + owner, result, failure = work.owner, SpanExportResult.FAILURE, None + try: + if owner.export_behavior == "held": + if not owner.gate: + raise ValueError("Held export requires its case-owned gate") + owner.trace.emit("OTEL_EXPORT_HELD", view=self.tracker.view) + if owner.shared: + + def held(at_ms): + self.tracker.observe( + owner, + "export_held", + exportId=work.identity, + gateId=owner.gate, + atMs=at_ms, + ) + + def released(release): + if release.held_at_ms is not None: + self.tracker.observe( + owner, + "export_released", + exportId=work.identity, + gateId=owner.gate, + mode=release.mode, + atMs=release.observed_at_ms, + ) + + owner.trace.gate_release( + owner.gate, on_hold=held, on_release=released + ) + else: + owner.trace.gate(owner.gate) + for span in work.spans: + owner.trace.emit( + "OTEL_EXPORT", + view=self.tracker.view, + outcome=owner.export_behavior, + span=span, + ) + self.tracker.observe( + owner, + "otel_export", + exportId=work.identity, + outcome=owner.export_behavior, + traceId=span["trace_id"], + spanId=span["span_id"], + span=span, + ) + result = ( + SpanExportResult.FAILURE + if owner.export_behavior == "failure" + else SpanExportResult.SUCCESS + ) + return result + except Exception as error: + failure = repr(error) + owner.trace.emit("OTEL_EXPORT_ERROR", view=self.tracker.view, error=failure) + return result + finally: + self.tracker.complete(work, result, failure) + + def shutdown(self) -> None: + self.shutdown_calls += 1 + self.tracker.shutdown() + + +class AsyncSpanProcessor(SpanProcessor): + def __init__(self, exporter: LedgerExporter) -> None: + self.exporter = exporter + self.executor = ThreadPoolExecutor( + max_workers=1, thread_name_prefix="lmi-otel-export" + ) + + def on_start(self, span, parent_context=None) -> None: + pass + + def on_end(self, span: ReadableSpan) -> None: + work = self.exporter.tracker.accept((span,)) + self.executor.submit(self.exporter.export_owned, work) + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self.exporter.tracker.force_flush(timeout_millis) + + def shutdown(self) -> None: + self.executor.shutdown(wait=True) + self.exporter.shutdown() diff --git a/lmi_tests/shared_lock.py b/lmi_tests/shared_lock.py new file mode 100644 index 000000000..663549d61 --- /dev/null +++ b/lmi_tests/shared_lock.py @@ -0,0 +1,239 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Conditional S3 ownership for one persistent SDK deployment. + +The lease token fences concurrent runs; it is not an authentication credential. +S3 access continues to require the caller's normal SigV4/IAM authorization. +There is deliberately no expiry-based stealing of a run with residual work. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import json +from typing import Any, Mapping +import uuid + +import boto3 +from botocore.config import Config +from botocore.exceptions import ClientError + + +class OwnershipError(RuntimeError): + """A persistent target cannot be safely acquired, updated, or released.""" + + +@dataclass(frozen=True) +class RunLease: + bucket: str + key: str + run_id: str + token: str + resources: tuple[str, ...] + s3: Any + + @classmethod + def acquire( + cls, + bucket: str, + stack: str, + run_id: str, + resources: tuple[str, ...], + *, + s3: Any = None, + ) -> RunLease: + lease = cls( + bucket, + f"ownership/{stack}/shared-run.json", + run_id, + uuid.uuid4().hex, + tuple(sorted(resources)), + s3 + if s3 is not None + else boto3.client( + "s3", + config=Config( + connect_timeout=3, + read_timeout=10, + retries={"total_max_attempts": 2}, + ), + ), + ) + previous, etag = lease._read() + if previous is not None and ( + previous.get("state") != "released" + or not isinstance(previous.get("profiles"), dict) + or not previous["profiles"] + or any(value != "quiescent" for value in previous["profiles"].values()) + ): + raise OwnershipError( + "Persistent LMI targets are owned or quarantined by another run" + ) + record = { + "schemaVersion": 1, + "runId": run_id, + "token": lease.token, + "state": "owned", + "profiles": {resource: "active" for resource in lease.resources}, + } + try: + lease._write(record, etag) + except ClientError as error: + if error.response["Error"]["Code"] in { + "PreconditionFailed", + "ConditionalRequestConflict", + }: + raise OwnershipError( + "Another run acquired the persistent LMI targets" + ) from error + raise + return lease + + @classmethod + def from_manifest(cls, sdk: Mapping[str, Any]) -> RunLease: + record = sdk.get("lease") + if not isinstance(record, dict): + raise OwnershipError( + "Shared LMI execution requires an acquired persistent run lease" + ) + required = ("bucket", "key", "runId", "token", "resources") + if any(key not in record for key in required): + raise OwnershipError("Run lease manifest is incomplete") + deployment = sdk["deployment"] + if ( + record["bucket"] != deployment["bucket"] + or record["runId"] != deployment["run"] + or record["key"] != f"ownership/{deployment['stack']}/shared-run.json" + or set(record["resources"]) != set(deployment["functions"].values()) + ): + raise OwnershipError("Run lease does not belong to this SDK deployment") + return cls( + record["bucket"], + record["key"], + record["runId"], + record["token"], + tuple(record["resources"]), + boto3.client( + "s3", + region_name=sdk["deployment"]["region"], + config=Config( + connect_timeout=3, + read_timeout=10, + retries={"total_max_attempts": 2}, + ), + ), + ) + + def to_manifest(self) -> dict[str, Any]: + return { + "bucket": self.bucket, + "key": self.key, + "runId": self.run_id, + "token": self.token, + "resources": list(self.resources), + } + + def _read(self) -> tuple[dict[str, Any] | None, str | None]: + try: + response = self.s3.get_object(Bucket=self.bucket, Key=self.key) + except ClientError as error: + if error.response["Error"]["Code"] in {"NoSuchKey", "404", "NotFound"}: + return None, None + raise + with response["Body"] as body: + record = json.loads(body.read()) + if ( + not isinstance(record, dict) + or type(record.get("schemaVersion")) is not int + or record["schemaVersion"] != 1 + ): + raise OwnershipError("Stored run ownership is malformed") + return record, response["ETag"] + + def _write(self, record: Mapping[str, Any], etag: str | None) -> None: + condition = {"IfMatch": etag} if etag is not None else {"IfNoneMatch": "*"} + self.s3.put_object( + Bucket=self.bucket, + Key=self.key, + Body=json.dumps(record, sort_keys=True).encode(), + ContentType="application/json", + **condition, + ) + + def _owned(self, record: Mapping[str, Any] | None) -> dict[str, Any]: + if ( + not isinstance(record, dict) + or record.get("state") != "owned" + or record.get("runId") != self.run_id + or record.get("token") != self.token + or not isinstance(record.get("profiles"), dict) + or set(record["profiles"]) != set(self.resources) + or any( + not isinstance(value, str) + or value not in {"active", "quiescent", "quarantined"} + for value in record["profiles"].values() + ) + ): + raise OwnershipError( + "Persistent target ownership no longer matches this run" + ) + return record + + def assert_owned(self, resource_id: str) -> None: + """Verify the owner and mark a previously settled profile active again.""" + for _ in range(3): + record, etag = self._read() + record = self._owned(record) + if ( + resource_id not in self.resources + or record["profiles"][resource_id] == "quarantined" + ): + raise OwnershipError( + "The requested function is not owned or has residual work" + ) + if record["profiles"][resource_id] == "active": + return + record["profiles"][resource_id] = "active" + try: + self._write(record, etag) + return + except ClientError as error: + if error.response["Error"]["Code"] not in { + "PreconditionFailed", + "ConditionalRequestConflict", + }: + raise + raise OwnershipError("Could not reactivate the owned profile without a race") + + def finish_profile(self, resource_id: str, quiescent: bool) -> None: + if resource_id not in self.resources: + raise OwnershipError("Cannot retire another run's function") + for _ in range(3): + record, etag = self._read() + record = self._owned(record) + previous = record["profiles"][resource_id] + record["profiles"][resource_id] = ( + "quarantined" + if not quiescent or previous == "quarantined" + else "quiescent" + ) + try: + self._write(record, etag) + return + except ClientError as error: + if error.response["Error"]["Code"] not in { + "PreconditionFailed", + "ConditionalRequestConflict", + }: + raise + raise OwnershipError("Could not record retirement without racing another owner") + + def release(self) -> None: + record, etag = self._read() + record = self._owned(record) + if any(value != "quiescent" for value in record["profiles"].values()): + raise OwnershipError( + "Unretired LMI work retains persistent target ownership" + ) + record["state"] = "released" + self._write(record, etag) diff --git a/lmi_tests/shared_observations.py b/lmi_tests/shared_observations.py new file mode 100644 index 000000000..61d40af9c --- /dev/null +++ b/lmi_tests/shared_observations.py @@ -0,0 +1,348 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Invocation-owned evidence from public SDK hooks, without conformance imports.""" + +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +import threading +from typing import Any, Protocol + +from aws_durable_execution_sdk_python.plugin import ( + DurableInstrumentationPlugin, + InvocationEndInfo, + InvocationStartInfo, + OperationChangeInfo, + OperationEndInfo, + OperationInfo, + OperationStartInfo, + OperationType, + UserFunctionEndInfo, + UserFunctionStartInfo, +) + + +class ObservationSink(Protocol): + """Shared with telemetry without a circular import of fixture.Trace.""" + + def observe(self, kind: str, **fields: Any) -> Mapping[str, Any]: ... + + +@dataclass(frozen=True) +class GateRelease: + gate_id: str + observed_at_ms: float + mode: str + held_at_ms: float | None = None + + +@dataclass(frozen=True) +class OperationObservation: + operation_id: str + parent_id: str | None + name: str | None + operation_type: str + sub_type: str | None + status: str + replayed: bool + attempt: int | None + result: str | None + error_type: str | None + error_message: str | None + started_at_ms: float | None + ended_at_ms: float | None + + @classmethod + def from_info(cls, info: OperationInfo) -> OperationObservation: + return cls( + operation_id=info.operation_id, + parent_id=info.parent_id, + name=info.name, + operation_type=info.operation_type.value, + sub_type=info.sub_type, + status=info.status.value, + replayed=info.is_replayed, + attempt=info.attempt, + result=info.result, + error_type=info.error.type if info.error else None, + error_message=info.error.message if info.error else None, + started_at_ms=info.start_time.timestamp() * 1000 + if info.start_time + else None, + ended_at_ms=info.end_time.timestamp() * 1000 if info.end_time else None, + ) + + def to_record(self) -> dict[str, Any]: + return { + "operationId": self.operation_id, + "parentId": self.parent_id, + "name": self.name, + "operationType": self.operation_type, + "subType": self.sub_type, + "status": self.status, + "isReplayed": self.replayed, + "attempt": self.attempt, + "serializedResult": self.result, + "error": {"type": self.error_type, "message": self.error_message} + if self.error_type is not None or self.error_message is not None + else None, + "startedAtMs": self.started_at_ms, + "endedAtMs": self.ended_at_ms, + } + + +@dataclass(frozen=True) +class UserTask: + operation: OperationObservation + task_id: str + + +class _ThreadTasks(threading.local): + def __init__(self) -> None: + self.stack: list[UserTask] = [] + + +class LifecycleObservations(DurableInstrumentationPlugin): + """One native invocation's actual operations and active public user scopes.""" + + def __init__(self, sink: ObservationSink) -> None: + self._sink = sink + self._lock = threading.Lock() + self._operations: dict[str, OperationObservation] = {} + self._closed: set[str] = set() + self._configurations: dict[tuple[str, str | None], int] = {} + self._threads = _ThreadTasks() + self._task_sequence = 0 + + def error(self, message: str) -> None: + self._sink.observe("observation_error", message=message) + + def _remember(self, info: OperationInfo) -> OperationObservation: + operation = OperationObservation.from_info(info) + with self._lock: + self._operations[operation.operation_id] = operation + return operation + + def current(self) -> OperationObservation | None: + return self._threads.stack[-1].operation if self._threads.stack else None + + def parent_id(self) -> str | None: + current = self.current() + if current is None: + self.error("Public user-function scope is unavailable") + return None + return ( + None + if current.operation_type == OperationType.EXECUTION.value + else current.operation_id + ) + + @contextmanager + def root_scope(self) -> Iterator[None]: + """Observe the actual public handler body; root user hooks do not fire.""" + with self._lock: + roots = [ + item + for item in self._operations.values() + if item.operation_type == OperationType.EXECUTION.value + ] + if len(roots) != 1: + self.error("Handler entry has no unique public invocation root operation") + yield + return + task = UserTask(roots[0], "root") + self._threads.stack.append(task) + self._sink.observe( + "task_start", + taskId="root", + source="public-handler", + **task.operation.to_record(), + ) + try: + yield + finally: + self._threads.stack.pop() + self._sink.observe( + "task_exit", + taskId="root", + operationId=task.operation.operation_id, + source="public-handler", + ) + + def body(self, name: str, value: Any, attempt: int) -> None: + current = self.current() + if ( + current is None + or current.operation_type != OperationType.STEP.value + or current.name != name + ): + self.error(f"Body {name!r} has no matching public STEP hook") + return + if current.attempt != attempt: + self.error(f"Body {name!r} attempt differs from its public STEP hook") + return + self._sink.observe( + "body", + operation=name, + operationId=current.operation_id, + parentId=current.parent_id, + attempt=attempt, + value=value, + ) + + def closed_ancestor(self) -> str | None: + current = self.current() + if current is None: + self.error("Late operation has no active public user-function scope") + return None + with self._lock: + operation: OperationObservation | None = current + while operation is not None: + if operation.operation_id in self._closed: + return operation.operation_id + operation = self._operations.get(operation.parent_id or "") + self.error("Late operation has no observed completed ancestor") + return None + + def completed_context(self, name: str) -> str | None: + parent_id = self.parent_id() + with self._lock: + matches = [ + operation.operation_id + for operation in self._operations.values() + if operation.name == name + and operation.parent_id == parent_id + and operation.operation_type == OperationType.CONTEXT.value + and operation.operation_id in self._closed + ] + if len(matches) != 1: + self.error( + f"Returned context {name!r} has no unique public completion hook" + ) + return None + return matches[0] + + @contextmanager + def configuration(self, name: str, max_concurrency: int) -> Iterator[None]: + """Bind an actual public call's config to its operation-start hook.""" + key = (name, self.parent_id()) + with self._lock: + self._configurations[key] = max_concurrency + try: + yield + finally: + with self._lock: + missing = self._configurations.pop(key, None) + if missing is not None: + self.error( + f"Configured call {name!r} had no matching public operation hook" + ) + + def on_invocation_start(self, info: InvocationStartInfo) -> None: + operations = [ + self._remember(operation) for operation in info.operations.values() + ] + with self._lock: + self._closed.update( + operation.operation_id + for operation in operations + if operation.operation_type == OperationType.CONTEXT.value + and operation.status + in {"SUCCEEDED", "FAILED", "CANCELLED", "TIMED_OUT", "STOPPED"} + ) + self._sink.observe( + "invocation_start", + requestId=info.request_id, + executionArn=info.execution_arn, + isFirstInvocation=info.is_first_invocation, + operations=[operation.to_record() for operation in operations], + ) + + def on_invocation_end(self, info: InvocationEndInfo) -> None: + operations = [ + self._remember(operation) for operation in info.operations.values() + ] + self._sink.observe( + "invocation_end", + outcome=info.status.value, + operations=[operation.to_record() for operation in operations], + serializedResult=info.execution_result, + error={"type": info.error.type, "message": info.error.message} + if info.error + else None, + ) + + def on_operation_start(self, info: OperationStartInfo) -> None: + operation = self._remember(info) + self._sink.observe("operation_start", **operation.to_record()) + with self._lock: + configured = self._configurations.pop( + (operation.name or "", operation.parent_id), None + ) + if configured is not None: + self._sink.observe( + "operation_config", + operationId=operation.operation_id, + maxConcurrency=configured, + source="public-fixture", + ) + + def on_operation_end(self, info: OperationEndInfo) -> None: + operation = self._remember(info) + self._sink.observe("operation_end", **operation.to_record()) + if info.operation_type is OperationType.CONTEXT: + with self._lock: + self._closed.add(operation.operation_id) + self._sink.observe( + "scope_closed", + scopeId=operation.operation_id, + parentId=operation.parent_id, + ) + + def on_operation_change(self, info: OperationChangeInfo) -> None: + changed = [ + self._remember(operation) for operation in info.updated_operations.values() + ] + self._sink.observe( + "operation_change", + operations=[operation.to_record() for operation in changed], + ) + + def on_user_function_start(self, info: UserFunctionStartInfo) -> None: + operation = self._remember(info) + with self._lock: + self._task_sequence += 1 + ordinal = self._task_sequence + task_id = ( + "root" + if info.operation_type is OperationType.EXECUTION + else f"user-task-{ordinal}" + ) + self._threads.stack.append(UserTask(operation, task_id)) + self._sink.observe("task_start", taskId=task_id, **operation.to_record()) + + def on_user_function_end(self, info: UserFunctionEndInfo) -> None: + stack = self._threads.stack + if not stack or stack[-1].operation.operation_id != info.operation_id: + self.error("Public user-function end did not match its active start") + return + task = stack.pop() + self._sink.observe( + "task_exit", + taskId=task.task_id, + operationId=info.operation_id, + parentId=info.parent_id, + outcome=info.outcome.value, + ) + + +@dataclass(frozen=True) +class LifecycleObservationFactory: + """One factory/observer is constructed for each native fixture invocation.""" + + observer: LifecycleObservations + + def create_plugin(self, info: InvocationStartInfo) -> DurableInstrumentationPlugin: + return self.observer diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py new file mode 100644 index 000000000..b2878dde7 --- /dev/null +++ b/lmi_tests/shared_scenarios.py @@ -0,0 +1,603 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Drive existing public-API fixtures through coordinated service boundaries.""" + +from __future__ import annotations + +from collections import Counter +from collections.abc import Callable, Mapping +from copy import deepcopy +import hashlib +import time +from typing import Any +import uuid + +from aws_durable_execution_conformance_tests.lmi.coordinator import ScenarioContext +from lmi_tests.cloud import Cloud +from lmi_tests.evidence import CollectionError, PlacementError, overlap, select + + +_SIMPLE = { + "native-concurrency": "barrier", + "replay-callback-retry": "replay", + "warm-lifecycle": "success", + "early-completion-parallel": "parallel", + "early-completion-map": "map", + "early-completion-nested": "nested", + "checkpoint-settlement": "checkpoint", + "deadline-recovery": "deadline", + "timeout-retry": "deadline", + "pending-cleanup": "suspend-cleanup", + "nested-progress": "nested-progress", + "return-inflight": "return-inflight", + "failure-inflight": "failure-inflight", + "late-operation": "late-operation", + "runtime-carrier-sampled": "success", + "runtime-carrier-unsampled": "success", + "otel-warm-lifecycle": "success", + "otel-concurrent": "barrier", + "export-error-recovery": "success", + "export-success": "success", + "export-failure-recovery": "success", + "held-export": "success", +} +_COHORT = {"native-concurrency", "nested-progress", "otel-concurrent"} +_ANCHORED = {"pending-cleanup", "return-inflight", "failure-inflight"} + + +def canonical(events: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Read actual observation records without replacing missing boundaries.""" + return [ + { + "requestId": event.get("request"), + "ownerRequestId": event.get("request"), + "executionArn": event.get("execution"), + **event["observation"], + } + for event in events + if event.get("phase") == "CONFORMANCE" + and isinstance(event.get("observation"), dict) + ] + + +class ScenarioDriver: + """Case-local orchestration; the shared package owns acceptance assertions.""" + + def __init__( + self, + context: ScenarioContext, + cloud: Cloud, + *, + release_capacity: Callable[[], None] | None = None, + reserve_capacity: Callable[[], None] | None = None, + ) -> None: + self.context = context + self.cloud = cloud + self.items: dict[str, dict[str, Any]] = {} + self.release_capacity = release_capacity + self.reserve_capacity = reserve_capacity + self.boundary: dict[str, Any] | None = None + self.peer_gate: str | None = None + self.cohort_gate: str | None = None + self.cohort_gates: dict[str, str] = {} + self.target_environment: str | None = None + + @property + def scenario(self) -> str: + return self.context.scenario.scenario + + def seconds(self, budget: str, fallback: float) -> float: + value = self.context.scenario.behavior_budgets.get(budget) + return value / 1000 if value is not None else fallback + + def gate(self, label: str) -> str: + owner = hashlib.sha256( + f"{self.context.run_id}:{self.context.scenario.case_id}:{self.context.scenario.cohort_id}".encode() + ).hexdigest()[:20] + return f"case-{owner}-{label}" + + def launch( + self, + role: str, + fixture: str, + *, + gate: str | None = None, + resume_roles: tuple[str, ...] = (), + target_environment: str | None = None, + exporter: str = "success", + ) -> dict[str, Any]: + if role in self.items: + raise CollectionError("A later launch cannot replace a selected original") + payload = self.context.dispatcher(role) + sampled = "0" if self.scenario == "runtime-carrier-unsampled" else "1" + # This header is sent on the real caller's HTTP Invoke request. It never + # supplies the runtime carrier through function input. + header = ( + f"Root=1-{int(time.time()):08x}-{uuid.uuid4().hex[:24]};" + f"Parent={uuid.uuid4().hex[:16]};Sampled={sampled}" + if self.context.scenario.otel_view + else None + ) + item = self.cloud.start( + fixture, + gate=gate, + target_environment=target_environment, + otel_view=self.context.scenario.otel_view, + trace_header=header, + otel_export=exporter, + role=role, + resume_roles=resume_roles, + dispatcher_payload=payload, + marker=self.gate(role) + "-" + uuid.uuid4().hex[:12], + ) + self.items[role] = item + return item + + def start(self) -> None: + if self.scenario not in _SIMPLE: + raise NotImplementedError(f"No Python fixture for {self.scenario}") + if self.scenario in _COHORT: + self.cohort_gate = self.gate("cohort") + for role in ("victim", "peer")[: self.cloud.concurrency]: + gate = ( + self.gate(f"cohort-{role}") + if self.scenario == "otel-concurrent" + else self.cohort_gate + ) + self.cohort_gates[role] = gate + self.launch(role, _SIMPLE[self.scenario], gate=gate) + return + if self.scenario in _ANCHORED and self.cloud.concurrency == 2: + self.peer_gate = self.gate("peer") + peer = self.launch("peer", "barrier", gate=self.peer_gate) + self.target_environment = self.cloud.phase(peer, "BLOCKED")[0][ + "environment" + ] + resume_roles = ( + ("resume",) + if self.scenario in {"replay-callback-retry", "pending-cleanup"} + else ("retry",) + if self.scenario == "timeout-retry" + else () + ) + exporter = ( + "held" + if self.scenario == "held-export" + else "failure" + if self.scenario in {"export-error-recovery", "export-failure-recovery"} + else "success" + ) + self.launch( + "victim", + _SIMPLE[self.scenario], + gate=self.gate("fault") + if self.scenario in {"deadline-recovery", "timeout-retry"} + else None, + resume_roles=resume_roles, + target_environment=self.target_environment, + exporter=exporter, + ) + + def wait_overlap( + self, roles: tuple[str, ...], environment: str | None = None + ) -> list[dict[str, Any]]: + markers = {self.items[role]["marker"] for role in roles} + + def established() -> list[dict[str, Any]] | None: + events = [ + event for event in self.cloud.refresh() if event["marker"] in markers + ] + if environment is not None and any( + event["environment"] != environment + for event in select(events, "BLOCKED") + ): + raise PlacementError( + "Selected cohort reached a replacement environment" + ) + try: + overlap(events, markers, len(roles), environment) + except PlacementError: + return None + return events + + return self.cloud.poll( + established, + seconds=self.cloud.manifest.get("driverTimeout", 120), + category=PlacementError, + message="Original cohort placement/overlap was not established", + ) + + def place(self) -> None: + victim = self.items["victim"] + if self.scenario in _COHORT: + self.wait_overlap(tuple(self.items)) + return + if self.scenario in _ANCHORED and self.cloud.concurrency == 2: + admission = self.cloud.poll( + lambda: [ + event + for event in self.cloud.for_item(victim) + if event["phase"] in {"PLACEMENT_ACCEPTED", "PLACEMENT_MISS"} + ], + category=PlacementError, + message="Victim placement admission was not observed", + )[0] + if admission["phase"] != "PLACEMENT_ACCEPTED": + raise PlacementError("Original victim missed the selected environment") + if self.scenario in {"deadline-recovery", "timeout-retry"}: + self.boundary = self.cloud.phase(victim, "BLOCKED")[0] + if self.scenario == "deadline-recovery" and self.cloud.concurrency == 2: + self.peer_gate = self.gate("peer") + self.launch("peer", "barrier", gate=self.peer_gate) + self.wait_overlap(("victim", "peer"), self.boundary["environment"]) + else: + self.cloud.phase(victim, "WRAPPER_ENTER") + + def _finish(self, item: dict[str, Any], status: str = "SUCCEEDED") -> None: + self.cloud.finish(item, status) + + def _callback(self, item: dict[str, Any]) -> None: + # Public callback waiting observes real service history. Waiting does not + # create a replacement durable execution. + self.cloud.poll( + lambda: [ + event + for event in self.cloud.for_item(item) + if event["phase"] == "WRAPPER_RETURN" + and event.get("status") == "PENDING" + ], + seconds=self.cloud.manifest.get("driverTimeout", 120), + message="Callback fixture did not actually suspend", + ) + if self.release_capacity is not None: + self.release_capacity() + try: + callback = item["runner"].wait_for_callback( + item["arn"], + name="completion", + timeout=self.cloud.manifest.get("driverTimeout", 120), + ) + finally: + if self.reserve_capacity is not None: + self.reserve_capacity() + self.cloud.check_active() + # Retain every earlier retry/wait invocation before sending completion. + self.cloud.for_item(item) + item["runner"].send_callback_success(callback, result=item["marker"].encode()) + self.cloud.callback_records.append( + { + "executionArn": item["arn"], + "callbackId": callback, + "action": "success", + "time": time.time(), + } + ) + + def _peer_progress(self, boundary: Mapping[str, Any]) -> None: + if "peer" not in self.items: + return + self.wait_overlap(("victim", "peer"), self.target_environment) + self.cloud.poll( + lambda: [ + event + for event in self.cloud.for_item(self.items["peer"]) + if event["phase"] == "ALIVE" and event["time"] >= boundary["time"] + ], + message="Selected healthy peer did not progress during the held boundary", + ) + + def _finish_peer(self) -> None: + if self.peer_gate is not None: + self.cloud.release(self.peer_gate) + self._finish(self.items["peer"]) + + def _warm(self) -> None: + count = 3 if self.scenario == "warm-lifecycle" else 2 + sequence = ["success", "failure", "replay"] * count + for index, fixture in enumerate(sequence): + if index == 0: + item = self.items["victim"] + else: + if self.scenario == "warm-lifecycle": + role = "warm" if index == 1 else f"warm-{index}" + else: + role = "warm" if index == 5 else f"warm-{index}" + item = self.launch( + role, + fixture, + resume_roles=(f"{role}-resume",) if fixture == "replay" else (), + ) + if fixture == "replay": + self._callback(item) + self._finish(item, "FAILED" if fixture == "failure" else "SUCCEEDED") + + def _deadline(self) -> None: + victim = self.items["victim"] + blocked = self.boundary + if blocked is None: + raise CollectionError("Deadline fixture lost its original placement record") + if self.scenario == "deadline-recovery": + self._finish_peer() + self.cloud.poll( + lambda: time.time() >= blocked["deadline"] - 1, + seconds=self.cloud.manifest["invocationTimeout"] + 5, + ) + gate = self.gate("recovery") + roles = ("recovery", "recovery-peer")[: self.cloud.concurrency] + for role in roles: + self.launch(role, "barrier", gate=gate) + # Demand the original environment before eventual service-history + # delivery consumes the measurement window. + self.wait_overlap(roles, blocked["environment"]) + self.cloud.release(gate) + for role in roles: + self._finish(self.items[role]) + self.cloud.platform_timeout(victim, blocked["request"]) + cutoff = blocked["deadline"] + self.seconds("cleanupMs", 5) + self.cloud.poll( + lambda: time.time() > cutoff + self.seconds("observationGraceMs", 1), + seconds=self.cloud.manifest["invocationTimeout"] + 30, + ) + if self.scenario == "timeout-retry": + self.cloud.poll( + lambda: ( + len( + { + event["request"] + for event in self.cloud.for_item(victim) + if event["phase"] == "WRAPPER_ENTER" + } + ) + >= 2 + ), + seconds=self.seconds("retryMs", 40), + message="No real service retry of the original execution was observed", + ) + # Close the measurement before the normal release. The later execution + # result cannot rewrite the original deadline, cohort or late-activity + # evidence, and no outcome is invented for a still-running retry. + self.cloud.history(victim) + events = self.cloud.refresh(validate_controls=False) + selected = { + attempt["role"]: attempt["requestId"] + for attempt in self.context.attempts + if attempt["attempt"] == 1 and attempt.get("requestId") + } + if selected.get("victim") != blocked["request"] or not set( + self.context.scenario.roles + ) <= set(selected): + raise CollectionError( + "Deadline measurement could not bind every original selected request" + ) + self.cloud.record_deadline_measurement( + { + "runId": self.context.run_id, + "caseId": self.context.scenario.case_id, + "cohortId": self.context.scenario.cohort_id, + "scenario": self.scenario, + "executionArn": victim["arn"], + "selectedRequests": selected, + "deadlineAtMs": blocked["deadline"] * 1000, + "cleanupCutoffAtMs": cutoff * 1000, + "observationCutoffAtMs": ( + cutoff + self.seconds("observationGraceMs", 1) + ) + * 1000, + "observedAtMs": time.time() * 1000, + "events": deepcopy(events), + "histories": deepcopy(self.cloud.raw_histories), + "sourceUris": sorted(self.cloud.source_uris), + } + ) + self.cloud.release(self.gate("fault")) + self.cloud.wait_terminal(victim) + + def _held_export(self) -> None: + victim = self.items["victim"] + self.cloud.phase(victim, "OTEL_EXPORT_HELD") + if self.context.scenario.export_mode == "async-bounded-flush": + self.cloud.poll( + lambda: [ + event + for event in canonical(self.cloud.for_item(victim)) + if event.get("kind") == "flush_wait_completed" + and event.get("outcome") == "timeout" + ], + message="Asynchronous held export did not reach the real bounded flush timeout", + ) + self.cloud.phase(victim, "WRAPPER_RETURN") + self.cloud.release(victim["marker"] + "-export") + self._finish(victim) + self.cloud.poll( + lambda: [ + event + for event in canonical(self.cloud.for_item(victim)) + if event.get("kind") == "export_completed" + ], + message="Held exporter did not actually finish after its case-owned release", + ) + + def execute(self) -> None: + victim = self.items["victim"] + if self.scenario in _COHORT: + if self.cohort_gate is None: + raise CollectionError("Missing cohort barrier") + # Separate gates let telemetry observe one request ending while its + # peer remains held. Ordinary cohorts share one case-owned barrier. + if self.scenario != "otel-concurrent": + self.cloud.release(self.cohort_gate) + for role, item in self.items.items(): + if self.scenario == "otel-concurrent": + self.cloud.release(self.cohort_gates[role]) + if self.scenario == "nested-progress": + self.cloud.phase( + item, "PROGRESS", seconds=self.seconds("progressMs", 20) + ) + self._finish(item) + if ( + self.scenario == "otel-concurrent" + and role == "victim" + and "peer" in self.items + ): + returned = self.cloud.phase(item, "WRAPPER_RETURN")[-1] + self.cloud.poll( + lambda: [ + event + for event in self.cloud.for_item(self.items["peer"]) + if event["phase"] == "ALIVE" + and event["time"] >= returned["time"] + ], + message="Ending the original OTel invocation did not preserve its held peer", + ) + elif self.scenario in {"warm-lifecycle", "otel-warm-lifecycle"}: + self._warm() + elif self.scenario == "replay-callback-retry": + self._callback(victim) + self._finish(victim) + elif self.scenario.startswith("early-completion-"): + winner = self.cloud.phase(victim, "WINNER_SELECTED")[0] + cutoff = ( + winner["time"] + + self.seconds("cleanupMs", 5) + + self.seconds("observationGraceMs", 1) + ) + self.cloud.poll( + lambda: time.time() >= cutoff, seconds=self.seconds("cleanupMs", 5) + 30 + ) + # Do not release the losing branch to manufacture earlier cleanup. + self.cloud.for_item(victim) + elif self.scenario in {"deadline-recovery", "timeout-retry"}: + self._deadline() + elif self.scenario == "checkpoint-settlement": + self.cloud.phase(victim, "ACK_HELD") + self.cloud.phase(victim, "BLOCKED") + self.cloud.release(victim["marker"] + "-checkpoint") + self._finish(victim) + elif self.scenario == "pending-cleanup": + blocked = self.cloud.phase(victim, "BLOCKED")[0] + self._peer_progress(blocked) + self.cloud.release(victim["marker"] + "-cleanup") + self._finish(victim) + self._finish_peer() + elif self.scenario in {"return-inflight", "failure-inflight"}: + self.cloud.phase(victim, "WINNER_SELECTED") + exit_boundary = self.cloud.phase(victim, "USER_EXIT")[0] + self._peer_progress(exit_boundary) + self.cloud.release(victim["marker"] + "-loser") + self._finish( + victim, "FAILED" if self.scenario == "failure-inflight" else "SUCCEEDED" + ) + self._finish_peer() + elif self.scenario == "late-operation": + self.cloud.phase(victim, "WINNER_SELECTED") + self.cloud.release(victim["marker"] + "-loser") + self.cloud.poll( + lambda: [ + event + for event in self.cloud.for_item(victim) + if event["phase"] in {"LATE_REJECTED", "LATE_ACCEPTED"} + ], + message="Residual child never attempted the late public durable operation", + ) + self._finish(victim) + elif self.scenario == "held-export": + self._held_export() + else: + self._finish(victim) + if self.scenario in {"export-error-recovery", "export-failure-recovery"}: + self._finish(self.launch("warm", "success")) + self.cloud.observation_ended_at = time.time() + + def collect(self) -> None: + if not self.scenario.startswith("early-completion-") and self.scenario not in { + "deadline-recovery", + "timeout-retry", + }: + self.cloud.poll( + self.quiescent, + seconds=self.cloud.manifest.get("driverTimeout", 120), + message="Completed scenario evidence still has unsettled task/export publication", + ) + self.cloud.wait(self.seconds("observationGraceMs", 1)) + self.cloud.collect(full_run=False) + self.cloud.observation_ended_at = time.time() + + def quiescent(self) -> bool: + events = self.cloud.refresh(validate_controls=False) + observations = canonical(events) + if any(item.get("kind") == "observation_error" for item in observations): + return False + if not self.cloud.invocations: + return True + if any( + not item.get("arn") or not item["native_requests"] + for item in self.cloud.invocations + ): + return False + entered = {event["request"] for event in select(events, "WRAPPER_ENTER")} + ended = { + event["request"] + for event in events + if event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + } + if not entered or not entered <= ended: + return False + for start, end, identity in ( + ("task_start", "task_exit", "taskId"), + ("io_start", "io_exit", "taskId"), + ("checkpoint_call", "checkpoint_exit", "callId"), + ("poll_call", "poll_exit", "callId"), + ("export_started", "export_completed", "exportId"), + ("flush_started", "flush_completed", "flushId"), + ): + starts = Counter( + (item.get("requestId"), item.get(identity)) + for item in observations + if item.get("kind") == start + ) + ends = Counter( + (item.get("requestId"), item.get(identity)) + for item in observations + if item.get("kind") == end + ) + if starts - ends: + return False + # The public root task hook is a positive witness that task instrumentation + # ran; absent instrumentation must not look like an empty, settled pool. + root_ends = { + item.get("requestId") + for item in observations + if item.get("kind") == "task_exit" and item.get("taskId") == "root" + } + if not entered <= root_ends: + return False + if self.context.scenario.otel_view: + published = { + item.get("requestId") + for item in observations + if item.get("kind") == "export_quiescence" + and item.get("pendingExportIds") == [] + and item.get("pendingFlushIds") == [] + and item.get("publicationCount") == 0 + } + if not entered <= published: + return False + return True + + def retire(self) -> bool: + # Recover native request IDs for responses lost during the cancelled + # phase before deciding which owned service executions can be stopped. + self.cloud.refresh(validate_controls=False) + if any(not item.get("arn") for item in self.cloud.invocations): + raise CollectionError( + "A launched invocation has no proven execution identity" + ) + self.cloud.settle_case() + self.cloud.poll( + self.quiescent, + seconds=self.cloud.manifest.get("driverTimeout", 120), + message="Case-owned task/request/export work is not quiescent", + ) + self.cloud.collect(full_run=False) + self.cloud.retired_at = time.time() + return True diff --git a/lmi_tests/shared_telemetry.py b/lmi_tests/shared_telemetry.py new file mode 100644 index 000000000..217bfc43f --- /dev/null +++ b/lmi_tests/shared_telemetry.py @@ -0,0 +1,207 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Driver-only conversion of actual OTel ledger data to the shared wire model. + +This module is not part of the Lambda fixture bundle. It imports the pinned +conformance model only when converting observed spans, not on runtime startup. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime +from typing import Any + + +def _canonical_span(raw: Mapping[str, Any]) -> dict[str, Any]: + from aws_durable_execution_conformance_tests_otel.model import ( + Span, + SpanLink, + span_to_dict, + ) + + return span_to_dict( + Span( + trace_id=raw["trace_id"], + span_id=raw["span_id"], + name=raw["name"], + start_time=datetime.fromtimestamp(raw["start_ns"] / 1_000_000_000, UTC), + end_time=datetime.fromtimestamp(raw["end_ns"] / 1_000_000_000, UTC), + parent_span_id=raw["parent_id"], + status=raw["status"], + kind=raw["kind"], + attributes=raw["attributes"], + service_name=raw["service_name"], + links=tuple(SpanLink(**link) for link in raw["links"]), + exception_event_count=raw["exception_event_count"], + ) + ) + + +def collect_telemetry( + raw_events: Sequence[Mapping[str, Any]], + attempts: Sequence[Mapping[str, Any]], + *, + view: str, + expected_sampled: bool, + caller_headers: Mapping[str, str], + observed_until_ms: float, + export_mode: str | None = None, +) -> dict[str, Any]: + """Preserve exporter ownership and report incomplete collection explicitly. + + Expected sampling and caller headers come from the admitted driver's actual + Invoke configuration. They are never inferred from the observed runtime + carrier or from whether telemetry happened to arrive. + """ + by_request = {attempt["requestId"]: attempt for attempt in attempts} + observations = [ + (record, record["observation"]) + for record in raw_events + if record.get("phase") == "CONFORMANCE" + and isinstance(record.get("observation"), Mapping) + ] + providers: dict[str, dict[str, Any]] = {} + bindings, carriers, spans, outcomes = [], [], [], [] + complete = bool(attempts) + for request_id, attempt in by_request.items(): + records = [ + record for record in raw_events if record.get("request") == request_id + ] + local = [ + (record, observation) + for record, observation in observations + if record.get("request") == request_id + ] + numbers = [record.get("sequence") for record in records] + complete &= bool(numbers) and all(type(number) is int for number in numbers) + if numbers and all(type(number) is int for number in numbers): + sequences = [number for number in numbers if isinstance(number, int)] + complete &= sorted(sequences) == list(range(1, max(sequences) + 1)) + complete &= any( + record.get("phase") in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + for record in records + ) + quiescent = [ + observation + for _, observation in local + if observation.get("kind") == "export_quiescence" + ] + complete &= bool(quiescent) and all( + item.get("pendingExportIds") == [] + and item.get("pendingFlushIds") == [] + and item.get("publicationCount") == 0 + for item in quiescent + ) + entries = [ + (record, item) for record, item in local if item.get("kind") == "otel_enter" + ] + if any(item.get("otelView") != view for _, item in entries): + raise ValueError("The observed OTel view differs from the admitted view") + if export_mode is not None and any( + item.get("exportMode") != export_mode for _, item in entries + ): + raise ValueError( + "The observed exporter mode differs from the admitted mode" + ) + completed = [ + item for _, item in local if item.get("kind") == "export_completed" + ] + started = [item for _, item in local if item.get("kind") == "export_started"] + complete &= len(entries) == 1 and {item["exportId"] for item in started} == { + item["exportId"] for item in completed + } + for _, observation in local: + if observation.get("kind") == "provider_observed": + provider = { + key: observation[key] + for key in ( + "providerId", + "exporterId", + "createdAtMs", + "environmentId", + "processId", + ) + } + previous = providers.setdefault(provider["providerId"], provider) + if previous != provider: + raise ValueError( + "The observed provider identity changed across requests" + ) + if observation.get("kind") == "otel_export": + raw_span = observation["span"] + spans.append( + { + "requestId": request_id, + "exportId": observation["exportId"], + "sampled": raw_span["sampled"], + "span": _canonical_span(raw_span), + } + ) + if len(entries) == 1: + source, entry = entries[0] + bindings.append( + {"requestId": request_id, "providerId": entry["providerId"]} + ) + carriers.append( + { + "requestId": request_id, + "environmentId": source["environment"], + "processId": source["process"], + "workerId": source.get("workerId", source["process"]), + "source": entry["carrierSource"], + "header": entry.get("carrier"), + "expectedSampled": expected_sampled, + **( + {"callerHeader": caller_headers[request_id]} + if request_id in caller_headers + else {} + ), + } + ) + failures = [item for item in completed if item.get("outcome") == "failed"] + if failures: + failed = failures[0] + outcomes.append( + { + "requestId": request_id, + "status": "export-error", + "error": failed.get("error"), + "fixture": failed.get("fixture"), + } + ) + else: + outcomes.append( + { + "requestId": request_id, + "status": "delivered" + if started + else "unsampled" + if not expected_sampled + else "missing", + } + ) + unknown = { + record.get("request") + for record, item in observations + if item.get("kind") == "otel_export" + } - set(by_request) + if unknown: + raise ValueError("Telemetry contains an unjournaled original invocation") + return { + "providers": list(providers.values()), + "providerBindings": bindings, + "views": { + view: { + "collection": { + "status": "complete" if complete else "incomplete", + "scope": "cohort", + "requestIds": list(by_request), + "throughMs": observed_until_ms, + }, + "carriers": carriers, + "spans": spans, + "exportOutcomes": outcomes, + } + }, + } diff --git a/lmi_tests/telemetry.py b/lmi_tests/telemetry.py index 3d4b7048a..adbee4125 100644 --- a/lmi_tests/telemetry.py +++ b/lmi_tests/telemetry.py @@ -1,25 +1,21 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Real OTel export into the LMI evidence ledger; no runtime carrier injection.""" +"""Real public OTel plugins with immutable runtime and invocation ownership.""" from __future__ import annotations -from collections.abc import Callable, Sequence -from contextvars import ContextVar +from collections.abc import Callable from importlib.metadata import version import os import threading +import time +from typing import Any import uuid from weakref import WeakKeyDictionary -from typing import Any, TYPE_CHECKING - -from opentelemetry import context as otel_context -from opentelemetry.sdk.trace import ReadableSpan, TracerProvider -from opentelemetry.sdk.trace.export import ( - SimpleSpanProcessor, - SpanExporter, - SpanExportResult, -) + +from opentelemetry import context as otel_context, trace as otel_trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.sampling import ALWAYS_ON from aws_durable_execution_sdk_python_otel import ( @@ -32,64 +28,57 @@ DurableInstrumentationPluginFactory, InvocationStartInfo, ) +from lmi_tests.shared_exporter import ( + AsyncSpanProcessor, + CURRENT_OWNER, + DEFAULT_FLUSH_TIMEOUT_MS, + LedgerExporter, + Owner, + Trace, + Tracker, + owned, +) -if TYPE_CHECKING: - from lmi_tests.fixture import Trace - -_CURRENT_TRACE: ContextVar[Trace | None] = ContextVar("lmi_otel_evidence", default=None) -_RUNTIMES: dict[str, Runtime] = {} +_RUNTIMES: dict[tuple[str, str], Runtime] = {} _RUNTIME_LOCK = threading.Lock() -class LedgerExporter(SpanExporter): - """Capture the actual exported data, independently of expected attribution.""" - - def __init__(self, view: str) -> None: - self.view = view - self.shutdown_calls = 0 - self.active_trace: Trace | None = None - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - # The checkpoint batcher may export outside the caller's ContextVar. - # Native Python LMI has one request per process. Keep its independently - # observed current invocation as a fallback; never infer ownership from - # the span's potentially corrupted execution attributes. - diagnostic = _CURRENT_TRACE.get() or self.active_trace - if diagnostic is None: - raise RuntimeError("OTel export lost the invocation evidence context") - mode = diagnostic.payload.get("otel_export", "success") - if mode == "held": - diagnostic.emit("OTEL_EXPORT_HELD", view=self.view) - diagnostic.gate(diagnostic.payload["otel_export_gate"]) - for span in spans: - context = span.context - if context is None: - raise ValueError("Exported span has no span context") - data = { - "name": span.name, - "trace_id": format(context.trace_id, "032x"), - "span_id": format(context.span_id, "016x"), - "parent_id": format(span.parent.span_id, "016x") - if span.parent - else None, - "sampled": context.trace_flags.sampled, - "start_ns": span.start_time, - "end_ns": span.end_time, - "status": span.status.status_code.name, - "attributes": dict(span.attributes or {}), - } - diagnostic.emit("OTEL_EXPORT", view=self.view, outcome=mode, span=data) - return ( - SpanExportResult.FAILURE if mode == "failure" else SpanExportResult.SUCCESS - ) +class ObservedPlugin(DurableInstrumentationPlugin): + """Forward the real public hooks, binding their immutable invocation owner.""" - def shutdown(self) -> None: - self.shutdown_calls += 1 + def __init__(self, delegate: DurableInstrumentationPlugin, owner: Owner) -> None: + self.delegate, self.owner = delegate, owner + def _call(self, name: str, info: Any) -> Any: + with owned(self.owner): + return getattr(self.delegate, name)(info) + + def on_invocation_start(self, info) -> None: + self._call("on_invocation_start", info) + + def on_invocation_end(self, info) -> None: + self._call("on_invocation_end", info) + + def on_operation_start(self, info) -> None: + self._call("on_operation_start", info) + + def on_operation_end(self, info) -> None: + self._call("on_operation_end", info) + + def on_operation_change(self, info) -> None: + self._call("on_operation_change", info) + + def on_user_function_start(self, info) -> None: + self._call("on_user_function_start", info) + + def on_user_function_end(self, info) -> None: + self._call("on_user_function_end", info) + + def provide_propagation_metadata(self, info): + return self._call("provide_propagation_metadata", info) -class EvidenceFactory: - """Observe factory ownership while returning the actual bundled OTel plugin.""" +class EvidenceFactory: __durable_registration_api__ = 1 exclusive_group = "aws-durable-execution-otel-view" @@ -108,30 +97,86 @@ def __init__(self, view: str, provider: TracerProvider) -> None: self.lock = threading.Lock() def create_plugin(self, info: InvocationStartInfo) -> DurableInstrumentationPlugin: - trace = _CURRENT_TRACE.get() - if trace is None: - raise RuntimeError("Factory lost the invocation evidence context") + owner = CURRENT_OWNER.get() + if owner is None: + raise RuntimeError("Factory lost the invocation evidence owner") plugin = self.delegate.create_plugin(info) with self.lock: identity = self.instances.setdefault(plugin, uuid.uuid4().hex) - trace.emit( + owner.trace.emit( "OTEL_FACTORY", plugin_instance=identity, plugin_execution=info.execution_arn, ) - return plugin + owner.tracker.observe( + owner, + "otel_factory", + pluginInstanceId=identity, + pluginExecutionArn=info.execution_arn, + ) + return ObservedPlugin(plugin, owner) class Runtime: - """A provider/exporter shared within one Python worker, across invocations.""" - - def __init__(self, view: str) -> None: + """An immutable provider variant shared across invocations in one process.""" + + def __init__( + self, + view: str, + export_mode: str = "sync", + *, + flush_timeout_ms: float = DEFAULT_FLUSH_TIMEOUT_MS, + ) -> None: + if view not in {"invocation", "execution"} or export_mode not in { + "sync", + "async-bounded-flush", + }: + raise ValueError("Unknown OTel view or exporter mode") self.identity = uuid.uuid4().hex - self.exporter = LedgerExporter(view) + self.created_at_ms = time.time_ns() / 1_000_000 + self.view, self.export_mode = view, export_mode + self.tracker = Tracker( + self.identity, uuid.uuid4().hex, view, export_mode, flush_timeout_ms + ) + self.exporter = LedgerExporter(self.tracker) self.provider = TracerProvider(sampler=ALWAYS_ON, shutdown_on_exit=False) - self.provider.add_span_processor(SimpleSpanProcessor(self.exporter)) + self.provider.add_span_processor( + SimpleSpanProcessor(self.exporter) + if export_mode == "sync" + else AsyncSpanProcessor(self.exporter) + ) self.factory = EvidenceFactory(view, self.provider) + def pending_for_case(self, case_id: str) -> dict[str, Any]: + return self.tracker.active(case_id) + + +def active_work(trace: Trace) -> dict[str, Any]: + """Actual outstanding work/publication across immutable runtime variants.""" + case_id = str(trace.payload.get("caseId", trace.identity.get("marker", ""))) + result: dict[str, Any] = { + "pendingExportIds": [], + "pendingFlushIds": [], + "publicationCount": 0, + "publicationErrors": [], + } + with _RUNTIME_LOCK: + runtimes = tuple(_RUNTIMES.values()) + for runtime in runtimes: + state = runtime.pending_for_case(case_id) + for field in ("pendingExportIds", "pendingFlushIds", "publicationErrors"): + result[field].extend(state[field]) + result["publicationCount"] += state["publicationCount"] + return result + + +def _context_snapshot() -> dict[str, str | None]: + context = otel_trace.get_current_span().get_span_context() + return { + "traceId": format(context.trace_id, "032x") if context.is_valid else None, + "spanId": format(context.span_id, "016x") if context.is_valid else None, + } + def invoke( trace: Trace, @@ -139,30 +184,51 @@ def invoke( [list[DurableInstrumentationPluginFactory]], dict[str, Any] ], ) -> dict[str, Any]: - """Enter before SDK dispatch so worker-context propagation is also exercised.""" - view = trace.payload["otel_view"] - if view not in {"invocation", "execution"}: - raise ValueError("Unknown OTel view") + view = trace.payload.get("otel_view", trace.payload.get("otelView")) + export_mode = str(trace.payload.get("exportMode", "sync")) + if view not in {"invocation", "execution"} or export_mode not in { + "sync", + "async-bounded-flush", + }: + raise ValueError("Unknown OTel view or exporter mode") + key = (view, export_mode) with _RUNTIME_LOCK: - if view not in _RUNTIMES: - _RUNTIMES[view] = Runtime(view) - runtime = _RUNTIMES[view] - token = _CURRENT_TRACE.set(trace) - runtime.exporter.active_trace = trace - before = otel_context.get_current() + if key not in _RUNTIMES: + _RUNTIMES[key] = Runtime(view, export_mode) + runtime = _RUNTIMES[key] + owner = runtime.tracker.owner(trace) + before, snapshot = otel_context.get_current(), _context_snapshot() + carrier = os.environ.get("_X_AMZN_TRACE_ID") trace.emit( "OTEL_ENTER", view=view, + export_mode=runtime.export_mode, provider=runtime.identity, - carrier=os.environ.get("_X_AMZN_TRACE_ID"), + carrier=carrier, carrier_source="runtime_environment", core_version=version("aws-durable-execution-sdk-python"), otel_version=version("aws-durable-execution-sdk-python-otel"), ) + runtime.tracker.observe( + owner, + "provider_observed", + createdAtMs=runtime.created_at_ms, + environmentId=trace.identity["environment"], + processId=trace.identity["process"], + ) + runtime.tracker.observe( + owner, + "otel_enter", + contextBefore=snapshot, + carrier=carrier, + carrierSource="process-environment", + ) try: - return invoke_handler([runtime.factory]) + with owned(owner): + return invoke_handler([runtime.factory]) finally: try: + # The SDK caller captures its real return before this diagnostic I/O. trace.emit( "OTEL_EXIT", view=view, @@ -170,5 +236,11 @@ def invoke( context_restored=otel_context.get_current() == before, exporter_shutdowns=runtime.exporter.shutdown_calls, ) + runtime.tracker.observe( + owner, + "otel_exit", + contextAfter=_context_snapshot(), + exporterShutdowns=runtime.exporter.shutdown_calls, + ) finally: - _CURRENT_TRACE.reset(token) + runtime.tracker.close_invocation(owner) diff --git a/lmi_tests/tests/e2e/test_shared_driver_local.py b/lmi_tests/tests/e2e/test_shared_driver_local.py new file mode 100644 index 000000000..bd81e8923 --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_driver_local.py @@ -0,0 +1,141 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Exercise shared phase routing against the real SDK's local durable runner. + +This checks public handler/callback/replay behavior, not real-LMI placement or +cloud conformance acceptance. +""" + +import json +from threading import Lock +import time + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests.fixture import workflow +from lmi_tests.shared_scenarios import ScenarioDriver + + +class LocalTrace: + def __init__(self): + self.events = [] + self.payload = {} + self.lock = Lock() + + def emit(self, phase, **fields): + with self.lock: + self.events.append({"phase": phase, "time": time.time(), **fields}) + + def body(self, name, value, step): + self.emit("BODY", operation=name, value=value, attempt=step.attempt) + return value + + def snapshot(self): + with self.lock: + return list(self.events) + + +class LocalService: + """Replace only AWS transport, keeping the actual fixture and SDK runner.""" + + def __init__(self): + self.manifest = {"driverTimeout": 15} + self.trace = LocalTrace() + self.invocations = [] + self.callback_records = [] + self.result = None + decorated = durable_execution( + lambda event, context: workflow(event, context, self.trace) + ) + + def handler(event, context): + self.trace.emit("WRAPPER_ENTER", request=context.aws_request_id) + result = decorated(event, context) + self.trace.emit( + "WRAPPER_RETURN", + request=context.aws_request_id, + status=result["Status"], + ) + return result + + self.runner = DurableFunctionTestRunner(handler, poll_interval=0.01) + + def start(self, fixture, **options): + payload = {"marker": options["marker"], "scenario": fixture} + arn = self.runner.run_async(input=payload) + item = {"arn": arn, "marker": payload["marker"], "runner": self.runner} + self.invocations.append(item) + return item + + def poll(self, predicate, seconds=15, **options): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + observed = predicate() + if observed: + return observed + time.sleep(0.01) + raise AssertionError(options.get("message", "local observation timeout")) + + def phase(self, item, phase, **options): + return self.poll( + lambda: [ + event for event in self.trace.snapshot() if event["phase"] == phase + ], + **options, + ) + + def for_item(self, item): + return self.trace.snapshot() + + def check_active(self): + pass + + def finish(self, item, status="SUCCEEDED"): + self.result = self.runner.wait_for_result(item["arn"], timeout=15) + assert self.result.status.value == status + + +def test_shared_callback_driver_resumes_the_original_public_sdk_execution(): + service = LocalService() + profile = Profile("native-1", "local-function", 1, 1, "local-sdk", "local-config") + context = ScenarioContext( + "local-run", + Scenario( + "callback-case", + "lmi-lifecycle-5", + profile.profile_id, + "replay-callback-retry", + roles=("victim", "resume"), + ), + profile, + ) + driver = ScenarioDriver(context, service) + with service.runner: + driver.start() + driver.place() + driver.execute() + assert len(service.invocations) == 1 + assert len(service.callback_records) == 1 + assert service.callback_records[0]["executionArn"] == service.invocations[0]["arn"] + assert json.loads(service.result.result) == service.invocations[0]["marker"] + events = service.trace.snapshot() + assert any( + event["phase"] == "WRAPPER_RETURN" and event["status"] == "PENDING" + for event in events + ) + # The local runner deliberately reuses its fake AWS request ID. Actual + # wrapper calls, PENDING results and skipped bodies establish local replay; + # this is not a claim about native LMI request identity or placement. + assert sum(event["phase"] == "WRAPPER_ENTER" for event in events) > 1 + bodies = [event for event in events if event["phase"] == "BODY"] + assert sum(event["operation"] == "success" for event in bodies) == 1 + assert sum(event["operation"] == "failure" for event in bodies) == 1 + assert [event["attempt"] for event in bodies if event["operation"] == "retry"] == [ + 1, + 2, + ] diff --git a/lmi_tests/tests/e2e/test_shared_evidence_local.py b/lmi_tests/tests/e2e/test_shared_evidence_local.py new file mode 100644 index 000000000..072ece0ef --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_evidence_local.py @@ -0,0 +1,100 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real SDK invocation/replay evidence reaches the shared validator unchanged.""" + +import json +import time +from pathlib import Path + +from aws_durable_execution_conformance_tests.lmi.app import validate +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.cloud import Cloud +from lmi_tests.shared_evidence import build_evidence +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( + monkeypatch, tmp_path +): + ledger = install_ledger(monkeypatch) + payload = shared_payload("replay") + payload["sharedScenario"] = "replay-callback-retry" + payload["behaviorBudgets"] = {"observationGraceMs": 1} + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + callback = runner.wait_for_callback(arn, name="completion", timeout=20) + runner.send_callback_success(callback, result=b"marker") + result = runner.wait_for_result(arn, timeout=20) + history = runner.get_execution_history( + arn, include_execution_data=True + ).to_dict()["Events"] + profile = Profile( + "c1", "local-function", 1, 1, "actual-local-sdk", "local-config", "reuse-worker" + ) + context = ScenarioContext( + "unit", + Scenario( + "case", + "lmi-lifecycle-5", + "c1", + "replay-callback-retry", + roles=("victim", "resume"), + coverage_profile="python-lifecycle", + behavior_budgets={"observationGraceMs": 1}, + ), + profile, + ) + raw = ledger.records() + entries = sorted( + (event for event in raw if event["phase"] == "WRAPPER_ENTER"), + key=lambda event: event["time"], + ) + roles = {} + for index, event in enumerate(entries): + role = "victim" if index == 0 else "resume" + roles[event["request"]] = role + context.record_attempt(event["request"], role=role, requestId=event["request"]) + cloud = object.__new__(Cloud) + cloud.manifest = { + "commit": "tested-commit", + "invocationTimeout": 30, + "conformanceRevision": "local-pinned-conformance", + } + cloud.events = {str(index): event for index, event in enumerate(raw)} + cloud.invocations = [ + { + "arn": arn, + "marker": "marker", + "role": "victim", + "payload": payload, + "native_requests": list(roles), + "trace_header": None, + } + ] + cloud.request_roles = roles + cloud.raw_histories = {arn: history} + cloud.execution_results = { + arn: {"Status": result.status.value, "Result": result.result} + } + cloud.control_records = [] + cloud.source_uris = {"memory://actual-sdk-ledger"} + time.sleep(0.002) + cloud.observation_ended_at = time.time() + cloud.retired_at = cloud.observation_ended_at + cloud.artifacts_dir = Path(tmp_path) + evidence = build_evidence(context, cloud, phase="final") + assert evidence["collectionErrors"] == [] + assert validate(evidence) == [] + bodies = [event for event in evidence["events"] if event["kind"] == "body"] + assert len([event for event in bodies if event["operation"] == "success"]) == 1 + # Losing a real source object must not turn into an inferred complete stream. + cloud.events.pop(next(iter(cloud.events))) + incomplete = build_evidence(context, cloud, phase="observations") + assert incomplete["collectionErrors"] + assert incomplete["collection"]["watermarks"] == {} diff --git a/lmi_tests/tests/e2e/test_shared_exports_local.py b/lmi_tests/tests/e2e/test_shared_exports_local.py new file mode 100644 index 000000000..faba730e8 --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_exports_local.py @@ -0,0 +1,239 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Actual fixture exports reach shared acceptance through the local service.""" + +import json +import time + +import pytest + +from aws_durable_execution_conformance_tests.lmi.app import validate +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture, telemetry +from lmi_tests.cloud import Cloud +from lmi_tests.shared_evidence import build_evidence +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload +from lmi_tests.tests.unit.test_shared_telemetry import HEADER + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) +@pytest.mark.parametrize( + "scenario", ["export-success", "export-failure-recovery", "held-export"] +) +def test_real_fixture_exports_satisfy_shared_mode_contract( + monkeypatch, tmp_path, view, mode, scenario +): + ledger = install_ledger(monkeypatch) + monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) + telemetry._RUNTIMES.clear() + factory = telemetry.Runtime + # Test-only processor configuration. The deployed fixture defaults to 5 s; + # its separately declared acceptance budget is 10 s. + monkeypatch.setattr( + telemetry, + "Runtime", + lambda selected_view, selected_mode="sync": factory( + selected_view, selected_mode, flush_timeout_ms=20 + ), + ) + budgets = {"exportCompletionMs": 5000, "observationGraceMs": 1} + if mode == "async-bounded-flush": + budgets["flushWaitMs"] = 100 + roles = ("victim", "warm") if scenario == "export-failure-recovery" else ("victim",) + context = ScenarioContext( + "unit", + Scenario( + "case", + "lmi-lifecycle-16", + "c1", + scenario, + roles=roles, + otel_view=view, + coverage_profile="export-lifecycle", + export_mode=mode, + behavior_budgets=budgets, + ), + Profile("c1", "local-function", 1, 1, "actual-local-sdk", "local-config"), + ) + cloud = object.__new__(Cloud) + cloud.manifest = { + "commit": "tested-commit", + "conformanceRevision": "local-pinned-conformance", + "invocationTimeout": 30, + "wheels": { + "aws_durable_execution_sdk_python_otel-local.whl": "local-plugin-source" + }, + } + cloud.invocations, cloud.control_records = [], [] + cloud.request_roles, cloud.raw_histories, cloud.execution_results = {}, {}, {} + cloud.artifacts_dir = tmp_path + cloud._save = lambda name, data: (tmp_path / name).write_text(json.dumps(data)) + held_count = 0 + try: + with DurableFunctionTestRunner( + fixture.handler, poll_interval=0.01, skip_time=False + ) as runner: + for role in roles: + payload = shared_payload("success") + payload.update( + marker=role, + role=role, + coverageProfile="export-lifecycle", + sharedScenario=scenario, + otelView=view, + exportMode=mode, + behaviorBudgets=budgets, + otel_export="held" + if scenario == "held-export" + else "failure" + if scenario == "export-failure-recovery" and role == "victim" + else "success", + otel_export_gate="export-gate", + ) + prefix = payload["controlPrefix"] + for gate in ("release-all", "export-gate"): + if prefix + gate not in ledger.objects: + started = time.time() + ledger.put_object( + Bucket="private-evidence", Key=prefix + gate, Body=b"hold" + ) + cloud.control_records.append( + { + "gate": gate, + "action": "hold", + "started": started, + "time": time.time(), + } + ) + context.record_attempt(role, role=role) + arn = runner.run_async( + input=payload, execution_name=role, execution_timeout=15 + ) + try: + if scenario == "held-export": + ledger.wait_for("blocked", timeout=3) + if mode == "async-bounded-flush": + waited = ledger.wait_for("flush_wait_completed", timeout=3) + assert waited["observation"]["outcome"] == "timeout" + ledger.wait_for("wrapper_return", timeout=3) + assert not any( + record["observation"]["kind"] == "export_completed" + for record in ledger.observations() + ) + else: + assert not any( + record["observation"]["kind"] == "wrapper_return" + for record in ledger.observations() + ) + finally: + if scenario == "held-export": + started = time.time() + ledger.put_object( + Bucket="private-evidence", + Key=prefix + "export-gate", + Body=b"release", + ) + cloud.control_records.append( + { + "gate": "export-gate", + "action": "release", + "started": started, + "time": time.time(), + } + ) + result = runner.wait_for_result(arn, timeout=10) + assert result.status.value == "SUCCEEDED" + history = runner.get_execution_history( + arn, include_execution_data=True + ).to_dict()["Events"] + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + complete = [ + record + for record in ledger.observations() + if record["execution"] == arn + and record["observation"]["kind"] == "export_quiescence" + ] + if complete: + break + time.sleep(0.001) + assert complete, "No observed export/publication quiescence" + entry = next( + record + for record in ledger.records() + if record["execution"] == arn and record["phase"] == "WRAPPER_ENTER" + ) + request = entry["request"] + context.update_attempt(role, requestId=request, executionArn=arn) + cloud.request_roles[request] = role + cloud.invocations.append( + { + "marker": role, + "arn": arn, + "payload": payload, + "role": role, + "trace_header": HEADER, + "native_requests": [request], + } + ) + cloud.raw_histories[arn] = history + cloud.execution_results[arn] = { + "Status": result.execution_status.value, + "Result": result.result, + } + held_count += sum( + record["execution"] == arn + and record["observation"]["kind"] == "export_held" + for record in ledger.observations() + ) + for gate in ("release-all", "export-gate"): + started = time.time() + ledger.put_object( + Bucket="private-evidence", Key=prefix + gate, Body=b"release" + ) + cloud.control_records.append( + { + "gate": gate, + "action": "release", + "started": started, + "time": time.time(), + } + ) + raw = ledger.records() + cloud.events = {str(index): record for index, record in enumerate(raw)} + cloud.source_uris = { + "memory://actual-sdk-ledger/" + key + for key in ledger.objects + if "/events/" in key + } + (tmp_path / "events.json").write_text(json.dumps(raw)) + time.sleep(0.002) + cloud.observation_ended_at = time.time() + cloud.retired_at = time.time() + evidence = build_evidence(context, cloud, phase="final") + assert evidence["collectionErrors"] == [] + assert validate(evidence) == [] + if scenario == "held-export": + # Later legitimate exports happen after the gate has opened. They + # stay in evidence without being falsely labeled as held work. + assert held_count == 1 + assert ( + len( + [ + event + for event in evidence["events"] + if event["kind"] == "export_started" + ] + ) + > held_count + ) + finally: + for runtime in telemetry._RUNTIMES.values(): + runtime.provider.shutdown() + telemetry._RUNTIMES.clear() diff --git a/lmi_tests/tests/e2e/test_shared_observations_local.py b/lmi_tests/tests/e2e/test_shared_observations_local.py new file mode 100644 index 000000000..5f48e9a8d --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_observations_local.py @@ -0,0 +1,171 @@ +"""Run the actual Lambda fixture through the public in-process SDK service.""" + +from __future__ import annotations + +import json +from collections import Counter + +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def values(ledger, kind): + return [ + record["observation"] + for record in ledger.observations() + if record["observation"]["kind"] == kind + ] + + +def test_actual_fixture_public_hooks_bind_bodies_checkpoints_and_root_lifetime( + monkeypatch, +): + ledger = install_ledger(monkeypatch) + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(shared_payload()), execution_timeout=30) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + body = values(ledger, "body") + assert ( + len(body) == 1 + and body[0]["operationId"] == result.get_step("success").operation_id + ) + assert body[0]["parentId"] == result.get_step("success").parent_id + assert body[0]["operationId"] in { + operation + for ack in values(ledger, "checkpoint_ack") + for operation in ack["operationIds"] + } + assert {item["callId"] for item in values(ledger, "checkpoint_call")} == { + item["callId"] for item in values(ledger, "checkpoint_exit") + } + assert {item["taskId"] for item in values(ledger, "task_start")} == { + item["taskId"] for item in values(ledger, "task_exit") + } + assert ( + next(item for item in values(ledger, "task_exit") if item["taskId"] == "root")[ + "atMs" + ] + <= values(ledger, "wrapper_return")[0]["atMs"] + ) + + +def test_actual_service_replay_keeps_operation_identity_and_skips_completed_bodies( + monkeypatch, +): + ledger = install_ledger(monkeypatch) + payload = shared_payload("replay") + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + callback = runner.wait_for_callback(arn, name="completion", timeout=20) + runner.send_callback_success(callback, result=b"marker") + result = runner.wait_for_result(arn, timeout=20) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + bodies = values(ledger, "body") + assert Counter(item["operation"] for item in bodies) == { + "success": 1, + "failure": 1, + "retry": 2, + } + assert { + item["operationId"] for item in bodies if item["operation"] == "success" + } == {result.get_step("success").operation_id} + starts = values(ledger, "invocation_start") + assert len(starts) >= 2 and any(not item["isFirstInvocation"] for item in starts) + replay = next(item for item in starts if not item["isFirstInvocation"]) + assert any( + item["operationId"] == result.get_step("success").operation_id + and item["status"] == "SUCCEEDED" + for item in replay["operations"] + ) + assert "PENDING" in {item["outcome"] for item in values(ledger, "wrapper_return")} + + +def test_actual_checkpoint_response_settles_before_join_observation(monkeypatch): + ledger = install_ledger(monkeypatch) + payload = shared_payload("checkpoint") + prefix = payload["controlPrefix"] + ledger.objects[prefix + "marker-checkpoint"] = b"hold" + ledger.objects[prefix + "release-all"] = b"hold" + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + held = ledger.wait_for("ack_held")["observation"] + ledger.wait_for("blocked") + assert not [ + item + for item in values(ledger, "checkpoint_exit") + if item["callId"] == held["callId"] + ] + assert values(ledger, "branch_join") == [] + ledger.objects[prefix + "marker-checkpoint"] = b"release" + result = runner.wait_for_result(arn, timeout=20) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + exited = next( + item + for item in values(ledger, "checkpoint_exit") + if item["callId"] == held["callId"] + ) + joined = values(ledger, "branch_join")[0] + assert held["operationId"] in { + op.operation_id + for op in result.get_context("checkpoint-pool") + .child_operations[0] + .child_operations + } + assert joined["scopeId"] == result.get_context("checkpoint-pool").operation_id + assert ( + exited["atMs"] <= joined["atMs"] <= values(ledger, "wrapper_return")[0]["atMs"] + ) + + +def test_actual_nested_public_calls_supply_configuration_ids(monkeypatch): + ledger = install_ledger(monkeypatch) + payload = shared_payload("nested-progress") + payload["gate"] = "entry" + ledger.objects[payload["controlPrefix"] + "entry"] = b"release" + ledger.objects[payload["controlPrefix"] + "release-all"] = b"hold" + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(payload), execution_timeout=30) + assert result.result == '"marker"' + assert values(ledger, "observation_error") == [] + configured = values(ledger, "operation_config") + assert len(configured) == 2 and all( + item["maxConcurrency"] == 1 for item in configured + ) + started = {item["operationId"]: item for item in values(ledger, "operation_start")} + assert {started[item["operationId"]]["name"] for item in configured} == { + "inner-map", + "outer-parallel", + } + assert values(ledger, "progress")[0]["value"] == [ + ["marker:0", "marker:1"], + "marker", + ] + + +def test_actual_parent_closure_is_captured_before_late_child_attempt(monkeypatch): + ledger = install_ledger(monkeypatch) + payload = shared_payload("late-operation") + prefix = payload["controlPrefix"] + ledger.objects[prefix + "marker-loser"] = b"hold" + ledger.objects[prefix + "marker-loser-started"] = b"hold" + ledger.objects[prefix + "release-all"] = b"hold" + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) + winner = ledger.wait_for("winner_selected")["observation"] + ledger.objects[prefix + "marker-loser"] = b"release" + runner.wait_for_result(arn, timeout=20) + assert values(ledger, "observation_error") == [] + attempted = values(ledger, "late_attempt")[0] + closed = next( + item + for item in values(ledger, "scope_closed") + if item["scopeId"] == attempted["scopeId"] + ) + assert closed["atMs"] <= winner["atMs"] < attempted["atMs"] + # Observe the actual SDK result, without fixing or hiding a lifecycle defect. + outcomes = values(ledger, "late_rejected") + values(ledger, "late_accepted") + assert len(outcomes) == 1 and outcomes[0]["scopeId"] == attempted["scopeId"] diff --git a/lmi_tests/tests/e2e/test_shared_telemetry_local.py b/lmi_tests/tests/e2e/test_shared_telemetry_local.py new file mode 100644 index 000000000..912f18a85 --- /dev/null +++ b/lmi_tests/tests/e2e/test_shared_telemetry_local.py @@ -0,0 +1,112 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Public SDK/plugin integration on the local durable service, without AWS.""" + +from dataclasses import replace +import time + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.config import Duration +from aws_durable_execution_sdk_python.execution import DurableExecutionInvocationInput +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import telemetry +from lmi_tests.shared_telemetry import collect_telemetry +from lmi_tests.tests.unit.test_shared_telemetry import ( + HEADER, + SharedTrace, + attempt, + observations, + wait_quiescence, +) + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("export_mode", ["sync", "async-bounded-flush"]) +def test_public_local_runner_replay_preserves_export_ownership( + monkeypatch, view, export_mode +): + monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) + monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) + traces, bodies = [], [] + telemetry._RUNTIMES.clear() + + def business(event, context): + value = context.step( + lambda _: bodies.append(event["marker"]) or event["marker"], name="saved" + ) + context.wait(Duration.from_seconds(1), name="pause") + return value + + def entry(event, context): + invocation = ( + event + if hasattr(event, "durable_execution_arn") + else DurableExecutionInvocationInput.from_json_dict(event) + ) + # The local runtime context is a test double. Give each simulated native + # request its own ID before entering the actual public SDK handler. + context = replace(context, aws_request_id="local-" + str(len(traces))) + trace = SharedTrace( + "replay", view=view, mode=export_mode, profile="python-lifecycle" + ) + trace.identity["request"] = context.aws_request_id + trace.identity["execution"] = invocation.durable_execution_arn + traces.append(trace) + trace.emit("WRAPPER_ENTER") + + def call(plugins): + try: + return durable_execution(business, plugins=plugins)(event, context) + finally: + trace.sdk_completed_at = time.time() + + result = telemetry.invoke(trace, call) + trace.emit( + "WRAPPER_RETURN", time=trace.sdk_completed_at, status=result["Status"] + ) + return result + + try: + with DurableFunctionTestRunner( + entry, poll_interval=0.01, skip_time=False + ) as runner: + execution = runner.run_async( + input={"marker": "saved"}, execution_timeout=15 + ) + result = runner.wait_for_result(execution, timeout=15) + history = runner.get_execution_history( + execution, include_execution_data=True + ) + assert result.status.value == "SUCCEEDED" + assert bodies == ["saved"] + assert len(traces) >= 2 + assert len(history.events) > 1 + for trace in traces: + wait_quiescence(trace) + assert ( + observations(trace, "otel_factory")[0]["pluginExecutionArn"] + == execution + ) + raw = [record for trace in traces for record in trace.events] + data = collect_telemetry( + raw, + [attempt(trace) for trace in traces], + view=view, + export_mode=export_mode, + expected_sampled=True, + caller_headers={trace.identity["request"]: HEADER for trace in traces}, + observed_until_ms=time.time_ns() / 1_000_000, + ) + assert data["views"][view]["collection"]["status"] == "complete" + spans = data["views"][view]["spans"] + assert sum(record["span"]["name"] == "Invocation" for record in spans) == len( + traces + ) + assert sum(record["span"]["name"] == "Workflow" for record in spans) == 1 + assert len({binding["providerId"] for binding in data["providerBindings"]}) == 1 + finally: + for runtime in telemetry._RUNTIMES.values(): + runtime.provider.shutdown() + telemetry._RUNTIMES.clear() diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py index 5b6e61dfc..d67394c95 100644 --- a/lmi_tests/tests/unit/test_control.py +++ b/lmi_tests/tests/unit/test_control.py @@ -22,6 +22,7 @@ @pytest.fixture def trace(): value = Trace.__new__(Trace) + value.capture_lock, value.capture_sequence = threading.Lock(), 0 value.s3 = boto3.client( "s3", region_name="us-west-2", @@ -30,6 +31,9 @@ def trace(): ) value.bucket = "test-bucket" value.identity = {"run": "unit"} + value.payload = {"scenario": "control"} + value.shared = False + value.control_prefix = "runs/unit/control/" value.emit = Mock() return value @@ -79,7 +83,12 @@ def test_fault_gate_outlives_invocation_deadline_but_remains_bounded( monkeypatch.setattr( fixture, "time", - SimpleNamespace(monotonic=lambda: next(times), sleep=lambda _: None), + SimpleNamespace( + monotonic=lambda: next(times), + sleep=lambda _: None, + time=time.time, + time_ns=time.time_ns, + ), ) if escaped: with pytest.raises(TimeoutError, match="emergency release"): @@ -159,10 +168,13 @@ def get(**kwargs): return {"Body": io.BytesIO(objects[kwargs["Key"]])} trace = Trace.__new__(Trace) + trace.capture_lock, trace.capture_sequence = threading.Lock(), 0 trace.s3, trace.bucket = Mock(), "test-bucket" trace.s3.put_object.side_effect, trace.s3.get_object.side_effect = put, get trace.identity = {"run": "unit", "marker": "local", "request": "request"} trace.lock, trace.sequence, trace.deadline = threading.Lock(), 0, time.time() + 30 + trace.payload, trace.shared = {"scenario": scenario}, False + trace.control_prefix = "runs/unit/control/" handler = durable_execution( lambda _e, c: workflow({"marker": "local", "scenario": scenario}, c, trace) ) diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index 2ae939f0a..daeef0a65 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -18,7 +18,9 @@ def manifest(): "stack": "python-lmi-e2e", "bucket": "test-bucket", "role": "role", - "provider": "provider", + "provider": "arn:aws:lambda:us-west-2:123456789012:capacity-provider:provider", + "region": "us-west-2", + "account": "123456789012", "runtime": "python3.14", "concurrencies": deploy.CONCURRENCIES, "functions": {}, @@ -84,6 +86,8 @@ def test_verify_rejects_standard_lambda_wrong_artifact_and_unbounded_placement( "Timeout", "DurableConfig", "Environment", + "Role", + "Handler", ): bad = copy.deepcopy(config) del bad[field] @@ -134,6 +138,25 @@ def apis(monkeypatch, tmp_path): clients = {name: Mock() for name in ("cloudformation", "s3", "lambda")} monkeypatch.setattr(deploy, "client", clients.__getitem__) monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) + from lmi_tests.tests.unit.test_shared_lock import Store + + lease_store = Store() + clients["s3"].get_public_access_block.return_value = { + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": True, + "IgnorePublicAcls": True, + "BlockPublicPolicy": True, + "RestrictPublicBuckets": True, + } + } + clients["s3"].get_object.side_effect = lease_store.get_object + + def store(**kwargs): + if kwargs["Key"].startswith("ownership/"): + return lease_store.put_object(**kwargs) + return {} + + clients["s3"].put_object.side_effect = store return clients @@ -150,6 +173,22 @@ def test_later_run_updates_fixed_functions_without_recreating_resources( verify = Mock() monkeypatch.setattr(deploy, "verify", verify) deploy.deploy_stack(manifest, b"first-code") + # No SDK work is launched by this deployment-only test. Retire its ownership + # window before testing a second revision against the same persistent targets. + from lmi_tests.shared_lock import RunLease + + record = manifest["sharedLease"] + lease = RunLease( + record["bucket"], + record["key"], + record["runId"], + record["token"], + tuple(record["resources"]), + apis["s3"], + ) + for resource in lease.resources: + lease.finish_profile(resource, True) + lease.release() second = { **manifest, "run": "second", @@ -273,7 +312,10 @@ def test_busy_or_incompletely_listed_functions_block_updates( with pytest.raises(ProvisioningError, match="code was not updated"): deploy.deploy_stack(manifest, b"code") cfn.update_stack.assert_not_called() - apis["s3"].put_object.assert_not_called() + assert not any( + call.kwargs["Key"] == manifest["codeKey"] + for call in apis["s3"].put_object.call_args_list + ) apis["lambda"].stop_durable_execution.assert_not_called() diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py index 68e6ed300..999971ccf 100644 --- a/lmi_tests/tests/unit/test_java_parity.py +++ b/lmi_tests/tests/unit/test_java_parity.py @@ -46,8 +46,11 @@ def snapshot(self): def trace(self, marker, scenario): trace = Trace.__new__(Trace) + trace.capture_lock, trace.capture_sequence = threading.Lock(), 0 trace.s3, trace.bucket = self, "unit" trace.payload = {"marker": marker, "scenario": scenario} + trace.shared = False + trace.control_prefix = "runs/unit/control/" trace.identity = { "run": "unit", "marker": marker, @@ -186,6 +189,9 @@ def run_shared_roots(nested): rendezvous = threading.Barrier(2, timeout=3) class QuietTrace: + def __init__(self): + self.payload = {} + def emit(self, *_args, **_kwargs): pass diff --git a/lmi_tests/tests/unit/test_shared_adapter.py b/lmi_tests/tests/unit/test_shared_adapter.py new file mode 100644 index 000000000..0d955e6da --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_adapter.py @@ -0,0 +1,752 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ownership, native-request journaling and cancellation of the shared driver.""" + +import asyncio +import json +from pathlib import Path +from threading import Event +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from lmi_tests import cloud as cloud_module +from lmi_tests import shared_adapter as adapter_module +from lmi_tests.evidence import CollectionError, PlacementError, ProvisioningError +from lmi_tests.shared_scenarios import ScenarioDriver + + +FUNCTION = "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" +EXECUTION = FUNCTION + "/durable-execution/name/id" +PROFILE = Profile("native-1", FUNCTION, 1, 1, "sdk-sha", "configuration-sha") + + +def context( + case_id="case", + scenario="replay-callback-retry", + roles=("victim", "resume"), + *, + native=1, + view=None, + mode=None, +): + profile = Profile( + f"native-{native}", FUNCTION, native, native, "sdk-sha", "configuration-sha" + ) + return ScenarioContext( + "run", + Scenario( + case_id, + "lmi-lifecycle-5", + profile.profile_id, + scenario, + roles=roles, + otel_view=view, + export_mode=mode, + coverage_profile="export-lifecycle" if mode else "python-lifecycle", + ), + profile, + ) + + +def deployment(): + return { + "functions": {"c1": FUNCTION, "c2": FUNCTION + "-c2"}, + "concurrencies": {"c1": 1, "c2": 2}, + "region": "us-west-2", + "run": "run", + "bucket": "private-evidence", + "commit": "sdk-sha", + "runtime": "python3.14", + "role": "role", + "provider": "provider", + "codeSha256": "code", + "invocationTimeout": 60, + "executionTimeout": 240, + "driverTimeout": 1, + } + + +@pytest.fixture +def cloud(monkeypatch, tmp_path): + clients = {name: Mock() for name in ("lambda", "s3", "logs")} + clients["lambda"].invoke.return_value = { + "StatusCode": 202, + "DurableExecutionArn": EXECUTION, + } + clients["s3"].get_paginator.return_value.paginate.return_value = [] + monkeypatch.setattr(cloud_module, "client", lambda service: clients[service]) + monkeypatch.setattr( + "aws_durable_execution_sdk_python_testing.runner.boto3.client", + lambda *a, **kw: clients["lambda"], + ) + return cloud_module.Cloud( + deployment(), "c1", artifacts_dir=tmp_path / "case", case_context=context() + ) + + +def enter(item, request, at): + return { + "phase": "WRAPPER_ENTER", + "marker": item["marker"], + "request": request, + "execution": EXECUTION, + "time": at, + "sequence": 0, + "run": "run", + "commit": "sdk-sha", + "runtime": "python3.14", + "role": "role", + "provider": "provider", + "codeSha256": "code", + "invocationTimeout": 60, + "executionTimeout": 240, + } + + +def refresh_entries(cloud, *events): + cloud.events.update({f"events/{event['request']}": event for event in events}) + return cloud.refresh() + + +def test_direct_launch_is_journaled_before_public_runner_transport(cloud): + observed = [] + + def invoke(**kwargs): + observed.append(kwargs) + assert len(cloud.case_context.attempts) == 1 + assert "requestId" not in cloud.case_context.attempts[0] + return {"StatusCode": 202, "DurableExecutionArn": EXECUTION} + + cloud.lam.invoke.side_effect = invoke + item = cloud.start( + "replay", + role="victim", + resume_roles=("resume",), + dispatcher_payload=cloud.case_context.dispatcher("victim"), + ) + payload = json.loads(observed[0]["Payload"]) + assert observed[0]["FunctionName"] == FUNCTION + assert payload["scenario"] == "replay" + assert payload["sharedScenario"] == "replay-callback-retry" + assert payload["caseId"] == "case" + assert payload["controlPrefix"] == cloud.control_prefix + assert "/cases/" in cloud.control_prefix + assert item["native_requests"] == [] + refresh_entries(cloud, enter(item, "original", 1)) + assert cloud.case_context.attempts[0]["requestId"] == "original" + + +def test_service_replays_are_journaled_without_another_invoke(cloud): + item = cloud.start("replay", role="victim", resume_roles=("resume",)) + refresh_entries( + cloud, + enter(item, "first", 1), + enter(item, "second", 2), + enter(item, "third", 3), + ) + assert cloud.lam.invoke.call_count == 1 + assert [ + (attempt["role"], attempt["attempt"], attempt["requestId"]) + for attempt in cloud.case_context.attempts + ] == [("victim", 1, "first"), ("resume", 1, "second"), ("resume", 2, "third")] + assert cloud.request_roles == { + "first": "victim", + "second": "resume", + "third": "resume", + } + cloud.refresh() + assert len(cloud.case_context.attempts) == 3 + + +def test_lost_invoke_response_retains_launch_and_recovers_identity(cloud): + cloud.lam.invoke.side_effect = TimeoutError("response lost") + with pytest.raises(CollectionError, match="API failed"): + cloud.start("success", role="victim") + item = cloud.invocations[0] + assert item["arn"] is None + record = json.loads( + (cloud.artifacts_dir / f"invocations/{item['marker']}.json").read_text() + ) + assert "response lost" in record["error"] + refresh_entries(cloud, enter(item, "original", 1)) + assert item["arn"] == EXECUTION + assert cloud.case_context.attempts[0]["requestId"] == "original" + assert cloud.lam.invoke.call_count == 1 + + +def test_replacement_execution_cannot_rebind_a_launched_marker(cloud): + item = cloud.start("success", role="victim") + event = enter(item, "replacement", 1) + event["execution"] = EXECUTION + "-other" + with pytest.raises(CollectionError, match="original durable execution"): + refresh_entries(cloud, event) + + +def test_case_controls_and_collection_cannot_touch_another_case(cloud, tmp_path): + cloud.hold("own-gate") + with pytest.raises(CollectionError, match="not owned"): + cloud.release("another-gate") + with pytest.raises(CollectionError, match="another case"): + cloud.refresh(markers=["another-marker"]) + with pytest.raises(CollectionError, match="another case"): + cloud.refresh(markers=[]) + with pytest.raises(CollectionError, match="only its own"): + cloud.collect(full_run=True) + cloud.release_all() + keys = [call.kwargs["Key"] for call in cloud.s3.put_object.call_args_list] + assert all(key.startswith(cloud.control_prefix) for key in keys) + assert not (tmp_path / "events.json").exists() + + +def test_unlaunched_case_does_not_scan_shared_event_prefix(cloud): + assert cloud.refresh() == [] + cloud.s3.get_paginator.assert_not_called() + + +def test_cancellation_stops_waits_and_new_transports(cloud): + cloud.cancel() + with pytest.raises(CollectionError, match="cancelled"): + cloud.wait(30) + with pytest.raises(CollectionError, match="cancelled"): + cloud.start("success", role="victim") + cloud.lam.invoke.assert_not_called() + cloud.begin_retirement() + with pytest.raises(CollectionError, match="retiring"): + cloud.start("success", role="victim") + + +class FakeCloud: + instances = [] + + def __init__(self, manifest, configuration, *, artifacts_dir, case_context=None): + self.manifest, self.configuration = manifest, configuration + self.artifacts_dir, self.case_context = Path(artifacts_dir), case_context + self.cancel_event = Event() + self.actions = [] + self.instances.append(self) + + def verify(self): + self.actions.append("verify") + + def check_active(self): + if self.cancel_event.is_set(): + raise CollectionError("cancelled") + + def cancel(self): + self.cancel_event.set() + + def begin_retirement(self): + self.actions.append("begin-retirement") + self.cancel_event.clear() + + def release_all(self): + self.actions.append("release") + + +def adapter(tmp_path, *, factory=FakeCloud, ownership=None): + return adapter_module.SharedAdapter( + {"sdk": {"deployment": deployment(), "artifacts_dir": str(tmp_path)}}, + ownership=ownership or Mock(spec=["assert_owned", "finish_profile", "release"]), + evidence_builder=lambda context, cloud, *, phase: { + "retirement": {"owner": context.scenario.case_id, "phase": phase} + }, + cloud_factory=factory, + ) + + +def test_prepare_checks_persistent_ownership_and_actual_profile(tmp_path): + owner = Mock(spec=["assert_owned", "finish_profile", "release"]) + value = adapter(tmp_path, ownership=owner) + from lmi_tests.shared_contract import configuration_identity + + valid = Profile( + PROFILE.profile_id, + PROFILE.resource_id, + 1, + 1, + "sdk-sha", + configuration_identity(deployment(), "c1"), + ) + asyncio.run(value.prepare(valid)) + with pytest.raises(ProvisioningError, match="configuration identity"): + asyncio.run(value.prepare(PROFILE)) + assert all(call.args == (FUNCTION,) for call in owner.assert_owned.call_args_list) + assert FakeCloud.instances[-1].actions == ["verify"] + bad = Profile("bad", FUNCTION, 1, 2, "sdk-sha", "configuration-sha") + with pytest.raises(ProvisioningError, match="Native concurrency"): + asyncio.run(value.prepare(bad)) + + +def test_lost_ownership_prevents_any_driver_launch(tmp_path): + owner = Mock(spec=["assert_owned", "finish_profile", "release"]) + owner.assert_owned.side_effect = RuntimeError("lease lost") + value = adapter(tmp_path, ownership=owner) + with pytest.raises(RuntimeError, match="lease lost"): + asyncio.run(value.start(context())) + assert value.cases == {} + + +def test_cancelled_worker_is_joined_before_case_cleanup(monkeypatch, tmp_path): + entered, unblock, exited = Event(), Event(), Event() + + class Driver: + def __init__(self, context, cloud, **kwargs): + self.cloud = cloud + + def start(self): + entered.set() + unblock.wait(2) + try: + self.cloud.check_active() + finally: + exited.set() + + def retire(self): + assert exited.is_set() + return True + + monkeypatch.setattr(adapter_module, "ScenarioDriver", Driver) + + async def exercise(): + value = adapter(tmp_path) + ctx = context() + starting = asyncio.create_task(value.start(ctx)) + assert await asyncio.to_thread(entered.wait, 1) + starting.cancel() + with pytest.raises(asyncio.CancelledError): + await starting + releasing = asyncio.create_task(value.release(ctx)) + await asyncio.sleep(0) + assert "release" not in value.cases[ctx.scenario.case_id].driver.cloud.actions + unblock.set() + await releasing + assert exited.is_set() + assert await value.retire(ctx) + assert ctx.evidence["retirement"] == {"owner": "case", "phase": "final"} + + asyncio.run(exercise()) + + +def test_finish_records_quarantine_without_releasing_the_whole_run(tmp_path): + owner = Mock(spec=["assert_owned", "finish_profile", "release"]) + value = adapter(tmp_path, ownership=owner) + asyncio.run(value.finish(PROFILE, quiescent=False)) + owner.finish_profile.assert_called_once_with(FUNCTION, False) + assert not owner.release.called + + +class ScriptCloud: + def __init__(self, native=1): + self.concurrency = native + self.manifest = {"driverTimeout": 1, "invocationTimeout": 60} + self.actions = [] + self.callback_records = [] + self.invocations = [] + self.raw_histories = {} + self.source_uris = set() + + def record_deadline_measurement(self, snapshot): + self.actions.append(("measurement", snapshot)) + + def wait_terminal(self, item): + self.actions.append(("terminal", item["marker"])) + return {"Status": "FAILED"} + + def refresh(self, **kwargs): + return [event for item in self.invocations for event in self.for_item(item)] + + def start(self, fixture, **kwargs): + item = { + "marker": kwargs["role"], + "arn": f"execution:{kwargs['role']}", + "runner": Mock(), + } + item["runner"].wait_for_callback.return_value = "callback-id" + self.actions.append(("start", fixture, kwargs)) + self.invocations.append(item) + return item + + def phase(self, item, phase, **kwargs): + self.actions.append(("phase", item["marker"], phase)) + return [ + { + "phase": phase, + "time": 0, + "deadline": 0, + "environment": "environment", + "request": "request", + "status": "PENDING", + } + ] + + def poll(self, predicate, **kwargs): + self.actions.append(("poll", kwargs.get("message"))) + return predicate() or [ + {"phase": "PLACEMENT_ACCEPTED", "environment": "environment"} + ] + + def release(self, gate): + self.actions.append(("release", gate)) + + def finish(self, item, status="SUCCEEDED"): + self.actions.append(("finish", item["marker"], status)) + + def for_item(self, item): + return [ + { + "phase": "WRAPPER_RETURN", + "status": "PENDING", + "time": 1, + "request": "first", + }, + {"phase": "WRAPPER_ENTER", "request": "first"}, + {"phase": "WRAPPER_ENTER", "request": "second"}, + { + "phase": "CONFORMANCE", + "observation": {"kind": "flush_wait_completed", "outcome": "timeout"}, + }, + {"phase": "CONFORMANCE", "observation": {"kind": "export_completed"}}, + {"phase": "LATE_REJECTED"}, + {"phase": "ALIVE", "time": 2}, + ] + + def platform_timeout(self, item, request): + self.actions.append(("platform-timeout", item["arn"])) + + def history(self, item): + self.actions.append(("history", item["arn"])) + + def check_active(self): + pass + + +def test_callback_uses_same_execution_and_actual_pending_before_response(): + cloud = ScriptCloud() + driver = ScenarioDriver(context(), cloud) + driver.start() + driver.place() + driver.execute() + assert len(cloud.invocations) == 1 + runner = cloud.invocations[0]["runner"] + runner.wait_for_callback.assert_called_once_with( + "execution:victim", name="completion", timeout=1 + ) + runner.send_callback_success.assert_called_once_with( + "callback-id", result=b"victim" + ) + assert cloud.callback_records[0]["executionArn"] == "execution:victim" + + +@pytest.mark.parametrize( + "scenario,fixture", + [ + ("early-completion-parallel", "parallel"), + ("early-completion-map", "map"), + ("early-completion-nested", "nested"), + ], +) +def test_early_completion_observations_do_not_release_losing_fault(scenario, fixture): + cloud = ScriptCloud() + driver = ScenarioDriver(context(scenario=scenario, roles=("victim",)), cloud) + driver.start() + driver.place() + driver.execute() + assert cloud.actions[0][1] == fixture + assert not any(action[0] == "release" for action in cloud.actions) + + +@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) +def test_held_async_export_observes_real_timeout_and_wrapper_before_release(mode): + cloud = ScriptCloud() + driver = ScenarioDriver( + context( + scenario="held-export", roles=("victim",), view="invocation", mode=mode + ), + cloud, + ) + driver.start() + driver.place() + driver.execute() + released = next( + index for index, action in enumerate(cloud.actions) if action[0] == "release" + ) + preceding = cloud.actions[:released] + assert ("phase", "victim", "OTEL_EXPORT_HELD") in preceding + assert (("phase", "victim", "WRAPPER_RETURN") in preceding) == ( + mode == "async-bounded-flush" + ) + assert cloud.actions[0][2]["otel_export"] == "held" + + +def test_missed_original_placement_is_not_retried_as_another_victim(): + cloud = ScriptCloud(2) + cloud.poll = lambda *args, **kwargs: [{"phase": "PLACEMENT_MISS"}] + driver = ScenarioDriver( + context(scenario="return-inflight", roles=("victim", "peer"), native=2), cloud + ) + driver.start() + with pytest.raises(PlacementError, match="Original victim"): + driver.place() + assert len(cloud.invocations) == 2 + + +@pytest.mark.parametrize( + "scenario,count", [("warm-lifecycle", 9), ("otel-warm-lifecycle", 6)] +) +def test_warm_sequence_uses_real_callback_resumes_not_extra_driver_invocations( + scenario, count +): + from aws_durable_execution_conformance_tests.lmi.matrix import required_cases + + cell = next( + case + for case in required_cases("python", "lifecycle-otel", "python-lifecycle") + if case.scenario == scenario and case.native_concurrency == 1 + ) + cloud = ScriptCloud() + driver = ScenarioDriver( + context(scenario=scenario, roles=cell.roles, view=cell.otel_view), cloud + ) + driver.start() + driver.place() + driver.execute() + launches = [action for action in cloud.actions if action[0] == "start"] + assert len(launches) == count + assert [action[1] for action in launches] == ["success", "failure", "replay"] * ( + count // 3 + ) + assert len(cloud.callback_records) == count // 3 + assert len({item["arn"] for item in cloud.invocations}) == count + + +@pytest.mark.parametrize("native", [1, 2]) +def test_deadline_recovery_demands_original_environment_before_fault_release(native): + roles = ( + ("victim", "recovery") + if native == 1 + else ("victim", "peer", "recovery", "recovery-peer") + ) + cloud = ScriptCloud(native) + driver = ScenarioDriver( + context(scenario="deadline-recovery", roles=roles, native=native), cloud + ) + driver.wait_overlap = Mock(return_value=[]) + for role in roles: + driver.context.record_attempt( + role, role=role, requestId="request" if role == "victim" else role + ) + driver.start() + driver.place() + driver.execute() + expected = ("recovery",) if native == 1 else ("recovery", "recovery-peer") + driver.wait_overlap.assert_called_with(expected, "environment") + measured = next( + index + for index, action in enumerate(cloud.actions) + if action[0] == "measurement" + ) + released = next( + index + for index, action in enumerate(cloud.actions) + if action == ("release", driver.gate("fault")) + ) + assert measured < released + assert ("platform-timeout", "execution:victim") in cloud.actions + + +def test_timeout_retry_never_launches_a_replacement_execution(): + cloud = ScriptCloud() + driver = ScenarioDriver( + context(scenario="timeout-retry", roles=("victim", "retry")), cloud + ) + driver.context.record_attempt("original", role="victim", requestId="request") + driver.context.record_attempt("retry", role="retry", requestId="second") + driver.start() + driver.place() + driver.execute() + assert len(cloud.invocations) == 1 + assert cloud.actions[0][2]["resume_roles"] == ("retry",) + assert [action[1] for action in cloud.actions if action[0] == "release"] == [ + driver.gate("fault") + ] + + +@pytest.mark.parametrize( + "scenario,gate_suffix,expected_status", + [ + ("checkpoint-settlement", "checkpoint", "SUCCEEDED"), + ("pending-cleanup", "cleanup", "SUCCEEDED"), + ("return-inflight", "loser", "SUCCEEDED"), + ("failure-inflight", "loser", "FAILED"), + ("late-operation", "loser", "SUCCEEDED"), + ], +) +def test_scope_drivers_observe_real_boundary_before_releasing_its_control( + scenario, gate_suffix, expected_status +): + cloud = ScriptCloud() + roles = ("victim", "resume") if scenario == "pending-cleanup" else ("victim",) + driver = ScenarioDriver(context(scenario=scenario, roles=roles), cloud) + driver.start() + driver.place() + driver.execute() + release = next( + index + for index, action in enumerate(cloud.actions) + if action == ("release", f"victim-{gate_suffix}") + ) + finished = next( + index + for index, action in enumerate(cloud.actions) + if action == ("finish", "victim", expected_status) + ) + assert release < finished + assert any(action[0] == "phase" for action in cloud.actions[:release]) + + +def test_each_case_has_independent_artifact_namespace(monkeypatch, tmp_path): + class Driver: + def __init__(self, context, cloud, **kwargs): + self.cloud = cloud + + def start(self): + pass + + monkeypatch.setattr(adapter_module, "ScenarioDriver", Driver) + + async def exercise(): + value = adapter(tmp_path) + await asyncio.gather(value.start(context("one")), value.start(context("two"))) + first = value.cases["one"].driver.cloud + second = value.cases["two"].driver.cloud + assert first.artifacts_dir != second.artifacts_dir + assert first.case_context.scenario.case_id == "one" + assert second.case_context.scenario.case_id == "two" + with pytest.raises(CollectionError, match="replace"): + await value.start(context("one")) + + asyncio.run(exercise()) + + +def settled_cloud(*, view=None): + cloud = ScriptCloud() + cloud.invocations = [{"arn": "execution", "native_requests": ["native"]}] + events = [ + {"phase": "WRAPPER_ENTER", "request": "native"}, + {"phase": "WRAPPER_RETURN", "request": "native"}, + { + "phase": "CONFORMANCE", + "observation": { + "kind": "task_start", + "taskId": "root", + "requestId": "native", + }, + }, + { + "phase": "CONFORMANCE", + "observation": { + "kind": "task_exit", + "taskId": "root", + "requestId": "native", + }, + }, + ] + cloud.refresh = lambda **kwargs: events + driver = ScenarioDriver(context(view=view), cloud) + return driver, events + + +@pytest.mark.parametrize( + "start,end,identity", + [ + ("task_start", "task_exit", "taskId"), + ("io_start", "io_exit", "taskId"), + ("checkpoint_call", "checkpoint_exit", "callId"), + ("poll_call", "poll_exit", "callId"), + ("export_started", "export_completed", "exportId"), + ("flush_started", "flush_completed", "flushId"), + ], +) +def test_retirement_requires_every_real_task_call_and_export_to_exit( + start, end, identity +): + driver, events = settled_cloud() + assert driver.quiescent() + event = {"requestId": "native", identity: "outstanding"} + events.append({"phase": "CONFORMANCE", "observation": {**event, "kind": start}}) + assert not driver.quiescent() + events.append({"phase": "CONFORMANCE", "observation": {**event, "kind": end}}) + assert driver.quiescent() + + +def test_export_quiescence_needs_positive_publication_evidence_even_when_unsampled(): + driver, events = settled_cloud(view="invocation") + assert not driver.quiescent() + observation = { + "kind": "export_quiescence", + "requestId": "native", + "pendingExportIds": [], + "pendingFlushIds": [], + "publicationCount": 1, + } + events.append({"phase": "CONFORMANCE", "observation": observation}) + assert not driver.quiescent() + observation["publicationCount"] = 0 + assert driver.quiescent() + + +def test_unidentified_launch_or_missing_root_hook_cannot_claim_retirement(): + driver, events = settled_cloud() + driver.cloud.invocations[0]["native_requests"] = [] + assert not driver.quiescent() + driver.cloud.invocations[0]["native_requests"] = ["native"] + events[:] = [ + event + for event in events + if event.get("observation", {}).get("kind") != "task_exit" + ] + assert not driver.quiescent() + + +def test_wrong_function_response_is_never_adopted_for_case_cleanup(cloud): + cloud.lam.invoke.return_value = { + "StatusCode": 202, + "DurableExecutionArn": "arn:another-function/durable-execution/name/id", + } + with pytest.raises(CollectionError, match="API failed"): + cloud.start("success", role="victim") + assert cloud.invocations[0]["arn"] is None + + +def test_canonical_observations_keep_authoritative_outer_identity_and_hook_mismatch(): + from lmi_tests.shared_scenarios import canonical + + records = canonical( + [ + { + "phase": "CONFORMANCE", + "request": "outer", + "execution": "execution", + "observation": {"kind": "task_exit", "taskId": "root"}, + }, + { + "phase": "CONFORMANCE", + "request": "outer", + "execution": "execution", + "observation": { + "kind": "invocation_start", + "requestId": "wrong-hook-request", + }, + }, + ] + ) + assert records[0]["requestId"] == records[0]["ownerRequestId"] == "outer" + assert records[0]["executionArn"] == "execution" + assert records[1]["requestId"] == "wrong-hook-request" + assert records[1]["ownerRequestId"] == "outer" diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py new file mode 100644 index 000000000..4026bc337 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_cleanup.py @@ -0,0 +1,94 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cleanup never uses a stale receipt or mutates work before ownership checks.""" + +import hashlib +import io +import json +from unittest.mock import Mock + +import pytest + +from lmi_tests import shared +from lmi_tests.shared_lock import OwnershipError + + +def test_stale_receipt_cannot_skip_current_ownership_verification( + monkeypatch, tmp_path +): + deployment = {"run": "same-run", "sharedLease": {"token": "current"}} + lease = Mock() + lease.to_manifest.return_value = deployment["sharedLease"] + lease._read.side_effect = OwnershipError("current ownership must be verified") + monkeypatch.setattr(shared.RunLease, "from_manifest", lambda value: lease) + receipt = tmp_path / "ownership-released.json" + receipt.write_text(json.dumps({"runId": "same-run", "lease": {"token": "old"}})) + with pytest.raises(OwnershipError): + shared.cleanup(deployment, tmp_path) + lease._read.assert_called_once() + lease._read.reset_mock() + receipt.write_text(json.dumps({"runId": "same-run", "lease": {"token": "current"}})) + assert shared.cleanup(deployment, tmp_path) == 0 + lease._read.assert_not_called() + + +@pytest.mark.parametrize("foreign", [True, False]) +def test_case_ownership_is_checked_before_control_release( + monkeypatch, tmp_path, foreign +): + target = "arn:aws:lambda:us-west-2:123:function:owned:$LATEST.PUBLISHED" + owned = target + "/durable-execution/name/id" + deployment = { + "run": "run", + "sharedLease": {}, + "bucket": "private", + "region": "us-west-2", + "functions": {"c1": target}, + } + lease = Mock() + lease._read.return_value = ({}, "etag") + lease._owned.return_value = {"profiles": {target: "active"}} + monkeypatch.setattr(shared.RunLease, "from_manifest", lambda value: lease) + case = {"case_id": "case", "cohort_id": "cohort", "profile_id": "c1"} + plan = { + "runId": "run", + "profiles": [{"profile_id": "c1", "resource_id": target}], + "scenarios": [case], + } + output = tmp_path / "python-lifecycle" + output.mkdir() + (output / "manifest.json").write_text(json.dumps(plan)) + identity = hashlib.sha256(b"case:cohort").hexdigest()[:20] + invocations = output / "source" / "cases" / identity / "invocations" + invocations.mkdir(parents=True) + (invocations / "intent.json").write_text( + json.dumps( + { + "payload": {"run": "run", "caseId": "case"}, + "arn": "arn:foreign" if foreign else owned, + } + ) + ) + storage, service = Mock(), Mock() + prefix = f"runs/run/cases/{identity}/control/" + storage.get_paginator.return_value.paginate.return_value = [ + {"Contents": [{"Key": prefix + "release-all"}]} + ] + storage.get_object.return_value = {"Body": io.BytesIO(b"hold")} + service.get_durable_execution.return_value = {"Status": "RUNNING"} + monkeypatch.setattr( + shared, "client", lambda name, region: storage if name == "s3" else service + ) + assert shared.cleanup(deployment, tmp_path) == 1 + if foreign: + storage.put_object.assert_not_called() + service.stop_durable_execution.assert_not_called() + else: + storage.put_object.assert_called_once_with( + Bucket="private", Key=prefix + "release-all", Body=b"release" + ) + service.stop_durable_execution.assert_called_once_with( + DurableExecutionArn=owned + ) + lease.finish_profile.assert_called_once_with(target, False) + lease.release.assert_not_called() diff --git a/lmi_tests/tests/unit/test_shared_control_receipts.py b/lmi_tests/tests/unit/test_shared_control_receipts.py new file mode 100644 index 000000000..553590f5e --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_control_receipts.py @@ -0,0 +1,46 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Control intent and acknowledgement are separate, retained observations.""" + +import json +from threading import Event + +import pytest + +from lmi_tests.cloud import Cloud + + +def test_release_intent_is_saved_before_transport_and_ack_only_after_success(tmp_path): + cloud = object.__new__(Cloud) + cloud.manifest = {"bucket": "private"} + cloud.case_context = object() + cloud.cancel_event = Event() + cloud.gates = {"gate", "second"} + cloud.released_gates = set() + cloud.control_prefix = "runs/run/cases/case/control/" + cloud.control_records = [] + cloud.artifacts_dir = tmp_path + cloud.source_uris = set() + + class S3: + fail = False + + def put_object(self, **kwargs): + recorded = json.loads((tmp_path / "control-records.json").read_text()) + assert recorded[-1]["state"] == "PENDING" + assert "time" not in recorded[-1] + if self.fail: + raise OSError("ambiguous transport failure") + + cloud.s3 = S3() + cloud.release("gate") + successful = cloud.control_records[0] + assert successful["state"] == "ACKNOWLEDGED" + assert successful["started"] <= successful["time"] + cloud.s3.fail = True + cloud.release("gate") + assert len(cloud.control_records) == 1 + with pytest.raises(OSError): + cloud.release("second") + assert cloud.control_records[-1]["state"] == "PENDING" + assert "time" not in cloud.control_records[-1] diff --git a/lmi_tests/tests/unit/test_shared_deadline_measurement.py b/lmi_tests/tests/unit/test_shared_deadline_measurement.py new file mode 100644 index 000000000..d5f768653 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_deadline_measurement.py @@ -0,0 +1,263 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Normal victim release must follow the immutable deadline measurement.""" + +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from lmi_tests import shared_scenarios as module +from lmi_tests.evidence import CollectionError, PlacementError +from lmi_tests.tests.unit.test_cloud import driver as driver + + +class MeasurementCloud: + def __init__(self, context, clock, *, retry_observed=True, write_fails=False): + self.context, self.clock = context, clock + self.concurrency = 1 + self.manifest = {"invocationTimeout": 60, "driverTimeout": 1} + self.actions = [] + self.events = [ + { + "phase": "WRAPPER_ENTER", + "request": "original", + "execution": "execution", + "time": 0, + } + ] + if retry_observed: + self.events.append( + { + "phase": "WRAPPER_ENTER", + "request": "retry", + "execution": "execution", + "time": 52, + } + ) + self.raw_histories = {"execution": [{"EventType": "InvocationCompleted"}]} + self.source_uris = {"s3://case/measurement-source"} + self.write_fails = write_fails + self.snapshot = None + + def poll(self, predicate, **options): + observed = predicate() + if not observed: + raise CollectionError(options.get("message", "measurement wait incomplete")) + return observed + + def platform_timeout(self, item, request): + self.actions.append(("timeout", request)) + + def for_item(self, item): + return self.events + + def refresh(self, **options): + return self.events + + def history(self, item): + self.actions.append(("history", item["arn"])) + return self.raw_histories[item["arn"]] + + def record_deadline_measurement(self, snapshot): + if self.write_fails: + raise OSError("evidence storage failed") + self.snapshot = deepcopy(snapshot) + self.actions.append(("measurement", self.clock.now)) + + def release(self, gate): + self.actions.append(("release", gate, self.clock.now)) + + def wait_terminal(self, item): + self.actions.append(("terminal", item["arn"])) + self.events.append( + {"phase": "WRAPPER_RETURN", "request": "retry", "status": "FAILED"} + ) + self.raw_histories[item["arn"]].append({"EventType": "ExecutionFailed"}) + return {"Status": "FAILED"} + + def start(self, fixture, **options): + role = options["role"] + self.context.record_attempt(role, role=role, requestId=role) + self.actions.append(("start", role)) + return {"arn": role, "marker": role} + + def finish(self, item, status): + self.actions.append(("finish", item["arn"], status)) + + +def measured_driver( + monkeypatch, + *, + scenario="timeout-retry", + now=100, + retry_observed=True, + write_fails=False, +): + clock = SimpleNamespace(now=now) + monkeypatch.setattr(module.time, "time", lambda: clock.now) + roles = ( + ("victim", "retry") if scenario == "timeout-retry" else ("victim", "recovery") + ) + profile = Profile("native-1", "function", 1, 1, "sdk", "configuration") + context = ScenarioContext( + "run", + Scenario( + "case", + "lmi-lifecycle-11", + profile.profile_id, + scenario, + roles=roles, + behavior_budgets={ + "cleanupMs": 5000, + "observationGraceMs": 1000, + "retryMs": 2000, + }, + ), + profile, + ) + context.record_attempt("original", role="victim", requestId="original") + if scenario == "timeout-retry" and retry_observed: + context.record_attempt("retry", role="retry", requestId="retry") + cloud = MeasurementCloud( + context, clock, retry_observed=retry_observed, write_fails=write_fails + ) + result = module.ScenarioDriver(context, cloud) + result.items["victim"] = {"arn": "execution", "marker": "victim"} + result.boundary = { + "deadline": 50, + "request": "original", + "environment": "original-environment", + } + return result, cloud, clock + + +def test_completed_measurement_is_persisted_before_normal_release_and_terminal_wait( + monkeypatch, +): + scenario, cloud, clock = measured_driver(monkeypatch) + scenario._deadline() + kinds = [action[0] for action in cloud.actions] + assert kinds.index("measurement") < kinds.index("release") < kinds.index("terminal") + assert [action[1] for action in cloud.actions if action[0] == "release"] == [ + scenario.gate("fault") + ] + assert cloud.snapshot["selectedRequests"] == { + "victim": "original", + "retry": "retry", + } + assert cloud.snapshot["cleanupCutoffAtMs"] == 55_000 + assert cloud.snapshot["observationCutoffAtMs"] == 56_000 + assert cloud.snapshot["observedAtMs"] == clock.now * 1000 + assert cloud.snapshot["sourceUris"] == ["s3://case/measurement-source"] + assert not any( + event["phase"] == "WRAPPER_RETURN" for event in cloud.snapshot["events"] + ) + assert cloud.snapshot["histories"]["execution"] == [ + {"EventType": "InvocationCompleted"} + ] + assert not any(action[0] == "finish" for action in cloud.actions) + + +def test_victim_fault_cannot_be_released_before_cutoff_plus_observation_grace( + monkeypatch, +): + scenario, cloud, _ = measured_driver(monkeypatch, now=55.5) + with pytest.raises(CollectionError): + scenario._deadline() + assert cloud.snapshot is None + assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) + + +def test_victim_fault_cannot_be_released_before_same_execution_retry_is_observed( + monkeypatch, +): + scenario, cloud, _ = measured_driver(monkeypatch, retry_observed=False) + with pytest.raises(CollectionError, match="real service retry"): + scenario._deadline() + assert cloud.snapshot is None + assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) + + +def test_original_recovery_placement_failure_is_not_repaired_by_victim_release( + monkeypatch, +): + scenario, cloud, _ = measured_driver(monkeypatch, scenario="deadline-recovery") + scenario.wait_overlap = Mock( + side_effect=PlacementError("original environment unavailable") + ) + with pytest.raises(PlacementError, match="original environment"): + scenario._deadline() + assert cloud.snapshot is None + assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) + + +def test_measurement_storage_failure_keeps_normal_fault_release_unperformed( + monkeypatch, +): + scenario, cloud, _ = measured_driver(monkeypatch, write_fails=True) + with pytest.raises(OSError, match="evidence storage"): + scenario._deadline() + assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) + + +@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"]) +def test_terminal_observation_preserves_actual_outcome_and_waits_for_all_wrappers( + driver, status +): + driver.manifest["driverTimeout"] = 1 + driver.wait = Mock() + driver.lam.get_durable_execution = Mock( + side_effect=[{"Status": "RUNNING"}, {"Status": status}] + ) + opened = [ + {"phase": "WRAPPER_ENTER", "request": "original"}, + {"phase": "WRAPPER_ENTER", "request": "retry"}, + ] + closed = [ + *opened, + {"phase": "WRAPPER_RAISE", "request": "original"}, + {"phase": "WRAPPER_RETURN", "request": "retry"}, + ] + driver.for_item = Mock(side_effect=[opened, closed]) + driver.history = Mock(return_value=[]) + observed = driver.wait_terminal({"arn": "execution", "marker": "victim"}) + assert observed == {"Status": status} + assert driver.for_item.call_count == 2 + assert driver.execution_results["execution"] == observed + driver.history.assert_called_once() + + +def test_terminal_service_status_does_not_replace_missing_native_wrapper_exit(driver): + driver.manifest["driverTimeout"] = 1 + driver.lam.get_durable_execution = Mock(return_value={"Status": "FAILED"}) + driver.for_item = Mock( + return_value=[{"phase": "WRAPPER_ENTER", "request": "still-running"}] + ) + + def poll(predicate, **options): + result = predicate() + if not result: + raise CollectionError(options["message"]) + return result + + driver.poll = poll + with pytest.raises(CollectionError, match="wrapper exits"): + driver.wait_terminal({"arn": "execution", "marker": "victim"}) + + +def test_persisted_measurement_does_not_follow_mutated_later_observations(driver): + snapshot = { + "selectedRequests": {"victim": "original"}, + "events": [{"phase": "EFFECT", "time": 54}], + } + driver.record_deadline_measurement(snapshot) + snapshot["events"][0]["time"] = 99 + assert driver.deadline_measurement["events"][0]["time"] == 54 + assert (driver.artifacts_dir / "measurements/deadline.json").exists() diff --git a/lmi_tests/tests/unit/test_shared_lock.py b/lmi_tests/tests/unit/test_shared_lock.py new file mode 100644 index 000000000..1e8e3d6f2 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_lock.py @@ -0,0 +1,110 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ownership must survive run failure and conditional-write races.""" + +import io +import json + +from botocore.exceptions import ClientError +import pytest + +from lmi_tests.shared_lock import OwnershipError, RunLease + + +class Store: + def __init__(self): + self.record = None + self.version = 0 + self.race = False + + def get_object(self, **kwargs): + if self.record is None: + raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") + return { + "Body": io.BytesIO(json.dumps(self.record).encode()), + "ETag": str(self.version), + } + + def put_object(self, **kwargs): + if ( + self.race + or (kwargs.get("IfNoneMatch") == "*" and self.record is not None) + or ("IfMatch" in kwargs and kwargs["IfMatch"] != str(self.version)) + ): + raise ClientError({"Error": {"Code": "PreconditionFailed"}}, "PutObject") + self.record = json.loads(kwargs["Body"]) + self.version += 1 + + +def test_run_holds_all_targets_until_both_profiles_retire(): + store = Store() + lease = RunLease.acquire("private", "stack", "run", ("c1", "c2"), s3=store) + lease.assert_owned("c1") + with pytest.raises(OwnershipError): + RunLease.acquire("private", "stack", "other", ("c1", "c2"), s3=store) + lease.finish_profile("c1", quiescent=True) + with pytest.raises(OwnershipError): + lease.release() + lease.finish_profile("c2", quiescent=True) + lease.release() + replacement = RunLease.acquire("private", "stack", "next", ("c1", "c2"), s3=store) + replacement.assert_owned("c2") + with pytest.raises(OwnershipError): + lease.assert_owned("c1") + + +def test_quarantine_cannot_be_erased_by_later_success_or_clock_expiry(): + store = Store() + lease = RunLease.acquire("private", "stack", "run", ("c1", "c2"), s3=store) + lease.finish_profile("c1", quiescent=False) + lease.finish_profile("c1", quiescent=True) + lease.assert_owned("c2") + with pytest.raises(OwnershipError): + lease.assert_owned("c1") + with pytest.raises(OwnershipError): + lease.release() + with pytest.raises(OwnershipError): + RunLease.acquire("private", "stack", "new", ("c1", "c2"), s3=store) + + +def test_conditional_write_race_does_not_steal_ownership(): + store = Store() + store.race = True + with pytest.raises(OwnershipError): + RunLease.acquire("private", "stack", "run", ("c1",), s3=store) + assert store.record is None + + +def test_unknown_resources_and_malformed_records_fail_closed(): + store = Store() + lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) + with pytest.raises(OwnershipError): + lease.finish_profile("foreign", quiescent=True) + with pytest.raises(OwnershipError): + lease.assert_owned("foreign") + store.record = {"schemaVersion": 999} + with pytest.raises(OwnershipError): + lease.assert_owned("c1") + + +def test_reusing_a_settled_profile_retains_the_lease_for_the_new_group(): + store = Store() + lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) + lease.finish_profile("c1", quiescent=True) + lease.assert_owned("c1") + with pytest.raises(OwnershipError): + lease.release() + lease.finish_profile("c1", quiescent=True) + lease.release() + + +@pytest.mark.parametrize("state", ["unknown", None, 1, {"unverified": True}]) +def test_unknown_retirement_state_cannot_be_claimed_or_cleared(state): + store = Store() + lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) + store.record["profiles"]["c1"] = state + with pytest.raises(OwnershipError): + lease.assert_owned("c1") + with pytest.raises(OwnershipError): + lease.finish_profile("c1", True) + assert store.record["profiles"]["c1"] == state diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py new file mode 100644 index 000000000..35f33143f --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -0,0 +1,98 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Plans pin the shared matrix and immutable SDK/deployment inputs.""" + +import json +from unittest.mock import patch + +import pytest + +from aws_durable_execution_conformance_tests.lmi.app import load_manifest +from lmi_tests.shared import make_manifest, verify_checkout +from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity + + +def deployment(): + return { + "run": "run-1", + "stack": "stack", + "region": "us-west-2", + "account": "123456789012", + "runtime": "python3.14", + "role": "arn:role", + "bucket": "private", + "commit": "sdk-commit", + "provider": "arn:aws:lambda:us-west-2:123456789012:capacity-provider:provider", + "codeSha256": "bundle-sha", + "invocationTimeout": 60, + "executionTimeout": 240, + "driverTimeout": 120, + "concurrencies": {"c1": 1, "c2": 2}, + "functions": { + "c1": "arn:function:c1:$LATEST.PUBLISHED", + "c2": "arn:function:c2:$LATEST.PUBLISHED", + }, + "wheels": { + "aws_durable_execution_sdk_python-3.0.0.whl": "core-sha", + "aws_durable_execution_sdk_python_otel-2.0.0.whl": "otel-sha", + }, + "sharedLease": { + "bucket": "private", + "key": "ownership/stack/shared-run.json", + "runId": "run-1", + "token": "fence", + "resources": ["c1", "c2"], + }, + } + + +@pytest.mark.parametrize( + "profile,count", [("python-lifecycle", 76), ("export-lifecycle", 24)] +) +def test_generated_manifest_covers_every_required_slot(tmp_path, profile, count): + source = deployment() + plan = make_manifest(source, profile, tmp_path) + assert len(plan["scenarios"]) == count + assert {case["profile_id"] for case in plan["scenarios"]} == {"c1", "c2"} + assert plan["sdk"]["deployment"]["conformanceRevision"] == CONFORMANCE_REVISION + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + assert load_manifest(path) == plan + plan["sdk"]["deployment"]["commit"] = "mutated" + assert source["commit"] == "sdk-commit" + + +def test_no_unowned_or_unidentified_artifact_can_be_planned(tmp_path): + source = deployment() + source.pop("sharedLease") + with pytest.raises(ValueError, match="ownership"): + make_manifest(source, "python-lifecycle", tmp_path) + source = deployment() + source["wheels"] = {} + with pytest.raises(ValueError, match="wheel"): + make_manifest(source, "python-lifecycle", tmp_path) + + +def test_configuration_identity_changes_with_the_tested_settings(): + source = deployment() + original = configuration_identity(source, "c1") + assert original != configuration_identity(source, "c2") + source["codeSha256"] = "other-bundle" + assert original != configuration_identity(source, "c1") + + +def test_only_the_exact_clean_shared_revision_is_accepted(tmp_path): + with patch("lmi_tests.shared.subprocess.check_output", side_effect=["wrong\n"]): + with pytest.raises(ValueError, match="pinned"): + verify_checkout(tmp_path) + with patch( + "lmi_tests.shared.subprocess.check_output", + side_effect=[CONFORMANCE_REVISION + "\n", " M packages/validator.py\n"], + ): + with pytest.raises(ValueError, match="local changes"): + verify_checkout(tmp_path) + with patch( + "lmi_tests.shared.subprocess.check_output", + side_effect=[CONFORMANCE_REVISION + "\n", ""], + ): + verify_checkout(tmp_path) diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py new file mode 100644 index 000000000..f3fad0abb --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -0,0 +1,396 @@ +"""Public hook records preserve actual IDs and never fill missing observations.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from datetime import UTC, datetime +from dataclasses import asdict +from itertools import count +import io +import json +import threading +import time +from typing import Any + +import pytest + +from aws_durable_execution_sdk_python.lambda_service import ErrorObject, OperationStatus +from aws_durable_execution_sdk_python.plugin import ( + InvocationStartInfo, + OperationEndInfo, + OperationInfo, + OperationStartInfo, + OperationType, + UserFunctionEndInfo, + UserFunctionStartInfo, +) +from lmi_tests import fixture +from aws_durable_execution_sdk_python_testing import invoker as local_invoker +from lmi_tests.shared_observations import ( + LifecycleObservationFactory, + LifecycleObservations, + OperationObservation, +) +from lmi_tests.tests.regressions.test_issue741 import lambda_context + + +class MemoryLedger: + """Replace only evidence/control storage; the SDK local service stays real.""" + + def __init__(self) -> None: + self.objects: dict[str, bytes] = {} + self.lock = threading.Lock() + + def put_object(self, *, Bucket, Key, Body, **_kwargs): + with self.lock: + self.objects[Key] = Body + return {} + + def get_object(self, *, Bucket, Key): + with self.lock: + return {"Body": io.BytesIO(self.objects[Key])} + + def records(self) -> list[dict[str, Any]]: + with self.lock: + records = [ + json.loads(body) + for key, body in self.objects.items() + if "/events/" in key + ] + return sorted( + records, key=lambda record: (record["request"], record["sequence"]) + ) + + def observations(self) -> list[dict[str, Any]]: + return [record for record in self.records() if record["phase"] == "CONFORMANCE"] + + def wait_for(self, kind: str, timeout: float = 10) -> dict[str, Any]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + found = [ + record + for record in self.observations() + if record["observation"]["kind"] == kind + ] + if found: + return found[-1] + time.sleep(0.01) + raise AssertionError(f"No actual {kind} observation") + + +class MemorySink: + def __init__(self) -> None: + self.records: list[dict[str, Any]] = [] + self.lock = threading.Lock() + + def observe(self, kind: str, **fields: Any) -> dict[str, Any]: + value = {"kind": kind, **fields} + with self.lock: + self.records.append(value) + return value + + +def shared_payload(scenario: str = "success") -> dict[str, Any]: + return { + "runId": "unit", + "caseId": "case", + "cohortId": "cohort-1", + "role": "victim", + "coverageProfile": "python-lifecycle", + "sharedScenario": scenario, + "scenario": scenario, + "marker": "marker", + "controlPrefix": "runs/unit/cases/owner/control/", + "behaviorBudgets": {"observationGraceMs": 1}, + } + + +def install_ledger(monkeypatch) -> MemoryLedger: + ledger = MemoryLedger() + monkeypatch.setenv("LMI_RUN_ID", "unit") + monkeypatch.setenv("LMI_COMMIT", "tested-commit") + monkeypatch.setenv("LMI_BUCKET", "private-evidence") + monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) + monkeypatch.setattr(fixture.boto3, "client", lambda *_args, **_kwargs: ledger) + monkeypatch.setattr(fixture, "environment_id", lambda: "local-environment") + original_context = local_invoker.create_test_lambda_context + requests = count(1) + + def distinct_local_context(**kwargs): + # The stock local service reuses one mock request ID. Supply its runtime + # context before invoking the real fixture, never alter captured evidence. + context = original_context(**kwargs) + return type(context)( + **{**asdict(context), "aws_request_id": f"local-request-{next(requests)}"} + ) + + monkeypatch.setattr( + local_invoker, "create_test_lambda_context", distinct_local_context + ) + return ledger + + +def user_info( + operation_id: str, + *, + parent_id: str | None = None, + name: str = "work", + kind: OperationType = OperationType.STEP, +) -> UserFunctionStartInfo: + return UserFunctionStartInfo( + operation_id=operation_id, + operation_type=kind, + sub_type=None, + name=name, + parent_id=parent_id, + start_time=datetime.now(UTC), + is_replayed=False, + status=OperationStatus.STARTED, + attempt=1, + ) + + +def test_missing_body_identity_stays_missing_instead_of_being_guessed() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.body("work", "marker", 1) + assert [item["kind"] for item in sink.records] == ["observation_error"] + assert "operationId" not in sink.records[0] + + +@pytest.mark.parametrize("wrong", ["name", "attempt"]) +def test_body_requires_matching_public_step_hook(wrong: str) -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.on_user_function_start(user_info("actual-id")) + observer.body( + "other" if wrong == "name" else "work", "marker", 2 if wrong == "attempt" else 1 + ) + assert sink.records[-1]["kind"] == "observation_error" + assert not any(item["kind"] == "body" for item in sink.records) + + +def test_concurrent_public_user_scopes_keep_their_actual_operation_ids() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + barrier = threading.Barrier(2) + + def body(identity: str) -> None: + info = user_info(identity, parent_id="actual-parent") + observer.on_user_function_start(info) + barrier.wait(timeout=3) + observer.body("work", identity, 1) + observer.on_user_function_end(UserFunctionEndInfo.from_start_info(info, None)) + assert observer.current() is None + + with ThreadPoolExecutor(2) as executor: + list(executor.map(body, ["first-id", "second-id"])) + bodies = [item for item in sink.records if item["kind"] == "body"] + assert {(item["operationId"], item["value"]) for item in bodies} == { + ("first-id", "first-id"), + ("second-id", "second-id"), + } + assert { + item["taskId"] for item in sink.records if item["kind"] == "task_start" + } == {item["taskId"] for item in sink.records if item["kind"] == "task_exit"} + + +def test_closed_ancestor_comes_from_the_real_parent_end_hook() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + parent = OperationEndInfo( + operation_id="parent-id", + operation_type=OperationType.CONTEXT, + sub_type="Parallel", + name="race", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.SUCCEEDED, + ) + observer.on_operation_end(parent) + observer.on_user_function_start( + user_info("child-id", parent_id="parent-id", kind=OperationType.CONTEXT) + ) + assert observer.closed_ancestor() == "parent-id" + assert sink.records[1] == { + "kind": "scope_closed", + "scopeId": "parent-id", + "parentId": None, + } + + +def test_missing_parent_completion_and_unmatched_task_end_are_visible() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + assert observer.closed_ancestor() is None + info = user_info("child-id", parent_id="not-observed", kind=OperationType.CONTEXT) + observer.on_user_function_start(info) + assert observer.closed_ancestor() is None + observer.on_user_function_end( + UserFunctionEndInfo.from_start_info(user_info("wrong-id"), None) + ) + assert sum(item["kind"] == "observation_error" for item in sink.records) == 3 + + +def test_configuration_uses_hook_identity_and_reports_a_missing_hook() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + observer.on_user_function_start(user_info("root", kind=OperationType.EXECUTION)) + with observer.configuration("map", 1): + observer.on_operation_start( + OperationStartInfo( + operation_id="actual-map", + operation_type=OperationType.CONTEXT, + sub_type="Map", + name="map", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.STARTED, + ) + ) + assert ( + next(item for item in sink.records if item["kind"] == "operation_config")[ + "operationId" + ] + == "actual-map" + ) + with observer.configuration("missing-map", 1): + pass + assert sink.records[-1]["kind"] == "observation_error" + + +def test_checkpoint_snapshot_does_not_claim_reexecution_or_drop_stored_errors() -> None: + original = OperationInfo( + operation_id="stored-id", + operation_type=OperationType.STEP, + sub_type="Step", + name="failure", + parent_id=None, + start_time=None, + is_replayed=False, + status=OperationStatus.FAILED, + error=ErrorObject( + type="ValueError", message="stored failure", data=None, stack_trace=None + ), + ) + record = OperationObservation.from_info(original).to_record() + assert record["error"] == {"type": "ValueError", "message": "stored failure"} + assert record["startedAtMs"] is None and record["isReplayed"] is False + sink = MemorySink() + observer = LifecycleObservations(sink) + info = InvocationStartInfo( + request_id="actual-request", + execution_arn="actual-execution", + is_first_invocation=False, + operations={"stored-id": original}, + ) + assert LifecycleObservationFactory(observer).create_plugin(info) is observer + observer.on_invocation_start(info) + assert sink.records[0]["operations"] == [record] + assert not any(item["kind"] == "body" for item in sink.records) + + +def test_trace_records_immutable_case_identity_and_actual_source_sequence( + monkeypatch, +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + payload.update(otelView="invocation", exportMode="async-bounded-flush") + trace = fixture.Trace(payload, lambda_context(30_000), "actual-execution") + observed_at = time.time() * 1000 - 10 + record = trace.observe("boundary", atMs=observed_at) + assert record["observation"] == { + "kind": "boundary", + "atMs": observed_at, + "sourceSequence": record["sequence"], + "captureSequence": 1, + } + assert record["caseId"] == "case" and record["exportMode"] == "async-bounded-flush" + assert trace.payload["otel_view"] == "invocation" and trace.payload["run"] == "unit" + assert ledger.observations() == [record] + + +@pytest.mark.parametrize("field", ["caseId", "role", "controlPrefix"]) +def test_shared_case_cannot_silently_drop_ownership_metadata( + monkeypatch, field: str +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + del payload[field] + with pytest.raises(ValueError, match=field): + fixture.Trace(payload, lambda_context(30_000), "execution") + assert ledger.records() == [] + + +def test_shared_case_cannot_fall_back_to_global_controls(monkeypatch) -> None: + install_ledger(monkeypatch) + payload = shared_payload() + payload["controlPrefix"] = "runs/unit/control/" + with pytest.raises(ValueError, match="case-owned"): + fixture.Trace(payload, lambda_context(30_000), "execution") + + +@pytest.mark.parametrize("mode", ["normal", "emergency"]) +def test_gate_release_reports_observed_provenance_and_keeps_controls_case_owned( + monkeypatch, mode: str +) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + trace = fixture.Trace(payload, lambda_context(30_000), "execution") + prefix = payload["controlPrefix"] + ledger.objects[prefix + "gate"] = b"release" if mode == "normal" else b"hold" + ledger.objects[prefix + "release-all"] = b"hold" if mode == "normal" else b"release" + ledger.objects["runs/unit/cases/other/control/gate"] = b"hold" + released = trace.gate_release("gate") + assert released.mode == mode + assert ledger.objects["runs/unit/cases/other/control/gate"] == b"hold" + observations = [item["observation"] for item in ledger.observations()] + assert [item["kind"] for item in observations] == [ + "io_start", + "fault_released", + "io_exit", + ] + assert observations[0]["taskId"] == observations[-1]["taskId"] + assert observations[1]["atMs"] == released.observed_at_ms + + +def test_root_scope_requires_the_public_invocation_operation() -> None: + sink = MemorySink() + observer = LifecycleObservations(sink) + with observer.root_scope(): + pass + assert [item["kind"] for item in sink.records] == ["observation_error"] + assert observer.completed_context("unknown") is None + assert sink.records[-1]["kind"] == "observation_error" + + +def test_gate_preserves_the_existing_checkpointed_none_result(monkeypatch) -> None: + ledger = install_ledger(monkeypatch) + payload = shared_payload() + ledger.objects[payload["controlPrefix"] + "gate"] = b"release" + ledger.objects[payload["controlPrefix"] + "release-all"] = b"hold" + trace = fixture.Trace(payload, lambda_context(30_000), "execution") + assert trace.gate("gate") is None + + +@pytest.mark.parametrize("method", ["checkpoint", "get_execution_state"]) +def test_failed_real_client_call_keeps_its_dispatch_and_exit_identity( + monkeypatch, method: str +) -> None: + from unittest.mock import Mock + + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30_000), "execution") + backend = Mock() + getattr(backend, method).side_effect = OSError("service unavailable") + client = fixture.ObservedClient(backend, trace) + with pytest.raises(OSError, match="service unavailable"): + getattr(client, method)(**({"updates": []} if method == "checkpoint" else {})) + observations = [record["observation"] for record in ledger.observations()] + prefix = "checkpoint" if method == "checkpoint" else "poll" + entered = next(item for item in observations if item["kind"] == prefix + "_call") + exited = next(item for item in observations if item["kind"] == prefix + "_exit") + assert entered["callId"] == exited["callId"] and entered["atMs"] <= exited["atMs"] + assert not any(item["kind"] == "checkpoint_ack" for item in observations) diff --git a/lmi_tests/tests/unit/test_shared_telemetry.py b/lmi_tests/tests/unit/test_shared_telemetry.py new file mode 100644 index 000000000..4c08b524d --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_telemetry.py @@ -0,0 +1,646 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Real public SDK/plugin calls with in-memory observation and gate transport.""" + +from concurrent.futures import ThreadPoolExecutor +import json +import os +import threading +import time +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_sdk_python import durable_execution +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInput, + DurableExecutionInvocationInputWithClient, +) +from aws_durable_execution_sdk_python.plugin import DurableInstrumentationPlugin +from lmi_tests import telemetry +from lmi_tests.shared_exporter import CURRENT_OWNER, owned +from lmi_tests.shared_telemetry import collect_telemetry +from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context + +HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1" + + +class SharedTrace: + def __init__( + self, + marker="victim", + *, + view="invocation", + mode="sync", + behavior="success", + profile="export-lifecycle", + ): + self.payload = { + "marker": marker, + "scenario": "success", + "run": "unit", + "caseId": marker, + "coverageProfile": profile, + "otel_view": view, + "exportMode": mode, + "otel_export": behavior, + "otel_export_gate": marker + "-export", + "behaviorBudgets": { + "flushWaitMs": 40, + "exportCompletionMs": 5000, + "exportMs": 5000, + "observationGraceMs": 1, + }, + } + self.identity = { + "request": "request-" + marker, + "execution": "arn:aws:lambda:us-west-2:123456789012:function:lmi:1/durable-execution/" + + marker + + "/local", + "environment": "memory-environment", + "process": "memory-process", + "pid": os.getpid(), + "marker": marker, + } + self.lock = threading.Lock() + self.capture_lock = threading.Lock() + self.capture_sequence = 0 + self.events = [] + self.sequence = 0 + self.held = threading.Event() + self.release = threading.Event() + self.published = threading.Event() + self.publication_entered = threading.Event() + self.allow_publication = threading.Event() + self.allow_publication.set() + self.fail_publication = False + self.sdk_completed_at = None + + def emit(self, phase, **fields): + with self.lock: + self.sequence += 1 + record = { + **self.identity, + "phase": phase, + "time": time.time(), + "sequence": self.sequence, + **fields, + } + self.events.append(record) + return record + + def capture_observation(self, kind, **fields): + with self.capture_lock: + self.capture_sequence += 1 + return { + "kind": kind, + "atMs": time.time_ns() / 1_000_000, + "captureSequence": self.capture_sequence, + **fields, + } + + def publish_observation(self, observation): + self.publication_entered.set() + if not self.allow_publication.wait(5): + raise TimeoutError("local observer fixture was not released") + if self.fail_publication: + raise RuntimeError("observed ledger write failure") + record = self.emit("CONFORMANCE", observation=observation) + if observation["kind"] == "export_quiescence": + self.published.set() + return record + + def observe(self, kind, **fields): + return self.publish_observation(self.capture_observation(kind, **fields)) + + def gate_release(self, name, *, on_hold=None, on_release=None): + assert name == self.payload["otel_export_gate"] + held_at_ms = None if self.release.is_set() else time.time_ns() / 1_000_000 + self.held.set() + if held_at_ms is not None and on_hold is not None: + on_hold(held_at_ms) + if not self.release.wait(5): + raise TimeoutError("local export fixture was not released") + return SimpleNamespace( + gate_id=name, observed_at_ms=time.time_ns() / 1_000_000, mode="normal" + ) + + def gate(self, name): + self.gate_release(name) + + +def run_public(trace): + source = invocation() + event = source.to_json_dict() + event["DurableExecutionArn"] = trace.identity["execution"] + event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps(trace.payload) + event["InitialExecutionState"]["Operations"][0]["StartTimestamp"] = int( + time.time() * 1000 + ) + typed = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( + DurableExecutionInvocationInput.from_json_dict(event), source.service_client + ) + context = lambda_context(30000) + context.aws_request_id = trace.identity["request"] + trace.emit("WRAPPER_ENTER") + + def call(plugins): + handler = durable_execution( + lambda event, ctx: ctx.step(lambda _: event["marker"], name="saved"), + plugins=plugins, + ) + try: + return handler(typed, context) + finally: + trace.sdk_completed_at = time.time() + + result = telemetry.invoke(trace, call) + trace.emit("WRAPPER_RETURN", status=result["Status"], time=trace.sdk_completed_at) + return result + + +def observations(trace, kind): + return [ + record["observation"] + for record in trace.events + if record["phase"] == "CONFORMANCE" and record["observation"]["kind"] == kind + ] + + +def wait_quiescence(trace): + assert trace.published.wait(5), "actual publication never reached quiescence" + until = time.monotonic() + 2 + while time.monotonic() < until: + active = telemetry.active_work(trace) + if ( + not active["pendingExportIds"] + and not active["pendingFlushIds"] + and not active["publicationCount"] + ): + assert active["publicationErrors"] == [] + return + threading.Event().wait(0.001) + pytest.fail("pending runtime ownership survived publication") + + +def attempt(trace): + return { + "requestId": trace.identity["request"], + "executionArn": trace.identity["execution"], + "environmentId": trace.identity["environment"], + "processId": trace.identity["process"], + "workerId": trace.identity["process"], + } + + +@pytest.fixture(autouse=True) +def runtimes(monkeypatch): + monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) + monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) + telemetry._RUNTIMES.clear() + factory = telemetry.Runtime + monkeypatch.setattr( + telemetry, + "Runtime", + lambda view, export_mode="sync", **options: factory( + view, export_mode, **{"flush_timeout_ms": 20, **options} + ), + ) + yield + for runtime in telemetry._RUNTIMES.values(): + runtime.provider.shutdown() + telemetry._RUNTIMES.clear() + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) +@pytest.mark.parametrize("behavior", ["success", "failure"]) +def test_public_sdk_modes_preserve_handler_and_actual_export_outcome( + view, mode, behavior +): + trace = SharedTrace(view=view, mode=mode, behavior=behavior) + assert run_public(trace)["Status"] == "SUCCEEDED" + wait_quiescence(trace) + completed = observations(trace, "export_completed") + assert completed and {item["outcome"] for item in completed} == { + "failed" if behavior == "failure" else "succeeded" + } + if mode == "sync": + assert max(item["atMs"] for item in completed) <= trace.sdk_completed_at * 1000 + else: + wait = observations(trace, "flush_wait_completed") + assert len(wait) == 1 and wait[0]["outcome"] == ( + "error" if behavior == "failure" else "completed" + ) + assert observations(trace, "flush_completed")[0]["atMs"] <= wait[0]["atMs"] + normalized = collect_telemetry( + trace.events, + [attempt(trace)], + view=view, + expected_sampled=True, + caller_headers={trace.identity["request"]: HEADER}, + observed_until_ms=time.time_ns() / 1_000_000, + ) + assert normalized["views"][view]["collection"]["status"] == "complete" + names = [record["span"]["name"] for record in normalized["views"][view]["spans"]] + assert names.count("Invocation") == names.count("Workflow") == 1 + assert normalized["views"][view]["carriers"][0]["header"] == HEADER + + +@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) +def test_held_export_has_mode_specific_real_sdk_boundary(mode): + trace = SharedTrace(mode=mode, behavior="held") + with ThreadPoolExecutor(max_workers=1) as worker: + result = worker.submit(run_public, trace) + try: + assert trace.held.wait(3) + if mode == "sync": + assert trace.sdk_completed_at is None + assert not result.done() + else: + assert result.result(timeout=3)["Status"] == "SUCCEEDED" + assert ( + observations(trace, "flush_wait_completed")[0]["outcome"] + == "timeout" + ) + assert telemetry.active_work(trace)["pendingExportIds"] + assert not trace.published.is_set() + assert not observations(trace, "flush_completed") + finally: + trace.release.set() + assert result.result(timeout=3)["Status"] == "SUCCEEDED" + wait_quiescence(trace) + if mode == "async-bounded-flush": + assert ( + min(item["atMs"] for item in observations(trace, "export_completed")) + > trace.sdk_completed_at * 1000 + ) + assert observations(trace, "export_quiescence")[-1]["pendingFlushIds"] == [] + + +def test_late_async_export_keeps_original_owner_during_warm_invocation(): + first = SharedTrace("first", mode="async-bounded-flush", behavior="held") + warm = SharedTrace("warm", mode="async-bounded-flush") + try: + assert run_public(first)["Status"] == "SUCCEEDED" + assert first.held.is_set() + assert run_public(warm)["Status"] == "SUCCEEDED" + assert telemetry.active_work(first)["pendingExportIds"] + assert telemetry.active_work(warm)["pendingExportIds"] + finally: + first.release.set() + wait_quiescence(first) + wait_quiescence(warm) + for trace in (first, warm): + assert all( + record["span"]["attributes"]["durable.execution.arn"] + == trace.identity["execution"] + for record in trace.events + if record["phase"] == "OTEL_EXPORT" + ) + assert ( + observations(first, "provider_observed")[0]["providerId"] + == observations(warm, "provider_observed")[0]["providerId"] + ) + + +def test_delayed_evidence_publication_does_not_extend_sdk_flush_wait(): + trace = SharedTrace(mode="async-bounded-flush") + trace.allow_publication.clear() + try: + assert run_public(trace)["Status"] == "SUCCEEDED" + assert trace.publication_entered.is_set() + assert trace.sdk_completed_at is not None + assert telemetry.active_work(trace)["publicationCount"] > 0 + assert not trace.published.is_set() + finally: + trace.allow_publication.set() + wait_quiescence(trace) + waited = observations(trace, "flush_wait_completed")[0] + started = observations(trace, "flush_started")[0] + assert ( + waited["atMs"] - started["atMs"] + < trace.payload["behaviorBudgets"]["flushWaitMs"] + ) + assert waited["atMs"] <= trace.sdk_completed_at * 1000 + + +def test_failed_publication_stays_owned_and_never_claims_quiescence(): + trace = SharedTrace(mode="async-bounded-flush") + trace.fail_publication = True + assert run_public(trace)["Status"] == "SUCCEEDED" + assert trace.publication_entered.wait(2) + active = telemetry.active_work(trace) + assert active["publicationCount"] > 0 + assert active["publicationErrors"] + assert not trace.published.is_set() + + +@pytest.mark.parametrize( + "hook", + [ + "on_invocation_start", + "on_invocation_end", + "on_operation_start", + "on_operation_end", + "on_operation_change", + "on_user_function_start", + "on_user_function_end", + "provide_propagation_metadata", + ], +) +def test_proxy_forwards_every_public_hook_and_restores_calling_owner(hook): + runtime = telemetry.Runtime("invocation") + first = runtime.tracker.owner(SharedTrace("first")) + newer = runtime.tracker.owner(SharedTrace("newer")) + delegate = Mock(spec=DurableInstrumentationPlugin) + info, answer = object(), object() + + def called(value): + assert value is info and CURRENT_OWNER.get() is first + return answer + + getattr(delegate, hook).side_effect = called + proxy = telemetry.ObservedPlugin(delegate, first) + try: + with owned(newer): + result = getattr(proxy, hook)(info) + assert CURRENT_OWNER.get() is newer + if hook == "provide_propagation_metadata": + assert result is answer + finally: + runtime.provider.shutdown() + getattr(delegate, hook).assert_called_once_with(info) + + +def test_missing_public_owner_cannot_borrow_current_runtime_state(): + runtime = telemetry.Runtime("invocation") + try: + with pytest.raises(RuntimeError, match="original public-plugin"): + runtime.tracker.accept(()) + finally: + runtime.provider.shutdown() + + +@pytest.mark.parametrize("sampled", [False, True]) +def test_collection_requires_quiescence_and_source_sequence_completeness( + monkeypatch, sampled +): + monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER[:-1] + str(int(sampled))) + trace = SharedTrace() + run_public(trace) + wait_quiescence(trace) + options = dict( + view="invocation", + expected_sampled=sampled, + caller_headers={}, + observed_until_ms=time.time_ns() / 1_000_000, + ) + assert ( + collect_telemetry(trace.events, [attempt(trace)], **options)["views"][ + "invocation" + ]["collection"]["status"] + == "complete" + ) + missing = [ + record + for record in trace.events + if record["phase"] != "CONFORMANCE" + or record["observation"]["kind"] != "export_quiescence" + ] + assert ( + collect_telemetry(missing, [attempt(trace)], **options)["views"]["invocation"][ + "collection" + ]["status"] + == "incomplete" + ) + assert ( + collect_telemetry(trace.events[1:], [attempt(trace)], **options)["views"][ + "invocation" + ]["collection"]["status"] + == "incomplete" + ) + + +def test_real_gate_reports_first_hold_and_does_not_rehold_an_open_gate(monkeypatch): + from lmi_tests import fixture + from lmi_tests.tests.unit.test_shared_observations import ( + install_ledger, + shared_payload, + ) + + ledger = install_ledger(monkeypatch) + payload = shared_payload() + trace = fixture.Trace(payload, lambda_context(30000), "execution") + prefix = payload["controlPrefix"] + ledger.objects[prefix + "gate"] = b"hold" + ledger.objects[prefix + "release-all"] = b"hold" + held, released = [], [] + with ThreadPoolExecutor(max_workers=1) as caller: + work = caller.submit( + trace.gate_release, "gate", on_hold=held.append, on_release=released.append + ) + try: + observed = ledger.wait_for("blocked") + finally: + ledger.put_object(Bucket=trace.bucket, Key=prefix + "gate", Body=b"release") + result = work.result(timeout=2) + assert held == [result.held_at_ms] == [observed["observation"]["atMs"]] + assert released == [result] + assert result.held_at_ms < result.observed_at_ms and result.mode == "normal" + held.clear() + opened = trace.gate_release("gate", on_hold=held.append) + assert opened.held_at_ms is None and held == [] + + +def test_actual_capture_order_survives_reversed_publication(monkeypatch): + from lmi_tests import fixture + from lmi_tests.tests.unit.test_shared_observations import ( + install_ledger, + shared_payload, + ) + + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") + at_ms = time.time_ns() / 1_000_000 + exported = trace.capture_observation("export_completed", atMs=at_ms) + wrapper = trace.observe("wrapper_return", atMs=at_ms) + published = trace.publish_observation(exported) + assert exported["captureSequence"] < wrapper["observation"]["captureSequence"] + assert published["sequence"] > wrapper["sequence"] + assert ledger.observations()[0]["observation"]["kind"] == "wrapper_return" + + +def test_capture_does_not_wait_for_s3_publication_lock(monkeypatch): + from lmi_tests import fixture + from lmi_tests.tests.unit.test_shared_observations import ( + install_ledger, + shared_payload, + ) + + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") + entered, release = threading.Event(), threading.Event() + original = ledger.put_object + + def blocked_put(**fields): + entered.set() + assert release.wait(3) + return original(**fields) + + monkeypatch.setattr(ledger, "put_object", blocked_put) + with ThreadPoolExecutor(max_workers=2) as workers: + publication = workers.submit(trace.observe, "first") + try: + assert entered.wait(2) + captured = workers.submit(trace.capture_observation, "second").result( + timeout=1 + ) + assert captured["captureSequence"] == 2 + finally: + release.set() + publication.result(timeout=2) + + +def test_proxy_restores_owner_when_real_delegate_hook_raises(): + runtime = telemetry.Runtime("invocation") + first = runtime.tracker.owner(SharedTrace("first")) + newer = runtime.tracker.owner(SharedTrace("newer")) + delegate = Mock(spec=DurableInstrumentationPlugin) + delegate.on_operation_end.side_effect = RuntimeError("hook failure") + try: + with owned(newer): + with pytest.raises(RuntimeError, match="hook failure"): + telemetry.ObservedPlugin(delegate, first).on_operation_end(object()) + assert CURRENT_OWNER.get() is newer + finally: + runtime.provider.shutdown() + + +def test_async_processor_respects_smaller_provider_wait_budget(): + runtime = telemetry.Runtime( + "invocation", "async-bounded-flush", flush_timeout_ms=2000 + ) + trace = SharedTrace(mode="async-bounded-flush", behavior="held") + owner = runtime.tracker.owner(trace) + try: + with owned(owner): + span = runtime.provider.get_tracer("public-fixture").start_span("held") + span.end() + assert trace.held.wait(2) + before = time.monotonic() + assert runtime.provider.force_flush(timeout_millis=20) is False + assert time.monotonic() - before < 0.5 + finally: + trace.release.set() + runtime.tracker.close_invocation(owner) + runtime.provider.shutdown() + assert observations(trace, "flush_wait_completed")[0]["outcome"] == "timeout" + assert observations(trace, "flush_completed")[0]["outcome"] == "succeeded" + + +def test_converter_rejects_mode_view_or_unjournaled_owner_substitution(): + trace = SharedTrace() + run_public(trace) + wait_quiescence(trace) + options = dict( + expected_sampled=True, + caller_headers={}, + observed_until_ms=time.time_ns() / 1_000_000, + ) + with pytest.raises(ValueError, match="view differs"): + collect_telemetry(trace.events, [attempt(trace)], view="execution", **options) + with pytest.raises(ValueError, match="exporter mode differs"): + collect_telemetry( + trace.events, + [attempt(trace)], + view="invocation", + export_mode="async-bounded-flush", + **options, + ) + with pytest.raises(ValueError, match="unjournaled"): + collect_telemetry(trace.events, [], view="invocation", **options) + + +def test_held_fixture_records_only_calls_that_actually_waited(): + trace = SharedTrace(mode="sync", behavior="held") + with ThreadPoolExecutor(max_workers=1) as caller: + work = caller.submit(run_public, trace) + try: + assert trace.held.wait(2) + finally: + trace.release.set() + assert work.result(timeout=3)["Status"] == "SUCCEEDED" + wait_quiescence(trace) + started = observations(trace, "export_started") + held = observations(trace, "export_held") + assert len(started) > len(held) == 1 + assert len(observations(trace, "export_completed")) == len(started) + + +@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) +def test_process_provider_does_not_retain_completed_invocations(mode): + import gc + import weakref + + references = [] + for index in range(12): + trace = SharedTrace("retired-" + str(index), mode=mode) + assert run_public(trace)["Status"] == "SUCCEEDED" + wait_quiescence(trace) + references.append(weakref.ref(trace)) + del trace + gc.collect() + assert all(reference() is None for reference in references) + assert len(telemetry._RUNTIMES) == 1 + runtime = next(iter(telemetry._RUNTIMES.values())) + assert ( + runtime.tracker.owners + == runtime.tracker.exports + == runtime.tracker.flushes + == {} + ) + + +def test_runtime_variants_keep_view_and_processor_configuration_immutable(): + for view, mode in ( + ("invocation", "sync"), + ("execution", "sync"), + ("invocation", "async-bounded-flush"), + ): + trace = SharedTrace(view + mode, view=view, mode=mode) + assert run_public(trace)["Status"] == "SUCCEEDED" + wait_quiescence(trace) + assert set(telemetry._RUNTIMES) == { + ("invocation", "sync"), + ("execution", "sync"), + ("invocation", "async-bounded-flush"), + } + assert len({runtime.identity for runtime in telemetry._RUNTIMES.values()}) == 3 + + +@pytest.mark.parametrize("shared,expected", [(True, "emergency"), (False, "normal")]) +def test_shared_gate_abort_has_priority_without_changing_legacy_release_order( + monkeypatch, shared, expected +): + from lmi_tests import fixture + from lmi_tests.tests.unit.test_shared_observations import ( + install_ledger, + shared_payload, + ) + + ledger = install_ledger(monkeypatch) + payload = ( + shared_payload() + if shared + else {"run": "unit", "marker": "legacy", "scenario": "success"} + ) + trace = fixture.Trace(payload, lambda_context(30000), "execution") + ledger.objects[trace.control_prefix + "gate"] = b"release" + ledger.objects[trace.control_prefix + "release-all"] = b"release" + result = trace.gate_release("gate") + assert result.mode == expected + assert result.held_at_ms is None diff --git a/lmi_tests/tests/unit/test_shared_workflow.py b/lmi_tests/tests/unit/test_shared_workflow.py new file mode 100644 index 000000000..a836a20b8 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_workflow.py @@ -0,0 +1,46 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Cloud acceptance must use the same pinned runner as local adapter tests.""" + +from pathlib import Path +import tomllib + +import yaml + +from lmi_tests.shared_contract import CONFORMANCE_REVISION + +ROOT = Path(__file__).resolve().parents[3] + + +def test_cloud_workflow_uses_pinned_shared_acceptance_and_final_cleanup(): + workflow = yaml.safe_load( + (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() + ) + assert workflow["env"]["CONFORMANCE_REVISION"] == CONFORMANCE_REVISION + assert "workflow_dispatch" in workflow["jobs"]["cloud"]["if"] + for job in workflow["jobs"].values(): + checkout = next( + step + for step in job["steps"] + if step.get("with", {}).get("repository") + == "aws/aws-durable-execution-conformance-tests" + ) + assert checkout["with"]["ref"] == "${{ env.CONFORMANCE_REVISION }}" + assert any( + step.get("run") == "hatch run lmi:shared-setup" for step in job["steps"] + ) + steps = workflow["jobs"]["cloud"]["steps"] + acceptance = next( + step for step in steps if step.get("run") == "hatch run lmi:shared" + ) + assert acceptance["timeout-minutes"] == 60 + cleanup = next( + step for step in steps if "lmi:shared-cleanup" in step.get("run", "") + ) + assert cleanup["if"] == "always()" + assert not any("lmi:cloud" in step.get("run", "") for step in steps) + scripts = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["hatch"][ + "envs" + ]["lmi"]["scripts"] + assert scripts["shared"].startswith("python -m lmi_tests.shared run") + assert scripts["shared-cleanup"].startswith("python -m lmi_tests.shared cleanup") diff --git a/pyproject.toml b/pyproject.toml index 5e8d2753c..8f6cd41a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,10 @@ cloud = "pytest lmi_tests/tests/e2e --lmi-cloud {args}" regressions = "pytest lmi_tests/tests/regressions --run-lmi-regressions {args}" lint = "ruff check lmi_tests && ruff format --check lmi_tests" build = "python -m lmi_tests.deploy build {args}" +shared-setup = "python -m lmi_tests.shared setup {args}" +shared = "python -m lmi_tests.shared run {args}" +shared-cleanup = "python -m lmi_tests.shared cleanup {args}" +shared-tests = "python -m pytest lmi_tests/tests/unit lmi_tests/tests/e2e --ignore=lmi_tests/tests/e2e/test_lifecycle.py --ignore=lmi_tests/tests/e2e/test_observability.py {args}" [tool.pytest.ini_options] # Declare custom markers to avoid warnings with --strict-markers From c6c9db0284a547bdbeb761cc507cca944e53fff2 Mon Sep 17 00:00:00 2001 From: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> Date: Thu, 12 Nov 2026 16:46:45 -0700 Subject: [PATCH 17/32] ci: run LMI PR tests on CodeBuild --- .github/workflows/lmi-e2e-tests.yml | 27 ++++++---- lmi_tests/README.md | 35 ++++++------ lmi_tests/shared-suite-design.md | 7 ++- lmi_tests/tests/unit/test_shared_workflow.py | 56 ++++++++++++++++++++ 4 files changed, 98 insertions(+), 27 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index f403da925..8e059865c 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -16,12 +16,16 @@ permissions: contents: read env: + # Ignore the CodeBuild image's shared pip target and Python search path. + PIP_CONFIG_FILE: /dev/null + PYTHONPATH: "" CONFORMANCE_REVISION: 6f899679c7eb6ede836c737acb9816e6fd7072fd LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} jobs: harness: - runs-on: ubuntu-latest + # Keep fork and Dependabot code outside the testing account. + runs-on: ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]') && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -32,10 +36,12 @@ jobs: ref: ${{ env.CONFORMANCE_REVISION }} path: .conformance persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + # uv supplies Python on the CodeBuild Amazon Linux image and on Ubuntu. + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: python-version: '3.14' - - run: pip install hatch==1.16.5 + activate-environment: true + - run: uv pip install hatch==1.16.5 - run: hatch run lmi:shared-setup - run: hatch run lmi:lint - run: hatch run lmi:shared-tests --junitxml=lmi_tests/artifacts/unit.xml @@ -57,11 +63,10 @@ jobs: cloud: name: cloud (Python 3.14, concurrency 1 and 2) needs: harness - # The #741 regressions remain strict failures on current main. Keep the - # cloud suite opt-in until its implementation contract is satisfied; the - # non-cloud harness still runs on every PR and main push. - if: github.event_name == 'workflow_dispatch' && github.actor != 'dependabot[bot]' - runs-on: ubuntu-latest + # Trusted PRs and manual runs exercise the full suite. The #741 regressions + # remain strict failures; fork and Dependabot PRs run only the harness. + if: github.actor != 'dependabot[bot]' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) + runs-on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }} timeout-minutes: 120 permissions: contents: read @@ -87,9 +92,11 @@ jobs: ref: ${{ env.CONFORMANCE_REVISION }} path: .conformance persist-credentials: false - - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + # uv supplies Python on the CodeBuild Amazon Linux image and on Ubuntu. + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 with: python-version: '3.14' + activate-environment: true - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 with: role-to-assume: ${{ secrets.TEST_ROLE_ARN }} @@ -101,7 +108,7 @@ jobs: test -n "$CAPACITY_PROVIDER_ARN" test -n "$TEST_LAMBDA_EXECUTION_ROLE_ARN" test "$(aws sts get-caller-identity --query Account --output text)" = "$TEST_ACCOUNT_ID" - - run: pip install hatch==1.16.5 + - run: uv pip install hatch==1.16.5 - run: hatch run lmi:shared-setup - name: Build SDK checkout timeout-minutes: 5 diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 4450a7796..6732a751b 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -17,8 +17,10 @@ It follows the separate cloud-suite approach in It changes no production SDK code. The deadline and early-completion regressions are **expected to fail on the current SDK**. They are ordinary failing assertions, with no `xfail`, swallowed failure, or expected-failure success status. CI runs the -non-cloud harness automatically on PR updates and main pushes. The cloud suite -requires `workflow_dispatch` until the #741 implementation satisfies its contract. +non-cloud harness automatically on PR updates and main pushes. Same-repository +PRs from non-Dependabot actors also run the complete cloud suite automatically, +and `workflow_dispatch` remains available. The #741 assertions remain strict +failures in those PR runs. Local cloud runs and local red regressions also require explicit commands; the ordinary local test suite remains usable while the fix is designed. @@ -109,8 +111,12 @@ gh workflow run lmi-e2e-tests.yml --repo aws/aws-durable-execution-sdk-python \ --ref YOUR_BRANCH -f region=us-west-2 ``` -CI first runs the local harness, then assumes the repository's configured test -role and deploys the checked-out SDK and OTel wheels. It uploads the raw S3/service +CI runs same-repository and manually dispatched jobs on the existing +`github-actions-runner` CodeBuild project. Fork and Dependabot PR harness checks +use GitHub-hosted runners. Python 3.14 comes from `setup-uv`; the runner image's +shared pip target and Python search path are disabled so Hatch environments stay +isolated. CI first runs the local harness, then assumes the repository's configured +test role and deploys the checked-out SDK and OTel wheels. It uploads the raw S3/service records, shared manifests, attempt journals, coverage inventories, and JSON/JUnit reports under `lmi_tests/artifacts/shared/`. Missing placement or source evidence is a failure, not an accepted skip. Local integration checks exercise the real @@ -288,14 +294,14 @@ An unexecuted scenario is never summarized as passing. The dedicated workflow runs the non-cloud harness when a PR is opened, updated, or reopened (including Draft PRs), and on every push to `main`. There -are no path filters or ready-for-review requirements for that harness. The cloud -job runs only with explicit `workflow_dispatch`; there are no scheduled jobs. -This keeps the deliberately failing #741 assertions from blocking every unrelated -PR/main build. Enable automatic cloud execution only after the implementation -satisfies those assertions; the manual cloud job still reports real failures. - -Each manual workflow run has one harness job and one cloud job using the persistent -resources. All cloud jobs share the repository-wide +are no path filters or ready-for-review requirements. After the harness passes, +same-repository PRs from non-Dependabot actors automatically run the cloud job. +Manual `workflow_dispatch` runs also execute both jobs; main pushes run only the +harness. There are no scheduled jobs. The #741 regressions stay ordinary failures, +so an affected SDK can produce a failing PR check. + +Each eligible PR or manual workflow run has one harness job and one cloud job +using the persistent resources. All cloud jobs share the repository-wide `lmi-e2e-shared-capacity-provider` concurrency group. Only one cloud job can deploy, test, or clean up at a time; the slot is held until the entire job finishes. Harness jobs can run @@ -308,9 +314,8 @@ cancels additional arrivals when that queue is full. This repository-wide group does not coordinate manual local deployments or jobs in other repositories using the same provider; those consumers must also leave sufficient capacity available. -PR events, including forks and Dependabot, run only the harness. A cloud run must -be dispatched explicitly by a trusted repository actor; Dependabot remains excluded. -Cloud jobs reuse +Fork and Dependabot PRs run only the harness on GitHub-hosted runners. Other PRs +and manual runs use CodeBuild. Cloud jobs reuse `TEST_ROLE_ARN`, `TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. The #741 regression assertions remain visibly failing until its fix lands; they are not skipped or converted into expected-success results to keep CI green. diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index ad25c66b9..3041d4180 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -50,8 +50,11 @@ its own controls and executions. Persistent ownership uses conditional S3 writes outside the expiring run prefix and is retained after unproven retirement. S3 stays private and uses native SigV4/IAM authentication. -CI retains the existing opt-in cloud policy and SDK-local unit/regression tests, -but invokes the shared runner for cloud acceptance. Reports and all attempted +CI runs the shared cloud suite automatically for same-repository PRs from +non-Dependabot actors and retains manual dispatch. Both jobs use the existing +CodeBuild runner; fork and Dependabot PRs run only the GitHub-hosted harness. +SDK-local unit/regression tests remain available, while cloud acceptance uses the +shared runner. Reports and all attempted case evidence are retained. The implementation and tests use the real public SDK handler/plugin paths against the local service where practical; real-LMI placement/recovery and deployed acceptance remain explicitly unverified until a diff --git a/lmi_tests/tests/unit/test_shared_workflow.py b/lmi_tests/tests/unit/test_shared_workflow.py index a836a20b8..135301fe2 100644 --- a/lmi_tests/tests/unit/test_shared_workflow.py +++ b/lmi_tests/tests/unit/test_shared_workflow.py @@ -44,3 +44,59 @@ def test_cloud_workflow_uses_pinned_shared_acceptance_and_final_cleanup(): ]["lmi"]["scripts"] assert scripts["shared"].startswith("python -m lmi_tests.shared run") assert scripts["shared-cleanup"].startswith("python -m lmi_tests.shared cleanup") + + +def test_codebuild_runner_has_isolated_python_and_keeps_untrusted_prs_hosted(): + workflow = yaml.safe_load( + (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() + ) + assert workflow["jobs"]["harness"]["runs-on"] == ( + "${{ github.event_name == 'pull_request' && " + "(github.event.pull_request.head.repo.full_name != github.repository || " + "github.actor == 'dependabot[bot]') && 'ubuntu-latest' || " + "format('codebuild-github-actions-runner-{0}-{1}', " + "github.run_id, github.run_attempt) }}" + ) + assert workflow["jobs"]["cloud"]["runs-on"] == ( + "codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}" + ) + assert workflow["env"]["PIP_CONFIG_FILE"] == "/dev/null" + assert workflow["env"]["PYTHONPATH"] == "" + for job in workflow["jobs"].values(): + setup = next( + step + for step in job["steps"] + if step.get("uses", "").startswith("astral-sh/setup-uv@") + ) + assert setup["with"]["python-version"] == "3.14" + assert setup["with"]["activate-environment"] is True + assert any( + step.get("run") == "uv pip install hatch==1.16.5" for step in job["steps"] + ) + assert not any( + step.get("uses", "").startswith("actions/setup-python@") + for step in job["steps"] + ) + + +def test_cloud_runs_automatically_for_trusted_prs_and_supports_manual_dispatch(): + workflow = yaml.safe_load( + (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() + ) + triggers = workflow.get("on", workflow.get(True)) + assert set(triggers["pull_request"]["types"]) == { + "opened", + "synchronize", + "reopened", + } + assert "workflow_dispatch" in triggers + cloud = workflow["jobs"]["cloud"] + assert cloud["if"] == ( + "github.actor != 'dependabot[bot]' && " + "(github.event_name == 'workflow_dispatch' || " + "(github.event_name == 'pull_request' && " + "github.event.pull_request.head.repo.full_name == github.repository))" + ) + assert cloud["needs"] == "harness" + assert cloud["concurrency"]["group"] == "lmi-e2e-shared-capacity-provider" + assert cloud["concurrency"]["cancel-in-progress"] is False From d835bcc387c4de0bdf33b245f31ccc7dbb17eb8b Mon Sep 17 00:00:00 2001 From: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 12:48:36 -0700 Subject: [PATCH 18/32] fix(lmi): consume corrected conformance evidence --- .github/workflows/lmi-e2e-tests.yml | 2 +- lmi_tests/README.md | 14 +- lmi_tests/shared-suite-design.md | 26 +- lmi_tests/shared_adapter.py | 27 ++ lmi_tests/shared_contract.py | 2 +- lmi_tests/shared_evidence.py | 87 ++++++- .../tests/unit/test_shared_evidence_fixes.py | 245 ++++++++++++++++++ .../unit/test_shared_partial_evidence.py | 160 ++++++++++++ 8 files changed, 554 insertions(+), 9 deletions(-) create mode 100644 lmi_tests/tests/unit/test_shared_evidence_fixes.py create mode 100644 lmi_tests/tests/unit/test_shared_partial_evidence.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 8e059865c..f4a9ffbf2 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -19,7 +19,7 @@ env: # Ignore the CodeBuild image's shared pip target and Python search path. PIP_CONFIG_FILE: /dev/null PYTHONPATH: "" - CONFORMANCE_REVISION: 6f899679c7eb6ede836c737acb9816e6fd7072fd + CONFORMANCE_REVISION: 86bcaff49829351b25a1b1ae76f6d6483d37df83 LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} jobs: diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 6732a751b..193381c5d 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -2,7 +2,7 @@ The cloud entry point is now the shared runner from [conformance PR #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186), -pinned to `6f899679c7eb6ede836c737acb9816e6fd7072fd`. The Python adapter runs +pinned to `86bcaff49829351b25a1b1ae76f6d6483d37df83`. The Python adapter runs `python-lifecycle` (76 cells) and `export-lifecycle` (24 cells) against the same persistent c1/c2 functions. The latter covers both OTel views with synchronous export and asynchronous export with a bounded flush wait. Shared requirements, @@ -31,7 +31,7 @@ Use Python 3.14 and Hatch from the repository root: ```sh # Check out the exact shared runner revision first. gh repo clone aws/aws-durable-execution-conformance-tests .conformance -git -C .conformance checkout 6f899679c7eb6ede836c737acb9816e6fd7072fd +git -C .conformance checkout 86bcaff49829351b25a1b1ae76f6d6483d37df83 hatch run lmi:shared-setup hatch run lmi:shared-tests hatch run lmi:lint @@ -125,9 +125,13 @@ public SDK and local checkpoint service; they do not establish LMI recovery. A non-expiring conditional S3 ownership record under `ownership//` fences updates to the shared functions. Case controls live under separate run/case prefixes. Normal completion releases ownership only when both profiles prove -quiescence. Interrupted or unproven retirement retains ownership for reconciliation; -`shared-cleanup` releases only that run's recorded controls and executions and -cannot turn an unknown retirement into success. Do not delete the record merely +quiescence. A failed case retains its original observations and errors. Available partial +observations are captured before fault release; a separate final snapshot can +prove that every actual launch and owned task/request/export has settled. Only +that independent audit permits later cases to reuse the target. Interrupted or +unproven retirement retains ownership for reconciliation; `shared-cleanup` +releases only that run's recorded controls and executions and cannot turn an +unknown retirement into success. Do not delete the record merely to bypass a blocked deployment. The fixture's asynchronous processor waits up to five seconds through the real diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index 3041d4180..653631460 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -70,7 +70,7 @@ claim to fix the open runtime deadline/cleanup defects tracked by Python #741. ## Reviewable decisions and limits - This PR targets current main, reuses the Python #743 harness, and does not modify that PR's branch. -- Conformance is pinned to PR #186 revision `6f899679c7eb6ede836c737acb9816e6fd7072fd`. +- Conformance is pinned to PR #186 revision `86bcaff49829351b25a1b1ae76f6d6483d37df83`. - The default target is the Python-specific 76-cell matrix plus the portable 24-cell exporter matrix; it does not imply ordinary SDK/OTel suite acceptance on LMI or acceptance against the separate standard profile. @@ -93,3 +93,27 @@ Native timeout and wrapper return are independent outcomes. Deadline measurement are frozen before cleanup releases the held step; cleanup cannot repair failed behavior assertions. The shared error expectations explicitly enumerate allowed service statuses where a measurement may precede execution termination. + +## Retained cloud evidence corrections + +The shared reader selects the declared held-work gate when the Python fixture +also records a winner-admission handshake. Execution-view operation spans that +cross a resume need matching service operation identity, status and timestamps; +the adapter retains the first actual service start and final terminal timestamp. +Per-request export records remain intact even when one logical span is delivered +again. Contradictory span observations remain failures. + +Deadline snapshots are read from the persisted pre-release measurement and bound +to the original case, execution, selected requests and declared time budgets. +The full event ledger remains available. Measurement does not authorize new user +work or SDK dispatch after the deadline cutoff; completion of already dispatched +requests is accounted for separately from the controller's normal fault release. + +A failed phase can provide a read-only partial collection before release. Its +original errors and incomplete behavior evidence stay in the report. The adapter +also supplies a separate post-cleanup retirement snapshot, with the actual final +read boundary. The shared coordinator independently checks every journaled launch, +known resource and completed task/request/export before allowing reuse. A missing +required role remains a test failure, while an unidentifiable actual launch or +unsettled work retains quarantine. An adapter's boolean retirement result alone +cannot make a failed case safe or turn it into a passing case. diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py index 6177ec6f6..9c67a9363 100644 --- a/lmi_tests/shared_adapter.py +++ b/lmi_tests/shared_adapter.py @@ -10,6 +10,7 @@ from copy import deepcopy import hashlib from pathlib import Path +import time from typing import Any, Protocol from aws_durable_execution_conformance_tests.lmi.coordinator import ( @@ -191,6 +192,29 @@ def collect() -> dict[str, Any]: return await case.call(collect) + async def collect_partial(self, context: ScenarioContext) -> dict[str, Any]: + """Retain a failed phase's facts before release, without awaiting cleanup.""" + case = self._case(context) + + def capture() -> dict[str, Any]: + cloud = case.driver.cloud + read_error = None + try: + cloud.collect(full_run=False) + except Exception as error: + read_error = ( + f"Partial source collection: {type(error).__name__}: {error}" + ) + cloud.observation_ended_at = time.time() + partial = self.evidence_builder(context, cloud, phase="observations") + if read_error is not None: + partial.setdefault("collectionErrors", []).append(read_error) + partial.setdefault("collection", {})["watermarks"] = {} + cloud._save("shared-partial-evidence.json", partial) + return partial + + return await case.call(capture) + async def release(self, context: ScenarioContext) -> None: case = self.cases.get(context.scenario.case_id) if case is None: @@ -210,6 +234,9 @@ def retire() -> bool: # The coordinator freezes observation evidence before fault release. # Only actual retirement facts may be appended by this phase. context.evidence["retirement"] = final.get("retirement", {}) + # Fresh completion facts prove physical retirement separately; they + # must never replace the case's pre-release behavior observations. + context.retirement_evidence = final return complete return await case.call(retire) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py index d8d273e7e..367ef7fa6 100644 --- a/lmi_tests/shared_contract.py +++ b/lmi_tests/shared_contract.py @@ -10,7 +10,7 @@ CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" # Updated only with the matching reviewed shared evidence contract. -CONFORMANCE_REVISION = "6f899679c7eb6ede836c737acb9816e6fd7072fd" +CONFORMANCE_REVISION = "86bcaff49829351b25a1b1ae76f6d6483d37df83" def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py index f7f3034d2..282a1825a 100644 --- a/lmi_tests/shared_evidence.py +++ b/lmi_tests/shared_evidence.py @@ -201,6 +201,25 @@ def histories( if "ParentId" in raw: operation["parentId"] = raw["ParentId"] details = raw.get(event_type + "Details", {}) + # Service operation lifetime is independent of native invocation + # and exporter timing. A retry cannot replace its original start. + if ( + event_type in {prefix + "Started" for prefix in _OPERATION_TYPES} + and raw.get("EventTimestamp") is not None + ): + started_at = _milliseconds(raw["EventTimestamp"]) + operation["startedAtMs"] = min( + operation.get("startedAtMs", started_at), started_at + ) + if ( + any( + event_type.endswith(suffix) + for suffix in ("Succeeded", "Failed", "TimedOut", "Stopped") + ) + and raw.get("EventTimestamp") is not None + ): + ended_at = _milliseconds(raw["EventTimestamp"]) + operation["endedAtMs"] = ended_at if event_type.endswith("Succeeded"): operation["status"] = "SUCCEEDED" result = details.get("Result", info.get("serializedResult")) @@ -213,6 +232,8 @@ def histories( ) elif event_type.endswith("TimedOut"): operation["status"] = "TIMED_OUT" + elif event_type.endswith("Stopped"): + operation["status"] = "STOPPED" result = self.cloud.execution_results[arn] started = result.get("StartTimestamp") if started is None: @@ -431,6 +452,57 @@ def _quiescent(request: str, events: list[dict[str, Any]], *, telemetry: bool) - ) +def _deadline_measurement( + context: ScenarioContext, cloud: Cloud, selected: Mapping[str, str] +) -> tuple[dict[str, Any] | None, list[str]]: + """Read the persisted pre-release measurement without discarding full events.""" + path = cloud.artifacts_dir / "measurements/deadline.json" + in_memory = getattr(cloud, "deadline_measurement", None) + if not path.exists(): + if in_memory is not None: + return None, ["Pre-release deadline measurement file is missing"] + return None, [] # Legacy sources retain the strict full-tail validator. + try: + recorded = json.loads(path.read_text()) + victim = next( + item for item in cloud.invocations if item.get("role") == "victim" + ) + expected = { + "runId": context.run_id, + "caseId": context.scenario.case_id, + "cohortId": context.scenario.cohort_id, + "scenario": context.scenario.scenario, + "executionArn": victim["arn"], + "selectedRequests": dict(selected), + } + if not isinstance(recorded, dict) or any( + recorded.get(key) != value for key, value in expected.items() + ): + raise ValueError( + "stored measurement belongs to another run, case or original cohort" + ) + fields = ( + "selectedRequests", + "deadlineAtMs", + "cleanupCutoffAtMs", + "observationCutoffAtMs", + "observedAtMs", + ) + if in_memory is not None and ( + not isinstance(in_memory, Mapping) + or any(in_memory.get(key) != recorded.get(key) for key in fields) + ): + raise ValueError( + "in-memory and persisted measurement identities/bounds differ" + ) + return { + **{key: recorded[key] for key in fields}, + "sourceEvidenceUri": path.resolve().as_uri(), + }, [] + except (OSError, ValueError, KeyError, StopIteration, TypeError) as error: + return None, [f"Invalid pre-release deadline measurement: {error}"] + + def build_evidence( context: ScenarioContext, cloud: Cloud, *, phase: str = "observations" ) -> dict[str, Any]: @@ -440,7 +512,11 @@ def build_evidence( attempts, selected, attempt_errors = sources.attempts(events) errors.extend(history_errors + attempt_errors) _causal_order(events) - ended = (cloud.observation_ended_at or 0) * 1000 + # The final snapshot describes actual retirement collection, independently + # of a failed phase's earlier (possibly incomplete) observation window. + ended = ( + (cloud.retired_at if phase == "final" else cloud.observation_ended_at) or 0 + ) * 1000 complete = bool(attempts) and all( _quiescent( item["requestId"], events, telemetry=context.scenario.otel_view is not None @@ -538,6 +614,13 @@ def build_evidence( "late-operation", }: evidence["regression"]["gateId"] = victim["marker"] + suffix + if context.scenario.scenario in _ACTIVE_FAULTS: + measurement, measurement_errors = _deadline_measurement( + context, cloud, selected + ) + errors.extend(measurement_errors) + if measurement is not None: + evidence["regression"]["deadlineMeasurement"] = measurement if context.scenario.scenario in {"warm-lifecycle", "otel-warm-lifecycle"}: evidence["regression"]["warmSequence"] = [ {"executionArn": item["arn"], "fixture": item["payload"]["scenario"]} @@ -683,5 +766,7 @@ def build_evidence( for fault in faults ], } + if errors: + evidence["collection"]["watermarks"] = {} cloud._save(f"shared-{phase}-evidence.json", evidence) return evidence diff --git a/lmi_tests/tests/unit/test_shared_evidence_fixes.py b/lmi_tests/tests/unit/test_shared_evidence_fixes.py new file mode 100644 index 000000000..b66e74ac9 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_evidence_fixes.py @@ -0,0 +1,245 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Service lifetimes and frozen measurements retain independently observed facts.""" + +from copy import deepcopy +import json +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from lmi_tests.shared_evidence import CaseSources, _deadline_measurement, build_evidence + + +def sources(raw): + context = SimpleNamespace( + scenario=SimpleNamespace(case_id="case", scenario="native-concurrency") + ) + cloud = SimpleNamespace( + invocations=[ + {"arn": "execution", "marker": "marker", "payload": {"scenario": "success"}} + ], + raw_histories={"execution": raw}, + execution_results={"execution": {"Status": "SUCCEEDED", "Result": '"marker"'}}, + ) + return CaseSources(context, cloud, ()) + + +@pytest.mark.parametrize( + "family", ["Step", "Wait", "Callback", "Context", "ChainedInvoke"] +) +def test_operation_lifetime_comes_from_its_service_events(family): + raw = [ + { + "EventType": family + "Started", + "Id": "op", + "Name": "controlled", + "EventTimestamp": 1.25, + }, + {"EventType": family + "Succeeded", "Id": "op", "EventTimestamp": 2.75}, + ] + history, _, errors = sources(raw).histories([]) + assert errors == [] + assert history[0]["operations"][0]["startedAtMs"] == 1250 + assert history[0]["operations"][0]["endedAtMs"] == 2750 + + +def test_retry_events_cannot_move_the_original_operation_start(): + raw = [ + { + "EventType": "StepStarted", + "Id": "op", + "Name": "controlled", + "EventTimestamp": 1, + }, + {"EventType": "StepRetryStarted", "Id": "op", "EventTimestamp": 2}, + {"EventType": "StepFailed", "Id": "op", "EventTimestamp": 3}, + {"EventType": "StepStarted", "Id": "op", "EventTimestamp": 4}, + {"EventType": "StepSucceeded", "Id": "op", "EventTimestamp": 5}, + ] + history, _, _ = sources(raw).histories([]) + operation = history[0]["operations"][0] + assert (operation["startedAtMs"], operation["endedAtMs"], operation["status"]) == ( + 1000, + 5000, + "SUCCEEDED", + ) + + +def test_missing_original_service_timestamp_stays_missing(): + raw = [ + {"EventType": "StepStarted", "Id": "op", "Name": "controlled"}, + {"EventType": "StepRetryStarted", "Id": "op", "EventTimestamp": 2}, + {"EventType": "StepStopped", "Id": "op", "EventTimestamp": 3}, + ] + history, _, _ = sources(raw).histories([]) + operation = history[0]["operations"][0] + assert "startedAtMs" not in operation + assert operation["endedAtMs"] == 3000 + assert operation["status"] == "STOPPED" + + +@pytest.fixture +def measurement(tmp_path): + # Preserve the archived c2 baseline's observed deadline/measurement deltas. + selected = {"victim": "original", "retry": "retry"} + record = { + "runId": "run", + "caseId": "case", + "cohortId": "cohort-1", + "scenario": "timeout-retry", + "executionArn": "execution", + "selectedRequests": selected, + "deadlineAtMs": 1794527680692.2056, + "cleanupCutoffAtMs": 1794527685692.2056, + "observationCutoffAtMs": 1794527686692.2056, + "observedAtMs": 1794527688724.1567, + "events": [{"kind": "effect", "atMs": 1794527685892.097}], + "histories": {"execution": []}, + "sourceUris": ["memory://frozen-source"], + } + path = tmp_path / "measurements/deadline.json" + path.parent.mkdir() + path.write_text(json.dumps(record)) + context = SimpleNamespace( + run_id="run", + scenario=SimpleNamespace( + case_id="case", cohort_id="cohort-1", scenario="timeout-retry" + ), + ) + cloud = SimpleNamespace( + artifacts_dir=tmp_path, + deadline_measurement=deepcopy(record), + invocations=[{"role": "victim", "arn": "execution"}], + ) + return context, cloud, selected, record, path + + +def test_persisted_measurement_exports_only_identity_bounds_and_source_reference( + measurement, +): + context, cloud, selected, record, path = measurement + value, errors = _deadline_measurement(context, cloud, selected) + assert errors == [] + assert value == { + **{ + key: record[key] + for key in ( + "selectedRequests", + "deadlineAtMs", + "cleanupCutoffAtMs", + "observationCutoffAtMs", + "observedAtMs", + ) + }, + "sourceEvidenceUri": path.resolve().as_uri(), + } + assert "events" not in value + + +@pytest.mark.parametrize( + "field", + ["runId", "caseId", "cohortId", "scenario", "executionArn", "selectedRequests"], +) +def test_measurement_cannot_be_substituted_between_cases_or_original_requests( + measurement, field +): + context, cloud, selected, record, path = measurement + record[field] = ( + {"victim": "replacement"} if field == "selectedRequests" else "other" + ) + path.write_text(json.dumps(record)) + value, errors = _deadline_measurement(context, cloud, selected) + assert value is None and errors + + +def test_measurement_must_match_the_record_written_before_release(measurement): + context, cloud, selected, _, _ = measurement + cloud.deadline_measurement["observedAtMs"] += 1 + value, errors = _deadline_measurement(context, cloud, selected) + assert value is None and errors + + +def test_missing_measurement_is_legacy_only_if_no_snapshot_was_recorded(measurement): + context, cloud, selected, _, path = measurement + path.unlink() + assert _deadline_measurement(context, cloud, selected)[1] + cloud.deadline_measurement = None + assert _deadline_measurement(context, cloud, selected) == (None, []) + + +def test_reloading_archived_measurement_does_not_require_the_old_driver_process( + measurement, +): + context, cloud, selected, _, _ = measurement + cloud.deadline_measurement = None + value, errors = _deadline_measurement(context, cloud, selected) + assert not errors and value["selectedRequests"] == selected + + +def test_assembler_keeps_post_measurement_events_in_the_main_evidence(measurement): + context, cloud, selected, record, _ = measurement + context.scenario = SimpleNamespace( + case_id="case", + cohort_id="cohort-1", + scenario="timeout-retry", + requirement_id="lmi-lifecycle-11", + coverage_profile="python-lifecycle", + otel_view=None, + export_mode=None, + behavior_budgets={ + "cleanupMs": 5000, + "observationGraceMs": 1000, + "retryMs": 40000, + }, + ) + context.profile = SimpleNamespace( + artifact_id="sdk", + configuration_id="config", + resource_id="function", + native_concurrency=2, + recovery_policy="reuse-worker", + ) + cloud.manifest = {"conformanceRevision": "conformance", "invocationTimeout": 60} + cloud.observation_ended_at = 1794527693 + cloud.retired_at = cloud.observation_ended_at + cloud.source_uris = {"memory://source"} + cloud.control_records = [] + cloud._save = lambda *args: None + cloud.invocations[0]["marker"] = "marker" + cloud.invocations[0]["payload"] = {"scenario": "deadline"} + events = [ + { + "kind": "effect", + "requestId": "original", + "ownerRequestId": "original", + "executionArn": "execution", + "atMs": record["cleanupCutoffAtMs"] + 200, + }, + { + "kind": "checkpoint_exit", + "requestId": "original", + "ownerRequestId": "original", + "executionArn": "execution", + "atMs": record["observedAtMs"] + 1000, + }, + ] + captured = SimpleNamespace( + observations=lambda: (deepcopy(events), []), + histories=lambda _: ([], {}, []), + attempts=lambda _: ( + [{"requestId": "original", "executionArn": "execution"}], + selected, + [], + ), + ) + with patch("lmi_tests.shared_evidence.CaseSources.capture", return_value=captured): + value = build_evidence(context, cloud) + assert [event["atMs"] for event in value["events"]] == [ + event["atMs"] for event in events + ] + assert ( + value["regression"]["deadlineMeasurement"]["observedAtMs"] + == record["observedAtMs"] + ) diff --git a/lmi_tests/tests/unit/test_shared_partial_evidence.py b/lmi_tests/tests/unit/test_shared_partial_evidence.py new file mode 100644 index 000000000..3aa3ea5d0 --- /dev/null +++ b/lmi_tests/tests/unit/test_shared_partial_evidence.py @@ -0,0 +1,160 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Partial observations remain separate from facts collected after cleanup.""" + +import asyncio +from copy import deepcopy +from types import SimpleNamespace +from unittest.mock import Mock, patch + + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from lmi_tests.shared_adapter import SharedAdapter, _Case +from lmi_tests.shared_evidence import build_evidence + + +def context(): + profile = Profile("c1", "function", 1, 1, "sdk", "configuration") + return ScenarioContext( + "run", + Scenario( + "case", + "lmi-lifecycle-11", + "c1", + "timeout-retry", + roles=("victim", "retry"), + coverage_profile="python-lifecycle", + ), + profile, + ) + + +def adapter(tmp_path, *, read_fails=False): + calls = [] + cloud = SimpleNamespace(observation_ended_at=None) + + def read(**options): + calls.append(("read", options)) + if read_fails: + raise OSError("source unavailable") + + cloud.collect = read + cloud._save = lambda name, value: calls.append(("save", name, deepcopy(value))) + cloud.begin_retirement = lambda: calls.append(("retirement",)) + cloud.release_all = lambda: calls.append(("release",)) + driver = SimpleNamespace( + cloud=cloud, + collect=Mock(side_effect=AssertionError("must not wait for quiescence")), + retire=lambda: True, + ) + + def build(ctx, observed, *, phase): + calls.append(("build", phase)) + return { + "events": [{"kind": "effect"}] + if phase == "observations" + else [{"kind": "effect"}, {"kind": "task_exit"}], + "collection": {"watermarks": {"lifecycle": 1}}, + "collectionErrors": [], + **({"retirement": {"resources": []}} if phase == "final" else {}), + } + + result = SharedAdapter( + {"sdk": {"deployment": {}, "artifacts_dir": str(tmp_path)}}, + ownership=Mock(), + evidence_builder=build, + ) + result.cases["case"] = _Case(driver) + return result, cloud, calls + + +def test_partial_capture_reads_once_without_waiting_for_cleanup(tmp_path): + value, cloud, calls = adapter(tmp_path) + partial = asyncio.run(value.collect_partial(context())) + assert [call[0] for call in calls] == ["read", "build", "save"] + assert calls[0] == ("read", {"full_run": False}) + assert calls[-1][1] == "shared-partial-evidence.json" + assert cloud.observation_ended_at is not None + assert partial["events"] == [{"kind": "effect"}] + + +def test_partial_read_error_keeps_cached_observations_but_no_complete_watermark( + tmp_path, +): + value, _, _ = adapter(tmp_path, read_fails=True) + partial = asyncio.run(value.collect_partial(context())) + assert partial["events"] == [{"kind": "effect"}] + assert partial["collection"]["watermarks"] == {} + assert any("source unavailable" in error for error in partial["collectionErrors"]) + + +def test_sdk_retirement_does_not_replace_the_pre_release_events(tmp_path): + async def exercise(): + value, _, _ = adapter(tmp_path) + ctx = context() + ctx.evidence = deepcopy(await value.collect_partial(ctx)) + await value.release(ctx) + assert await value.retire(ctx) is True + assert ctx.evidence["events"] == [{"kind": "effect"}] + assert ctx.retirement_evidence["events"] == [ + {"kind": "effect"}, + {"kind": "task_exit"}, + ] + assert ctx.evidence["retirement"] == ctx.retirement_evidence["retirement"] + + asyncio.run(exercise()) + + +def test_final_read_bound_alone_does_not_invent_quiescence(tmp_path): + ctx = context() + ctx.scenario = Scenario( + "case", + "lmi-lifecycle-1", + "c1", + "native-concurrency", + roles=("victim",), + coverage_profile="python-lifecycle", + ) + cloud = SimpleNamespace( + manifest={"invocationTimeout": 60, "conformanceRevision": "conformance"}, + observation_ended_at=None, + retired_at=2, + source_uris={"memory://source"}, + control_records=[], + invocations=[], + artifacts_dir=tmp_path, + _save=lambda *args: None, + ) + event = { + "kind": "task_start", + "taskId": "root", + "requestId": "request", + "ownerRequestId": "request", + "executionArn": "execution", + "atMs": 1000, + } + sources = SimpleNamespace( + observations=lambda: ([event], []), + histories=lambda _: ([], {}, []), + attempts=lambda _: ( + [{"requestId": "request", "executionArn": "execution"}], + {"victim": "request"}, + [], + ), + ) + cloud.execution_results = {"execution": {"Status": "STOPPED"}} + with patch("lmi_tests.shared_evidence.CaseSources.capture", return_value=sources): + evidence = build_evidence(ctx, cloud, phase="final") + assert evidence["observationEndedAtMs"] == 2000 + assert evidence["collection"]["watermarks"] == {} + assert evidence["collectionErrors"] + invocation = next( + item + for item in evidence["retirement"]["resources"] + if item["kind"] == "invocation" + ) + assert invocation["outcome"] == "UNKNOWN" From 33fc193a133dd863c7c13385ba31ed9bf4d3edd3 Mon Sep 17 00:00:00 2001 From: Zhongke Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 13:35:49 -0700 Subject: [PATCH 19/32] test(lmi): own Python-specific regression contracts --- .github/workflows/lmi-e2e-tests.yml | 4 +- lmi_tests/README.md | 34 +- lmi_tests/python-lifecycle-schema.md | 155 +++ lmi_tests/python-otel.md | 67 + lmi_tests/python_catalog.py | 612 +++++++++ lmi_tests/python_lifecycle.py | 1224 +++++++++++++++++ lmi_tests/python_otel.py | 740 ++++++++++ lmi_tests/shared-suite-design.md | 181 +-- lmi_tests/shared.py | 28 +- lmi_tests/shared_contract.py | 2 +- .../tests/e2e/test_shared_driver_local.py | 2 +- .../tests/e2e/test_shared_evidence_local.py | 4 +- lmi_tests/tests/unit/test_python_lifecycle.py | 1189 ++++++++++++++++ lmi_tests/tests/unit/test_python_matrix.py | 248 ++++ lmi_tests/tests/unit/test_python_otel.py | 1157 ++++++++++++++++ lmi_tests/tests/unit/test_python_profile.py | 319 +++++ lmi_tests/tests/unit/test_shared_adapter.py | 6 +- .../unit/test_shared_deadline_measurement.py | 2 +- .../tests/unit/test_shared_evidence_fixes.py | 2 +- lmi_tests/tests/unit/test_shared_manifest.py | 13 +- .../unit/test_shared_partial_evidence.py | 4 +- lmi_tests/tests/unit/test_shared_workflow.py | 34 + 22 files changed, 5887 insertions(+), 140 deletions(-) create mode 100644 lmi_tests/python-lifecycle-schema.md create mode 100644 lmi_tests/python-otel.md create mode 100644 lmi_tests/python_catalog.py create mode 100644 lmi_tests/python_lifecycle.py create mode 100644 lmi_tests/python_otel.py create mode 100644 lmi_tests/tests/unit/test_python_lifecycle.py create mode 100644 lmi_tests/tests/unit/test_python_matrix.py create mode 100644 lmi_tests/tests/unit/test_python_otel.py create mode 100644 lmi_tests/tests/unit/test_python_profile.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index f4a9ffbf2..82ab581bf 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -19,7 +19,7 @@ env: # Ignore the CodeBuild image's shared pip target and Python search path. PIP_CONFIG_FILE: /dev/null PYTHONPATH: "" - CONFORMANCE_REVISION: 86bcaff49829351b25a1b1ae76f6d6483d37df83 + CONFORMANCE_REVISION: 59352e1097d92fe034c7cd55a9618837b95b905b LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} jobs: @@ -128,7 +128,7 @@ jobs: role-session-name: python-lmi-assertions aws-region: ${{ env.AWS_REGION }} allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Run shared lifecycle and exporter conformance + - name: Run Python lifecycle regressions and shared exporter conformance timeout-minutes: 60 run: hatch run lmi:shared - name: Refresh credentials for evidence and cleanup diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 193381c5d..a81343d8c 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -1,13 +1,17 @@ # Python LMI shared conformance adapter -The cloud entry point is now the shared runner from +The cloud entry point runs two separately owned suites against the same persistent +c1/c2 functions. The Python SDK owns `python-lifecycle`: 28 lifecycle cells plus +48 instrumented cells, using `python-lmi-*` requirement IDs. Their definitions, +matrices and validators live in `python_catalog.py`, `python_lifecycle.py`, and +`python_otel.py`. They are Python regressions, not shared conformance requirements. + +The language-neutral `export-lifecycle` profile contributes another 24 cells from [conformance PR #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186), -pinned to `86bcaff49829351b25a1b1ae76f6d6483d37df83`. The Python adapter runs -`python-lifecycle` (76 cells) and `export-lifecycle` (24 cells) against the same -persistent c1/c2 functions. The latter covers both OTel views with synchronous -export and asynchronous export with a bounded flush wait. Shared requirements, -validation, JSON/JUnit reports and ownership journals come from that pinned -checkout; the conformance packages are installed on the driver only. +pinned to `59352e1097d92fe034c7cd55a9618837b95b905b`. Both suites reuse its generic +coordinator, launch journals, common evidence/retirement audit, and JSON/JUnit +reporting. Only the 24 exporter cells count toward that shared profile; these runs +do not claim acceptance of the separate shared `standard` profile. This suite implements [#742](https://github.com/aws/aws-durable-execution-sdk-python/issues/742) and asserts the invocation-lifetime behavior requested by @@ -31,7 +35,7 @@ Use Python 3.14 and Hatch from the repository root: ```sh # Check out the exact shared runner revision first. gh repo clone aws/aws-durable-execution-conformance-tests .conformance -git -C .conformance checkout 86bcaff49829351b25a1b1ae76f6d6483d37df83 +git -C .conformance checkout 59352e1097d92fe034c7cd55a9618837b95b905b hatch run lmi:shared-setup hatch run lmi:shared-tests hatch run lmi:lint @@ -99,7 +103,7 @@ any evidence you need before starting another run. The run manifest is invalidat first, so a failed build, invalid configuration, or account/provider check cannot make an exit trap collect or clean up the preceding run. This deployment supplies both native environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud -command runs 100 shared coverage cells. The shared coordinator reserves native +command runs 76 Python-owned regression cells and 24 shared exporter cells. The shared coordinator reserves native capacity for each case and serializes exclusive cases, without redeploying or changing function configuration between cases. Independent SDK branch-concurrency settings include 1, 2, and 3. @@ -117,7 +121,7 @@ use GitHub-hosted runners. Python 3.14 comes from `setup-uv`; the runner image's shared pip target and Python search path are disabled so Hatch environments stay isolated. CI first runs the local harness, then assumes the repository's configured test role and deploys the checked-out SDK and OTel wheels. It uploads the raw S3/service -records, shared manifests, attempt journals, coverage inventories, and JSON/JUnit +records, suite manifests, attempt journals, coverage inventories, and JSON/JUnit reports under `lmi_tests/artifacts/shared/`. Missing placement or source evidence is a failure, not an accepted skip. Local integration checks exercise the real public SDK and local checkpoint service; they do not establish LMI recovery. @@ -139,7 +143,9 @@ provider flush API; shared acceptance allows ten seconds for that wait boundary. Export completion is tracked separately and remains owned after wrapper return. These are declared test budgets, not SDK SLAs. See [shared telemetry observations](shared-telemetry.md) for capture/publication -ordering and [the adapter design](shared-suite-design.md) for ownership details. +ordering, [Python lifecycle rules](python-lifecycle-schema.md) and +[Python OTel rules](python-otel.md) for SDK-owned assertions, and +[the adapter design](shared-suite-design.md) for suite ownership details. The first run creates the persistent `python-lmi-e2e` stack, one private bucket, and two functions/log groups (`python-lmi-e2e-c1`, `python-lmi-e2e-c2`). Later runs @@ -180,8 +186,8 @@ waiting for results, with finite boto transport timeouts. | Timeout/retry | A second service invocation of the **same** durable execution, completed step skipped, original attempt effects cease; repeated interrupted at-least-once work is recorded separately and is not called exactly-once | The following sections describe the retained SDK-local fixtures and legacy -pytest regression entry points. Cloud acceptance uses the shared manifests and -validators described above. Java-specific executor, handler-owner and Context/MDC +pytest regression entry points. The automated run uses the Python-owned and shared exporter suites described +above. Java-specific executor, handler-owner and Context/MDC assertions remain the Java SDK's responsibility. ### Java lifecycle coverage in Python @@ -282,7 +288,7 @@ transport timeout). Invocation and callback writes are not retried by this layer | Cleanup acceptance grace | 5 s | | Fault-fixture emergency I/O release | 150 s | | Per-case retirement after assertions | 30 s | -| Workflow cloud assertions (all 76 cases) | 60 min | +| Workflow cloud assertions (76 Python + 24 shared exporter cells) | 60 min | | Waiting for prior executions before updating | 270 s | | Single cloud job including deployment/run cleanup | 120 min | diff --git a/lmi_tests/python-lifecycle-schema.md b/lmi_tests/python-lifecycle-schema.md new file mode 100644 index 000000000..10b8b6166 --- /dev/null +++ b/lmi_tests/python-lifecycle-schema.md @@ -0,0 +1,155 @@ +# Python lifecycle regression observations + +This SDK-owned version 1 contract reuses the common evidence envelope and +retirement helpers with local `python-lmi-*` requirement IDs. It is validated by +`lmi_tests.python_catalog`, not registered in the shared CLI or its packaged schema. +It applies when `coverageProfile` is `python-lifecycle` and `runtime` is `python`, +and preserves the 14 SDK-owned base fixture variants on each native-concurrency profile +1 and 2. It is not a portable promise to terminate arbitrary non-cooperative I/O. +The standard LMI profile remains unchanged. + +The runner validates the common envelope, original launch ownership, manifest +identity, declared budgets, and final retirement independently of these scenario +assertions. `validate_lifecycle_regressions(evidence, phase=...)` validates each +base cell. `validate_lifecycle_events` also supplies the lifetime checks for OTel +cases, and `validate_replay_evidence(evidence, execution_arn)` validates one replay +execution within a warm sequence. OTel cells additionally use `lmi_tests.python_otel` and the shared canonical +OTel model; passing lifecycle checks alone cannot accept telemetry. + +## Common observations + +Each retained attempt adds its immutable fixture input `marker`; warm sequences +also record `fixture` (`success`, `failure`, or `replay`). Native-overlap evidence +records both the OS `pid` and the stable `processId`, plus the common environment, +worker, request and execution identities. Source-specific diagnostics belong in +the retained raw evidence rather than durable operation names. + +Every event uses the common `requestId`, `ownerRequestId`, `executionArn`, `kind` +and `atMs`, plus a nonnegative, unique per-request `sequence`. The canonical +sequence follows actual semantic boundary times and resolves equal timestamps +using observed causal order. It is **not necessarily the source's emission +index**. The Python fixture records actual SDK completion time and later emits a +retrospective wrapper diagnostic after OTel exit. An adapter must normalize that +record to the actual SDK boundary, retaining the original `sourceSequence`, +`observedAtMs` and source timestamps in raw evidence. Do not invent ordering from +missing observations or use a later resource snapshot to claim an earlier +boundary was quiescent. OTel envelopes use UTC Unix epoch milliseconds. + +Each request needs `wrapper_enter`, `user_exit`, and exactly one `wrapper_return` +or `wrapper_raise`, with the actual observed `outcome`. User exit must precede +wrapper exit. The native platform outcome and SDK wrapper outcome remain +independent after a proven `platform_timeout`: a process can later return or +raise (normalized wrapper outcome `FAILED`) after its native request timed out. +Retain that actual wrapper outcome and its error; do not relabel it `TIMED_OUT`. +Non-timeout requests still require matching outcomes. Task/request settlement, +clock, collection, and stale-activity assertions apply to both paths. Wrapper entry and exit carry `resources` snapshots: + +```json +{"threads": [{"id": "thread-id", "name": "MainThread"}], "fds": 12, "rssBytes": 1048576} +``` + +Thread identities must be unique. New `dex-`/`ThreadPoolExecutor` workers or +threads explicitly tagged with that invocation's `ownerRequestId` cannot survive +wrapper exit. This is a Python-profile diagnostic, not a universal thread naming +contract. Finite FD and RSS counters support the warm baseline checks. + +Tracked `task_start`/`task_exit` and `io_start`/`io_exit` pairs use stable `taskId`. +`checkpoint_call`/`checkpoint_exit` and `poll_call`/`poll_exit` pairs use `callId`. +Every observed start must settle before wrapper exit. `body`, `effect`, +`checkpoint_call`, `checkpoint_ack`, `checkpoint_exit`, `poll_call`, and `poll_exit` +are forbidden after that boundary, including acknowledgment/settlement of an +earlier request. `body` carries `operation`, `operationId`, `attempt` and `value`; +its operation ID/name must match the collected service history. Names may repeat +under distinct parent contexts; IDs remain authoritative. + +An actual timed-out invocation may lack a wrapper exit. It must then have an +observed `platform_timeout` with `source: platform`, plus actual root +`task_exit` (`taskId: root`) or `worker_retired` with `source: platform` under the +explicit `replace-worker` recovery policy. Deadline/recovery cases require an +explicit `reuse-worker` or `replace-worker` policy, bound to the manifest. Absence of a final log is not positive +retirement evidence. Recovery and final owned-resource retirement still apply. +`escape` and `control_error` records cannot establish successful SDK cleanup. + +All gates belong to `faultsCreated`. `blocked`, `effect`, `alive`, and `io_exit` +identify the `gateId`; `fault_released` records the same gate, actual time and +`mode: normal`. Emergency or unclassified releases fail the assertion. The +validator also examines final `retirement.faults`, so later release evidence +cannot erase an earlier failure. + +A timeout fixture also records the invocation's actual `deadlineAtMs`. An owned +`runtime_deadline` event captures `source: runtime`, `atMs`, and the runtime +context's positive `remainingTimeMs`. The remaining time cannot exceed the +deployed `profile.timeoutMs`; `deadlineAtMs` must equal that observation's time +plus remaining time, allowing at most one millisecond for timestamp rounding. +The platform timeout must cross this actual deadline. Cleanup/recovery/retry +budgets start there. Do not derive it from wrapper entry plus the configured full +timeout: native dispatch and setup may already have consumed invocation time. +An input-injected deadline or missing runtime observation cannot establish this +precondition. The configured timeout remains deployment identity. + +## Manifest-declared budgets + +`regression.budgets` must exactly match the case's manifest `behavior_budgets`. +The adapter cannot increase these limits after seeing observations. All base +cases require positive `observationGraceMs`, covered by collection after wrapper +exit. Additional fields are: + +| Cases | Additional declared budgets | +| --- | --- | +| `early-completion-parallel/map/nested` | `cleanupMs` | +| `deadline-recovery` | `cleanupMs`, `recoveryMs` | +| `timeout-retry` | `cleanupMs`, `retryMs` | +| `nested-progress` | `progressMs` | +| `warm-lifecycle` | Nonnegative `maxThreadGrowth`, `maxFdGrowth`, `maxRssGrowthBytes` | + +Budgets are fixture acceptance settings, not product SLAs. Runner provisioning, +placement, execution, collection and retirement budgets remain independent. + +## Required scenario evidence + +| ID and selector | Observations and assertions | +| --- | --- | +| 1, `native-concurrency` | c1 selects `victim`; c2 selects `victim` and `peer`. All selected requests block in one environment, with distinct execution/request/process/PID identities. `alive` or `effect` positively covers the overlap pivot; missing exits alone do not prove liveness. | +| 4, `deadline-recovery` | Original `victim` has controlled work blocked before its actual platform deadline. No original BODY/EFFECT/checkpoint activity survives deadline plus `cleanupMs`. c2 first establishes a healthy `peer`. A full recovered cohort (`recovery`, plus `recovery-peer` for c2) overlaps in the original environment within `recoveryMs`, before fault release. `reuse-worker` includes the original process/worker; `replace-worker` identifies it through `replacesWorkerId`. Recovery requests succeed. | +| 5, `replay-callback-retry` | At least one real PENDING request precedes a successful resume of the same execution. `success` and `failure` bodies each occur exactly once with the original marker and stable service operation ID. `stored_failure` is observed on at least two requests with normalized `{type: ValueError, message: expected:}`. `retry` body attempts are exactly integer `[1, 2]` (booleans are invalid), each with the original marker/ID. Actual history contains successful WAIT, CALLBACK, and CONTEXT operations. | +| 6, `warm-lifecycle` | `regression.warmSequence` lists nine distinct executions in `[success, failure, replay] * 3` order, each `{executionArn, fixture}`. Executions run sequentially; fixture routing and terminal outcomes agree. Replay executions satisfy the full replay validator. At least three wrapper samples reuse one environment/process. Compare samples after the second against that post-cache baseline using the declared thread/FD/RSS allowances. | +| 9, `early-completion-parallel/map/nested` | `regression.gateId` selects the held loser. An effect precedes `winner_ready`, with no prior loser exit; `winner_selected` follows. Wrapper return and cessation of activity occur within `cleanupMs`, before any driver release. Actual service history contains `race-parallel`, `race-map`, or nested `outer-map`/`nested-child`/`race-parallel` scopes respectively. | +| 10, `checkpoint-settlement` | A real `checkpoint_call` receives `checkpoint_ack` (`source: service`, `operationIds`) for the actual `checkpoint-loser` operation. `ack_held` carries the matching `callId`, `operationId`, and gate. After positive `blocked`, normal release precedes `checkpoint_exit`, then `branch_join` of the actual `checkpoint-pool` context, then wrapper exit. An unrelated acknowledgment cannot establish this precondition. | +| 11, `timeout-retry` | Actual selected timeout is followed within `retryMs` by selected `retry` on the same execution ARN. `service_retry` is observed from the platform and binds `previousRequestId` to the original request. The original successful step body occurs exactly once across all retained requests. Interrupted I/O effects are not given an exactly-once requirement. | +| 12, `late-operation` | `scope_closed` and `late_attempt` identify the same actual service child-context ID. Both observed closure and `winner_selected` precede the late attempt; their relative order is not prescribed. In particular, Python's public operation-end hook runs before the SDK call returns and the fixture emits `winner_selected`. The losing I/O settles before the late attempt. `late_rejected` binds the same scope with normalized `OrphanedChildException`. No `late_accepted`, late body, late checkpoint dispatch or `late-work` service operation is allowed. The invocation may still be open: child closure is a separate boundary. | +| 13, `return-inflight` / `failure-inflight` | Root `user_exit` occurs after winner selection while losing I/O is still held. Normal release/IO settlement precedes the actual wrapper exit. c2 positively observes peer progress during this interval. Preserve success or the original normalized `ValueError`/marker failure after settlement. | +| 14, `pending-cleanup` | Root `cleanup_enter` precedes the held gate; normal release and IO exit precede `cleanup_exit`, `user_exit`, then PENDING wrapper return. A later request resumes the same execution, completes a real WAIT, and does not repeat the completed success body. c2 peer progress is observed while cleanup is held. | +| 15, `nested-progress` | All native workers establish live overlap, then `progress_begin`/`progress` bound the nested operation budget. Each request returns its own `[[marker:0, marker:1], marker]` result. Actual `outer-parallel`, `inner-map`, and `progress-child` operations succeed. `operation_config` records the public fixture's integer `maxConcurrency: 1` for the two pool operations, bound by operation ID and `source: public-fixture`; this is not invented service-history metadata. | + +The six OTel-only selectors (`runtime-carrier-sampled`, +`runtime-carrier-unsampled`, `otel-warm-lifecycle`, `otel-concurrent`, +`export-error-recovery`, `held-export`) retain the common settled-lifecycle checks +and delegate their remaining assertions to the OTel extension. The SDK adapter +must implement these fixtures through real public APIs and preserve raw source +evidence. Synthetic positive controls validate the harness; they do not count as +real LMI SDK acceptance or authorize replacing Python-specific regression tests. + +For in-flight return/failure and root-cleanup peer measurements, +`regression.gateId` selects the victim's actual held-work gate. A separate +winner-admission handshake may produce additional `blocked` events, but cannot +supply that gate's liveness or prevent its early exit from being detected. Peer +and other native cohort roles still require one unambiguous blocked gate. + +A deadline fixture that normally releases its held work after measurement can +retain `regression.deadlineMeasurement`: `selectedRequests`, `deadlineAtMs`, +`cleanupCutoffAtMs`, `observationCutoffAtMs`, `observedAtMs`, and +`sourceEvidenceUri`. The selected requests and cutoffs must match the actual +runtime deadline and immutable case budgets. Its source URI must be retained in +`collection.evidenceUris`, and capture must cover the full cleanup-plus-grace +window before the fault's `releaseRequestedAtMs`. + +Only a valid snapshot permits the stale-activity window to end when the +controller begins normal release. Activity between capture and release is still +checked. New user bodies and SDK dispatch after the cutoff remain failures even +after release; only paired completion of already dispatched I/O is cleanup. +All events remain in the main ledger, including later normal cleanup, and task, +request, wrapper and retirement checks still inspect that complete ledger. +Missing snapshots retain the legacy full-window check; inconsistent snapshots +fail rather than shorten the observation window. This separates deliberately +released cleanup from the measured deadline behavior without removing actual +pre-release late effects. diff --git a/lmi_tests/python-otel.md b/lmi_tests/python-otel.md new file mode 100644 index 000000000..303370f0a --- /dev/null +++ b/lmi_tests/python-otel.md @@ -0,0 +1,67 @@ +# Python-owned OTel lifecycle checks + +The SDK-owned `python-lifecycle` suite defines 48 observability cells: twelve scenario selectors, both +OTel views, and deployed native concurrency 1 and 2. It supplements the 28 base +lifecycle cells. Each cell declares one `otelView` and collects exactly that view +under `telemetry.views`. The original generic LMI profile still collects both +views in a cohort. Python requirements use `python-lmi-*` IDs and are registered by +`lmi_tests.python_catalog.suite_definition()`. Local positive controls do not +establish real-LMI acceptance. + +`validate_lifecycle_telemetry` checks all plugin-enabled lifecycle scenarios, +including deadline, timeout retry, pending cleanup, late operation and root +return/failure with work in flight. Call it together with the SDK lifecycle +validator. The matrix fixes the required scenarios, views, configurations and +`behavior_budgets`; the adapter must not select weaker assertions after observing +results. `regression.budgets.exportMs` is the configured exporter-call bound, +alongside `observationGraceMs` and applicable lifecycle budgets. + +Use the existing owned-event fields (`requestId`, `ownerRequestId`, +`executionArn`, `atMs`, `sequence`) for these observations: + +| Event | Additional observed fields | +| --- | --- | +| `otel_enter` | `providerId`, `exporterId`, `contextBefore: {traceId, spanId}`; both context IDs may be null. | +| `otel_factory` | The actual `pluginInstanceId` and factory `pluginExecutionArn`. Python creates a distinct plugin instance per native request. | +| `otel_export_started` | Actual exporter entry with `traceId` and `spanId`. | +| `otel_export` | Matching exporter completion, with `outcome: success | failure | held`. | +| `otel_exit` | Post-call `contextAfter: {traceId, spanId}` and observed integer `exporterShutdowns`, which must be zero. | +| `export_held` / `export_released` | The same case-owned `gateId`; held export also binds `traceId` and `spanId`, and normal release records `mode: normal`. | + +The real boundary order is `otel_enter`, factory creation inside the SDK call, +export completion, actual SDK `wrapper_return`/`wrapper_raise`, then the +`otel_exit` diagnostic. The latter observes restored context after the SDK call; +it is not the SDK completion timestamp. An adapter may record the saved SDK +boundary after emitting a diagnostic, but must retain that original boundary +time. Normalize semantic event order separately from raw emission order and +retain raw emission sequence/timestamps in the referenced evidence. Held export +must prevent actual SDK completion until normal release, independently of when +post-call diagnostics arrive. Emergency release cannot establish this result. + +Every `telemetry.views[otelView].spans` record retains its independent export +request, a boolean `sampled` from the actual SpanContext, and the canonical +`span_to_dict` payload. Failed exporter attempts retain their actual canonical +spans and matching failure events; they do not claim backend delivery. Carrier +records use the genuine Python `process-environment` source. The sampled and +unsampled runtime cases additionally retain `callerHeader` from the actual Invoke +request, allowing the runtime Root and sampling decision to be compared with the +caller. Missing, empty, malformed and injected carriers are local negative +controls, never substitutes for genuine runtime coverage. + +The validators require one logical Invocation span per sampled request and one +terminal Workflow with its actual success/failure status in terminal fixtures. +Logical identities are counted across replay; identical transport re-exports are +retained without promising exactly-once export. A Workflow may start at the +service execution boundary before native admission and export from a terminal +invocation, retaining its original runtime parent. Its recorded service history +must establish that earlier boundary. Export failure must be followed by a +successful invocation in the same worker/process with the same provider. Held +peers must remain active and make progress after another request ends. + +`regression.warmSequence` identifies six distinct executions and their actual +`fixture` values in the order `[success, failure, replay]` twice. Their invocation +records retain `fixture` and the checkpointed `marker`. Replays must satisfy the +same stored-result, stored-failure, callback/wait and retry proof used by the Python +lifecycle validator. Warm process/provider identity is observed rather than +assumed from consecutive requests. These object-allocation and fixture details +belong to this explicit Python profile, not the language-neutral generic suite. diff --git a/lmi_tests/python_catalog.py b/lmi_tests/python_catalog.py new file mode 100644 index 000000000..b76edaf7f --- /dev/null +++ b/lmi_tests/python_catalog.py @@ -0,0 +1,612 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python-owned LMI requirements and coverage using the shared runner machinery.""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import json +from pathlib import Path +from typing import Any + +from aws_durable_execution_conformance_tests.lmi.app import SuiteDefinition +from aws_durable_execution_conformance_tests.lmi.matrix import CaseRequirement + + +@dataclass(frozen=True) +class ScenarioDefinition: + selector: str + roles: tuple[str, ...] + slots: int + exclusive: bool + budget_names: tuple[str, ...] + source_tests: tuple[str, ...] + roles_at_two: tuple[str, ...] | None = None + slots_at_two: int | None = None + + +@dataclass(frozen=True) +class Requirement: + number: int + description: str + contract: str + baseline: tuple[ScenarioDefinition, ...] + instrumented: tuple[ScenarioDefinition, ...] + + +# These are Python SDK regression contracts, not shared conformance requirements. +_REQUIREMENTS = ( + Requirement( + number=1, + description="Configured native worker capacity and execution isolation", + contract="Verify genuine native concurrency one and two deployments independently. Establish one original request per configured worker under a bounded barrier; at concurrency two require distinct worker processes overlapping in the same environment. Preserve the selected original requests, results and service histories. A single request to a concurrency-two deployment cannot replace the concurrency-one baseline.", + baseline=( + ScenarioDefinition( + selector="native-concurrency", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_actual_environment_process_concurrency", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ), + instrumented=(), + ), + Requirement( + number=10, + description="Synchronous checkpoint settlement at the branch boundary", + contract="Hold an actual SDK checkpoint request in flight while a child branch completes. The durable wrapper must preserve the supported checkpoint and child-scope settlement boundary. Require independently observed request entry, held interval, service response and wrapper exit; a mocked checkpoint flag or a callback without a real dispatch is insufficient.", + baseline=( + ScenarioDefinition( + selector="checkpoint-settlement", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_real_synchronous_checkpoint_settles_before_branch_join", + ), + ), + ), + instrumented=(), + ), + Requirement( + number=11, + description="Platform timeout retries preserve completed work", + contract="Cross the configured platform invocation timeout and observe a service-started retry with a different request ID for the same durable execution. Completed successful side effects remain checkpointed and are not repeated. Record interrupted work per attempt without imposing exactly-once execution on at-least-once steps; an independently invoked replacement execution cannot satisfy retry.", + baseline=( + ScenarioDefinition( + selector="timeout-retry", + roles=("victim", "retry"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs", "retryMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_service_timeout_retry_does_not_repeat_completed_step", + ), + ), + ), + instrumented=( + ScenarioDefinition( + selector="timeout-retry", + roles=("victim", "retry"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs", "retryMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_service_timeout_retry_does_not_repeat_completed_step", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + ), + ), + ), + Requirement( + number=12, + description="Abandoned child rejects a late durable operation", + contract="After a parent or sibling result abandons a child, allow that residual child to attempt another public durable operation. Prove the attempt was rejected before its user body and checkpoint dispatch; keep the operation absent from service history. Retain the original invocation and scope identities and distinguish fault release from the assertion boundary.", + baseline=( + ScenarioDefinition( + selector="late-operation", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_abandoned_child_rejects_late_durable_operation", + ), + ), + ), + instrumented=( + ScenarioDefinition( + selector="late-operation", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_abandoned_child_rejects_late_durable_operation", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + ), + ), + ), + Requirement( + number=13, + description="Root return and failure settle in-flight work", + contract="Exercise successful root return and root failure while an admitted child operation remains in flight. Observe root completion, child cleanup, request settlement and wrapper exit independently, preserving the original returned result or failure. At native concurrency two, keep a selected healthy peer progressing in the same environment without inheriting the victim scope.", + baseline=( + ScenarioDefinition( + selector="return-inflight", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ScenarioDefinition( + selector="failure-inflight", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ), + instrumented=( + ScenarioDefinition( + selector="return-inflight", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ScenarioDefinition( + selector="failure-inflight", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ), + ), + Requirement( + number=14, + description="Pending waits for root cleanup before replay", + contract="Suspend through the public durable API while controlled work remains in the root cleanup path. Observe root-finally entry and exit, actual PENDING wrapper completion, service resume and stable checkpoint history. The wrapper must honor the supported scope-cleanup boundary and replay must not repeat the completed operation body. Native concurrency two includes an established healthy peer.", + baseline=( + ScenarioDefinition( + selector="pending-cleanup", + roles=("victim", "resume"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_pending_waits_for_root_finally_and_then_replays", + ), + roles_at_two=("victim", "resume", "peer"), + slots_at_two=2, + ), + ), + instrumented=( + ScenarioDefinition( + selector="pending-cleanup", + roles=("victim", "resume"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_pending_waits_for_root_finally_and_then_replays", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + roles_at_two=("victim", "resume", "peer"), + slots_at_two=2, + ), + ), + ), + Requirement( + number=15, + description="Nested durable work progresses with bounded branch capacity", + contract="Use the real public child, map and parallel APIs with a single sibling slot to establish nested progress. Launch one root per configured native worker and positively establish the required environment and process placement. Nested work must progress within its declared test budget without starving behind its owning branch, preserving result attribution for every original root.", + baseline=( + ScenarioDefinition( + selector="nested-progress", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "progressMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_nested_single_lane_pools_progress_for_all_runtime_workers", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ), + instrumented=(), + ), + Requirement( + number=4, + description="Deadline admission and original environment capacity recovery", + contract="Observe a real platform timeout of the original victim with controlled work in flight. At native concurrency two, establish a healthy peer in the same environment and verify its progress and successful completion. Before fault release, demand every configured worker slot in the original environment and prove full capacity recovers within the declared deadline-relative cleanup budget. Preserve victim admission and stale-side-effect assertions independently of recovery and final retirement.", + baseline=( + ScenarioDefinition( + selector="deadline-recovery", + roles=("victim", "recovery"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs", "recoveryMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_invocation_deadline_isolation_and_capacity_recovery", + ), + roles_at_two=("victim", "peer", "recovery", "recovery-peer"), + slots_at_two=2, + ), + ), + instrumented=( + ScenarioDefinition( + selector="deadline-recovery", + roles=("victim", "recovery"), + slots=1, + exclusive=True, + budget_names=( + "observationGraceMs", + "cleanupMs", + "recoveryMs", + "exportMs", + ), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_invocation_deadline_isolation_and_capacity_recovery", + "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", + ), + roles_at_two=("victim", "peer", "recovery", "recovery-peer"), + slots_at_two=2, + ), + ), + ), + Requirement( + number=5, + description="Callback suspension and retry preserve checkpointed work", + contract="Exercise real durable callback suspension, operation retry and stored success and failure results. Resume the same durable execution through the service callback path, retaining every invocation and checkpoint history. Completed user bodies and side effects must not repeat on replay, and retry effects follow the configured per-attempt contract.", + baseline=( + ScenarioDefinition( + selector="replay-callback-retry", + roles=("victim", "resume"), + slots=1, + exclusive=False, + budget_names=("observationGraceMs",), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_checkpoint_replay_callback_and_retry", + ), + ), + ), + instrumented=(), + ), + Requirement( + number=6, + description="Warm success, failure and suspension resource cleanup", + contract="Exercise a repeated sequence of successful, failed and suspended/resumed public handlers. Identify actual process reuse and compare invocation-owned state and process resource baselines over the sequence. Verify branch/thread ownership and cleanup, preserving failure and replay outcomes; an unrelated process cannot replace the observed warm worker.", + baseline=( + ScenarioDefinition( + selector="warm-lifecycle", + roles=( + "victim", + "warm", + "warm-2", + "warm-2-resume", + "warm-3", + "warm-4", + "warm-5", + "warm-5-resume", + "warm-6", + "warm-7", + "warm-8", + "warm-8-resume", + ), + slots=1, + exclusive=True, + budget_names=( + "observationGraceMs", + "maxThreadGrowth", + "maxFdGrowth", + "maxRssGrowthBytes", + ), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_warm_success_failure_and_suspension_cleanup", + ), + ), + ), + instrumented=(), + ), + Requirement( + number=7, + description="Runtime carriers, warm telemetry and native worker attribution", + contract="In both OTel views and both native configurations, separately verify genuine sampled and unsampled runtime carriers, identified warm success/failure/replay, and one original request per configured worker. Preserve caller Root and sampling correlation and execution attribution. At concurrency two, ending one request must not end a held healthy peer; controlled malformed or missing-carrier unit fixtures cannot substitute for native runtime evidence.", + baseline=(), + instrumented=( + ScenarioDefinition( + selector="runtime-carrier-sampled", + roles=("victim",), + slots=1, + exclusive=False, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_native_runtime_carrier_and_sampling", + ), + ), + ScenarioDefinition( + selector="runtime-carrier-unsampled", + roles=("victim",), + slots=1, + exclusive=False, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_native_runtime_carrier_and_sampling", + ), + ), + ScenarioDefinition( + selector="otel-warm-lifecycle", + roles=( + "victim", + "warm-1", + "warm-2", + "warm-2-resume", + "warm-3", + "warm-4", + "warm", + "warm-resume", + ), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_warm_process_cleanup_and_replay", + ), + ), + ScenarioDefinition( + selector="otel-concurrent", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_concurrent_worker_attribution_and_peer_end", + ), + roles_at_two=("victim", "peer"), + slots_at_two=2, + ), + ), + ), + Requirement( + number=8, + description="Exporter failure recovery and held-export settlement", + contract="In both OTel views and both native configurations, separately observe an exporter failure followed by a healthy invocation and an intentionally held export. Record provider/exporter ownership and all export outcomes. Successful wrapper/OTel cleanup cannot precede the held export settlement; driver release does not turn an earlier failure or incomplete collection into a pass.", + baseline=(), + instrumented=( + ScenarioDefinition( + selector="export-error-recovery", + roles=("victim", "warm"), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_export_failure_then_healthy_invocation", + ), + ), + ScenarioDefinition( + selector="held-export", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "exportMs"), + source_tests=( + "lmi_tests/tests/e2e/test_observability.py::test_held_export_must_settle_before_successful_cleanup", + ), + ), + ), + ), + Requirement( + number=9, + description="Early completion closes losing branch scopes", + contract="For parallel, map and nested public operations configured to finish after a winning branch, observe the winner and controlled losing user work separately. The supported wrapper and branch cleanup boundaries must satisfy the declared cleanup budget before the driver releases the fault. Record actual task and request dispatch observations; final success alone cannot prove cleanup.", + baseline=( + ScenarioDefinition( + selector="early-completion-parallel", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", + ), + ), + ScenarioDefinition( + selector="early-completion-map", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", + ), + ), + ScenarioDefinition( + selector="early-completion-nested", + roles=("victim",), + slots=1, + exclusive=True, + budget_names=("observationGraceMs", "cleanupMs"), + source_tests=( + "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", + ), + ), + ), + instrumented=(), + ), +) + + +def requirements(capability: str = "lifecycle-otel") -> dict[str, dict[str, Any]]: + if capability not in ("lifecycle", "lifecycle-otel"): + raise ValueError("Unknown Python lifecycle capability") + result = {} + for item in _REQUIREMENTS: + definitions = item.baseline + ( + item.instrumented if capability == "lifecycle-otel" else () + ) + if not definitions: + continue + identity = f"python-lmi-{item.number}" + result[identity] = { + "id": identity, + "description": item.description, + "contract": item.contract, + "mandatory": True, + "capability": "lifecycle" if item.baseline else "otel", + "runtimes": ["python"], + "requiredScenarios": list( + dict.fromkeys(case.selector for case in definitions) + ), + } + return result + + +def required_cases(capability: str = "lifecycle-otel") -> tuple[CaseRequirement, ...]: + requirements(capability) + cases = [] + for item in _REQUIREMENTS: + groups: tuple[ + tuple[tuple[ScenarioDefinition, ...], tuple[str | None, ...]], ... + ] = ((item.baseline, (None,)),) + if capability == "lifecycle-otel": + groups += ((item.instrumented, ("invocation", "execution")),) + for definitions, views in groups: + for case in definitions: + for native in (1, 2): + roles = ( + case.roles_at_two + if native == 2 and case.roles_at_two is not None + else case.roles + ) + slots = ( + case.slots_at_two + if native == 2 and case.slots_at_two is not None + else case.slots + ) + for view in views: + cases.append( + CaseRequirement( + requirement_id=f"python-lmi-{item.number}", + scenario=case.selector, + native_concurrency=native, + otel_view=view, + roles=roles, + exclusive=case.exclusive, + minimum_native_concurrency=slots, + source_tests=case.source_tests, + budget_names=case.budget_names, + ) + ) + return tuple(cases) + + +def validate(evidence: Mapping[str, Any], *, phase: str = "final") -> list[str]: + from aws_durable_execution_conformance_tests.lmi.evidence import ( + validate_common_evidence, + ) + from lmi_tests.python_lifecycle import validate_lifecycle_regressions + + if phase not in ("observations", "final"): + return ["INCOMPLETE: unknown Python lifecycle validation phase"] + if ( + not isinstance(evidence, Mapping) + or evidence.get("coverageProfile") != "python-lifecycle" + ): + return [ + "INCOMPLETE: Python lifecycle validation requires its SDK-owned coverage profile" + ] + if evidence.get("runtime") != "python": + return ["INCOMPLETE: Python lifecycle contracts apply only to the Python SDK"] + profile = evidence.get("profile") + if not isinstance(profile, Mapping) or profile.get("recoveryPolicy") not in ( + "reuse-worker", + "replace-worker", + ): + return [ + "INCOMPLETE: Python lifecycle evidence requires an explicit supported recoveryPolicy" + ] + errors = validate_common_evidence( + evidence, phase=phase, requirement_ids=tuple(requirements()) + ) + errors.extend(validate_lifecycle_regressions(evidence, phase=phase)) + if evidence.get("otelView") is not None: + from lmi_tests.python_otel import validate_lifecycle_telemetry + + errors.extend(validate_lifecycle_telemetry(evidence, phase=phase)) + return errors + + +def suite_definition(capability: str = "lifecycle-otel") -> SuiteDefinition: + return SuiteDefinition( + coverage_profile="python-lifecycle", + suite_name="python-lmi", + requirements=requirements(capability), + cases=required_cases(capability), + validator=validate, + evidence_policy_key="regression", + nonnegative_budgets=frozenset( + {"maxThreadGrowth", "maxFdGrowth", "maxRssGrowthBytes"} + ), + ) + + +def load_manifest(path: Path) -> dict[str, Any]: + from aws_durable_execution_conformance_tests.lmi.app import ( + load_manifest as load_shared_manifest, + ) + + manifest = json.loads(path.read_text()) + if not isinstance(manifest, Mapping) or manifest.get("runtime") != "python": + raise ValueError("Python lifecycle manifests require runtime python") + if manifest.get("coverageProfile") != "python-lifecycle": + raise ValueError( + "Python lifecycle manifests require the SDK-owned coverage profile" + ) + profiles = manifest.get("profiles") + if isinstance(profiles, list): + for profile in profiles: + if not isinstance(profile, Mapping) or profile.get( + "recovery_policy" + ) not in ("reuse-worker", "replace-worker"): + raise ValueError( + "Python lifecycle profiles require an explicit recovery_policy" + ) + capability = manifest.get("capabilityProfile") + if not isinstance(capability, str): + raise ValueError( + "Python lifecycle manifests require an explicit capabilityProfile" + ) + return load_shared_manifest(path, suite=suite_definition(capability)) diff --git a/lmi_tests/python_lifecycle.py b/lmi_tests/python_lifecycle.py new file mode 100644 index 000000000..a002db787 --- /dev/null +++ b/lmi_tests/python_lifecycle.py @@ -0,0 +1,1224 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Python SDK LMI lifecycle, replay, deadline and settled-scope regressions.""" + +from __future__ import annotations + +import math +from collections.abc import Mapping +from itertools import pairwise +from typing import Any + +SCENARIOS = { + "native-concurrency": 1, + "deadline-recovery": 4, + "replay-callback-retry": 5, + "warm-lifecycle": 6, + "early-completion-parallel": 9, + "early-completion-map": 9, + "early-completion-nested": 9, + "checkpoint-settlement": 10, + "timeout-retry": 11, + "late-operation": 12, + "return-inflight": 13, + "failure-inflight": 13, + "pending-cleanup": 14, + "nested-progress": 15, +} +OTEL_SCENARIOS = { + "runtime-carrier-sampled": 7, + "runtime-carrier-unsampled": 7, + "otel-warm-lifecycle": 7, + "otel-concurrent": 7, + "export-error-recovery": 8, + "held-export": 8, +} +ACTIVITY = { + "body", + "effect", + "checkpoint_call", + "checkpoint_ack", + "checkpoint_exit", + "poll_call", + "poll_exit", +} +ENDINGS = {"wrapper_return", "wrapper_raise"} + + +class _Observations: + def __init__(self, evidence: Mapping[str, Any]) -> None: + self.evidence = evidence + self.errors: list[str] = [] + self.config = self.mapping(evidence.get("regression"), "regression") + self.profile = self.mapping(evidence.get("profile"), "profile") + self.attempts = { + self.string(item.get("requestId"), "attempt.requestId"): item + for item in self.records(evidence.get("attempts"), "attempts") + } + self.events = self.records(evidence.get("events"), "events") + self.histories = self.records(evidence.get("histories"), "histories") + self.selected = self.mapping( + evidence.get("selectedRequests"), "selectedRequests" + ) + self.end = self.number( + evidence.get("observationEndedAtMs"), "observationEndedAtMs" + ) + + def fail(self, message: str, category: str = "ASSERTION") -> None: + self.errors.append(f"{category}: {message}") + + def mapping(self, value: Any, path: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + self.fail(f"{path} must be an object", "INCOMPLETE") + return {} + return value + + def records(self, value: Any, path: str) -> list[Mapping[str, Any]]: + if not isinstance(value, list): + self.fail(f"{path} must be an array", "INCOMPLETE") + return [] + return [self.mapping(item, path) for item in value] + + def string(self, value: Any, path: str) -> str: + if not isinstance(value, str) or not value: + self.fail(f"{path} must be a non-empty string", "INCOMPLETE") + return "" + return value + + def number(self, value: Any, path: str) -> float: + if type(value) not in (int, float) or not math.isfinite(value) or value < 0: + self.fail(f"{path} must be a finite non-negative number", "INCOMPLETE") + return 0 + return float(value) + + def budget(self, name: str) -> float: + budgets = self.mapping(self.config.get("budgets"), "regression.budgets") + value = self.number(budgets.get(name), f"regression.budgets.{name}") + if value <= 0: + self.fail(f"regression.{name} must be positive", "INCOMPLETE") + return value + + def role(self, role: str) -> Mapping[str, Any]: + request_id = self.selected.get(role) + if not isinstance(request_id, str) or request_id not in self.attempts: + self.fail(f"selected original {role} request is missing", "INCOMPLETE") + return {} + return self.attempts[request_id] + + def select( + self, + kind: str | set[str], + request: Mapping[str, Any] | None = None, + **fields: Any, + ) -> list[Mapping[str, Any]]: + kinds = {kind} if isinstance(kind, str) else kind + return [ + item + for item in self.events + if item.get("kind") in kinds + and (request is None or item.get("requestId") == request.get("requestId")) + and all(item.get(key) == value for key, value in fields.items()) + ] + + def one( + self, kind: str | set[str], request: Mapping[str, Any], **fields: Any + ) -> Mapping[str, Any]: + matches = self.select(kind, request, **fields) + if len(matches) != 1: + self.fail( + f"request {request.get('requestId')} requires exactly one {kind} observation {fields}", + "INCOMPLETE", + ) + return {} + return matches[0] + + def ordered(self, *events: Mapping[str, Any]) -> bool: + return all( + first.get("requestId") == second.get("requestId") + and first.get("sequence", -1) < second.get("sequence", -1) + and first.get("atMs", -1) <= second.get("atMs", -1) + for first, second in pairwise(events) + ) + + def history(self, request: Mapping[str, Any]) -> Mapping[str, Any]: + matches = [ + item + for item in self.histories + if item.get("executionArn") == request.get("executionArn") + ] + if len(matches) != 1: + self.fail( + "one collected history is required for each selected execution", + "INCOMPLETE", + ) + return {} + return matches[0] + + def operation( + self, request: Mapping[str, Any], name: str, status: str | None = None + ) -> Mapping[str, Any]: + matches = [ + item + for item in self.records( + self.history(request).get("operations"), "history.operations" + ) + if item.get("name") == name + ] + if len(matches) != 1 or ( + status is not None and matches[0].get("status") != status + ): + self.fail( + f"collected history needs one {status or ''} {name} operation", + "INCOMPLETE", + ) + return {} + return matches[0] + + def resources(self, event: Mapping[str, Any]) -> Mapping[str, Any]: + resources = self.mapping(event.get("resources"), "wrapper.resources") + threads = self.records(resources.get("threads"), "wrapper.resources.threads") + ids = [] + for thread in threads: + ids.append(self.string(thread.get("id"), "thread.id")) + self.string(thread.get("name"), "thread.name") + if len(ids) != len(set(ids)): + self.fail("duplicate thread identities in resource snapshot", "INCOMPLETE") + for counter in ("fds", "rssBytes"): + self.number(resources.get(counter), f"wrapper.resources.{counter}") + return resources + + def lifecycle(self) -> None: + """Validate every request, including unselected/resumed attempts.""" + if self.evidence.get("runtime") != "python": + self.fail("Python lifecycle coverage requires runtime python", "INCOMPLETE") + grace = self.budget("observationGraceMs") + if not self.attempts or not self.events: + self.fail("lifecycle attempts and events must not be empty", "INCOMPLETE") + per_request: dict[str, list[Mapping[str, Any]]] = {} + for event in self.events: + request_id = event.get("requestId") + if not isinstance(request_id, str) or request_id not in self.attempts: + self.fail("event has no retained invocation owner", "INCOMPLETE") + continue + if event.get("ownerRequestId") != request_id or event.get( + "executionArn" + ) != self.attempts[request_id].get("executionArn"): + self.fail("lifecycle event belongs to another request or execution") + if type(event.get("sequence")) is not int or event["sequence"] < 0: + self.fail("event.sequence must be a non-negative integer", "INCOMPLETE") + when = self.number(event.get("atMs"), "event.atMs") + if when > self.end or when < self.attempts[request_id].get( + "startedAtMs", 0 + ): + self.fail( + "event lies outside retained observation window", "INCOMPLETE" + ) + per_request.setdefault(request_id, []).append(event) + if event.get("kind") in ("escape", "control_error"): + self.fail( + "emergency escape or failed fault control cannot establish SDK cleanup" + ) + for request_id, request in self.attempts.items(): + self.string(request.get("marker"), "attempt.marker") + events = sorted( + per_request.get(request_id, []), + key=lambda item: item.get("sequence", -1), + ) + if len({item.get("sequence") for item in events}) != len(events) or any( + a.get("atMs", 0) > b.get("atMs", 0) for a, b in pairwise(events) + ): + self.fail( + "lifecycle sequence is duplicated or contradicts the normalized clock", + "INCOMPLETE", + ) + entry = self.one("wrapper_enter", request) + baseline = self.resources(entry) + if request.get("outcome") == "TIMED_OUT": + self.one("platform_timeout", request, source="platform") + exits = self.select(ENDINGS, request) + if not exits: + if request.get("outcome") != "TIMED_OUT": + self.fail( + "non-timeout invocation has no actual wrapper exit", + "INCOMPLETE", + ) + retirement = self.select("worker_retired", request, source="platform") + task_exit = self.select("task_exit", request, taskId="root") + if not retirement and not task_exit: + self.fail( + "timed-out request has neither actual root exit nor platform worker retirement", + "INCOMPLETE", + ) + if ( + retirement + and self.profile.get("recoveryPolicy") != "replace-worker" + ): + self.fail( + "worker retirement requires the declared runtime replacement policy" + ) + continue + end = self.one(ENDINGS, request) + wrapper_outcome = end.get("outcome") + if wrapper_outcome not in ("SUCCEEDED", "FAILED", "PENDING", "TIMED_OUT"): + self.fail( + "actual wrapper outcome is missing or unrecognized", "INCOMPLETE" + ) + elif request.get( + "outcome" + ) != "TIMED_OUT" and wrapper_outcome != request.get("outcome"): + self.fail("actual wrapper outcome differs from its original request") + # A platform timeout does not stop the process. A later SDK return + # or exception is a separate boundary, still subject to settlement + # and stale-activity checks below. + user_exit = self.one("user_exit", request) + if not self.ordered(entry, user_exit, end): + self.fail("wrapper exited before actual user-scope exit") + if self.end < end.get("atMs", 0) + grace: + self.fail( + "collection ended before the declared post-wrapper observation window", + "INCOMPLETE", + ) + resources = self.resources(end) + baseline_ids = {thread.get("id") for thread in baseline.get("threads", [])} + for thread in resources.get("threads", []): + if thread.get("id") not in baseline_ids and ( + str(thread.get("name", "")).startswith( + ("dex-", "ThreadPoolExecutor") + ) + or thread.get("ownerRequestId") == request_id + ): + self.fail( + "invocation-owned worker survived the actual wrapper exit" + ) + if any( + item.get("kind") in ACTIVITY + and item.get("sequence", -1) > end.get("sequence", -1) + for item in events + ): + self.fail( + "user activity or request settlement continued after wrapper exit" + ) + for start_kind, end_kind, identity in ( + ("task_start", "task_exit", "taskId"), + ("io_start", "io_exit", "taskId"), + ("checkpoint_call", "checkpoint_exit", "callId"), + ("poll_call", "poll_exit", "callId"), + ): + for start in self.select(start_kind, request): + token = self.string(start.get(identity), f"{start_kind}.{identity}") + settled = self.one(end_kind, request, **{identity: token}) + if not self.ordered(start, settled, end): + self.fail(f"{start_kind} remained unsettled at wrapper exit") + for effect in self.select("body", request): + name = self.string(effect.get("operation"), "body.operation") + # Static names may repeat in distinct child contexts. Bind to + # service identity rather than requiring globally unique names. + operations = self.records( + self.history(request).get("operations"), "history.operations" + ) + matches = [ + item + for item in operations + if item.get("id") == effect.get("operationId") + and item.get("name") == name + ] + if len(matches) != 1: + self.fail("body observation changed its service operation identity") + + def overlap( + self, + roles: list[str], + *, + environment: str | None = None, + gate_ids: Mapping[str, str] | None = None, + ) -> list[Mapping[str, Any]]: + requests = [self.role(role) for role in roles] + if not requests or any(not request for request in requests): + return requests + n = len(requests) + for field in ("requestId", "executionArn", "processId", "pid"): + if len({request.get(field) for request in requests}) != n: + self.fail( + f"native cohort lacks distinct {field} identities", "PLACEMENT" + ) + if any( + type(request.get("pid")) is not int or request["pid"] <= 0 + for request in requests + ): + self.fail("actual runtime PID evidence is required", "INCOMPLETE") + environments = {request.get("environmentId") for request in requests} + if len(environments) != 1 or ( + environment is not None and environments != {environment} + ): + self.fail("native cohort reached an unrelated environment", "PLACEMENT") + # A fixture may also hold a handshake gate. Only its explicitly + # selected work gate establishes this role's cohort participation. + # Roles without a declared selector still require an unambiguous gate. + blocked = [ + self.one( + "blocked", + request, + **({"gateId": gate_ids[role]} if gate_ids and role in gate_ids else {}), + ) + for role, request in zip(roles, requests, strict=True) + ] + pivot = max(item.get("atMs", 0) for item in blocked) + for request, start in zip(requests, blocked, strict=True): + alive = self.select( + {"alive", "effect"}, request, gateId=start.get("gateId") + ) + if ( + not any(item.get("atMs", -1) >= pivot for item in alive) + or request.get("endedAtMs", 0) <= pivot + ): + self.fail( + "native overlap lacks positive continued liveness", "PLACEMENT" + ) + if any( + item.get("atMs", 0) <= pivot + for item in self.select("io_exit", request, gateId=start.get("gateId")) + ): + self.fail( + "native capacity stopped before overlap was established", + "PLACEMENT", + ) + return requests + + def gate(self, request: Mapping[str, Any]) -> str: + gate_id = self.string(self.config.get("gateId"), "regression.gateId") + faults = self.records(self.evidence.get("faultsCreated"), "faultsCreated") + if not any( + item.get("id") == gate_id + and item.get("ownerCaseId") == self.evidence.get("caseId") + for item in faults + ): + self.fail("fault gate lacks retained case ownership", "INCOMPLETE") + self.one("blocked", request, gateId=gate_id) + return gate_id + + def released(self, gate_id: str) -> float | None: + releases = self.select("fault_released", gateId=gate_id) + retirement = self.mapping(self.evidence.get("retirement", {}), "retirement") + releases += [ + item + for item in self.records(retirement.get("faults", []), "retirement.faults") + if item.get("id") == gate_id + ] + times = [] + for release in releases: + if release.get("mode") != "normal": + self.fail( + "emergency or unclassified fault release cannot establish the lifecycle assertion" + ) + times.append( + self.number( + release.get("atMs", release.get("releasedAtMs")), + "fault release time", + ) + ) + return min(times) if times else None + + def held_loser(self, request: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]: + gate_id = self.gate(request) + ready = self.one("winner_ready", request) + selected = self.one("winner_selected", request) + if not self.ordered(ready, selected): + self.fail("parent winner selection preceded readiness") + if not any( + self.ordered(item, ready) + for item in self.select("effect", request, gateId=gate_id) + ) or any( + self.ordered(item, ready) + for item in self.select("io_exit", request, gateId=gate_id) + ): + self.fail( + "losing branch was not actually held when its winner became ready", + "INCOMPLETE", + ) + return gate_id, selected + + def early(self) -> None: + victim = self.role("victim") + gate_id, winner = self.held_loser(victim) + cutoff = winner.get("atMs", 0) + self.budget("cleanupMs") + end = self.one(ENDINGS, victim) + if end.get("atMs", self.end) > cutoff: + self.fail( + "early completion left the wrapper pinned beyond the cleanup budget" + ) + if self.end < cutoff + self.budget("observationGraceMs"): + self.fail( + "early-completion collection ended before the cleanup assertion window", + "INCOMPLETE", + ) + if any(item.get("atMs", 0) > cutoff for item in self.select(ACTIVITY, victim)): + self.fail( + "abandoned scope continued effects or request activity beyond its cleanup budget" + ) + released = self.released(gate_id) + if released is not None and released <= cutoff: + self.fail( + "driver release supplied the losing-branch cleanup being asserted" + ) + names = { + "early-completion-parallel": ("race-parallel",), + "early-completion-map": ("race-map",), + "early-completion-nested": ("outer-map", "nested-child", "race-parallel"), + } + for name in names[self.evidence["scenario"]]: + self.operation(victim, name, "SUCCEEDED") + + def checkpoint(self) -> None: + victim = self.role("victim") + gate_id = self.gate(victim) + held = self.one("ack_held", victim, gateId=gate_id) + blocked = self.one("blocked", victim, gateId=gate_id) + call_id = self.string(held.get("callId"), "ack_held.callId") + call = self.one("checkpoint_call", victim, callId=call_id) + ack = self.one("checkpoint_ack", victim, callId=call_id, source="service") + exited = self.one("checkpoint_exit", victim, callId=call_id) + end = self.one(ENDINGS, victim) + operation = self.operation(victim, "checkpoint-loser", "SUCCEEDED") + pool = self.operation(victim, "checkpoint-pool", "SUCCEEDED") + joined = self.one("branch_join", victim, scopeId=pool.get("id")) + if held.get("operationId") != operation.get("id") or operation.get( + "id" + ) not in ack.get("operationIds", []): + self.fail( + "held acknowledgment does not belong to the selected real checkpoint" + ) + release = self.released(gate_id) + if not self.ordered(call, ack, held, blocked, exited, joined, end): + self.fail( + "real checkpoint acknowledgment/settlement did not precede branch join and wrapper return" + ) + if release is None or not blocked.get("atMs", 0) < release <= exited.get( + "atMs", 0 + ): + self.fail( + "checkpoint did not stay held until the scenario's normal release" + ) + + def deadline_measurement_end( + self, gate_id: str, deadline: float, cutoff: float, grace: float + ) -> float: + if "deadlineMeasurement" not in self.config: + return self.end + prior_errors = len(self.errors) + measurement = self.mapping( + self.config["deadlineMeasurement"], "deadline measurement" + ) + if measurement.get("selectedRequests") != self.selected: + self.fail( + "deadline measurement changed the selected original requests", + "INCOMPLETE", + ) + for field, expected in ( + ("deadlineAtMs", deadline), + ("cleanupCutoffAtMs", cutoff), + ("observationCutoffAtMs", cutoff + grace), + ): + actual = self.number( + measurement.get(field), f"deadline measurement.{field}" + ) + if not math.isclose(actual, expected, rel_tol=0, abs_tol=1): + self.fail( + "deadline measurement differs from the runtime deadline or declared budgets", + "INCOMPLETE", + ) + observed = self.number( + measurement.get("observedAtMs"), "deadline measurement.observedAtMs" + ) + if not cutoff + grace <= observed <= self.end: + self.fail( + "deadline measurement does not cover its complete original observation window", + "INCOMPLETE", + ) + uri = measurement.get("sourceEvidenceUri") + collection = self.mapping(self.evidence.get("collection"), "collection") + sources = collection.get("evidenceUris") + if ( + not isinstance(uri, str) + or not uri + or not isinstance(sources, list) + or uri not in sources + ): + self.fail( + "deadline measurement has no retained source evidence", "INCOMPLETE" + ) + faults = [ + fault + for fault in self.records( + self.evidence.get("faultsCreated"), "faultsCreated" + ) + if fault.get("id") == gate_id + and fault.get("ownerCaseId") == self.evidence.get("caseId") + ] + requested = self.end + if len(faults) != 1: + self.fail( + "deadline measurement must retain its original fault gate", "INCOMPLETE" + ) + else: + requested = self.number( + faults[0].get("releaseRequestedAtMs"), + "deadline measurement.releaseRequestedAtMs", + ) + acknowledged = self.number( + faults[0].get("releaseAcknowledgedAtMs"), + "deadline measurement.releaseAcknowledgedAtMs", + ) + receipt = self.number( + faults[0].get("releasedAtMs"), "deadline measurement.releasedAtMs" + ) + earliest_release = self.released(gate_id) + if ( + not observed <= requested <= acknowledged <= self.end + or receipt < requested + or (earliest_release is not None and earliest_release < requested) + ): + self.fail( + "deadline measurement contradicts the controller request or release evidence", + "INCOMPLETE", + ) + # Observe the capture-to-release gap too; invalid snapshots never + # shorten validation of the full observation tail. + return requested if len(self.errors) == prior_errors else self.end + + def late_deadline_activity( + self, victim: Mapping[str, Any], cutoff: float, released_after: float + ) -> bool: + for event in self.select(ACTIVITY, victim): + when = event.get("atMs", 0) + if when <= cutoff: + continue + # Release cannot authorize new user bodies or SDK dispatch. Only + # completion of previously dispatched I/O is cleanup activity. + if when <= released_after or event.get("kind") in { + "body", + "effect", + "checkpoint_call", + "poll_call", + }: + return True + start_kind = { + "checkpoint_ack": "checkpoint_call", + "checkpoint_exit": "checkpoint_call", + "poll_exit": "poll_call", + }.get(event["kind"]) + if start_kind is not None: + calls = self.select(start_kind, victim, callId=event.get("callId")) + if len(calls) != 1 or calls[0].get("atMs", when) > when: + return True + return False + + def timeout(self, victim: Mapping[str, Any]) -> float: + timeout = self.one("platform_timeout", victim, source="platform") + blocked = self.one("blocked", victim) + observed = self.one("runtime_deadline", victim, source="runtime") + remaining = self.number( + observed.get("remainingTimeMs"), "runtime_deadline.remainingTimeMs" + ) + configured = self.number(self.profile.get("timeoutMs"), "profile.timeoutMs") + deadline = self.number(victim.get("deadlineAtMs"), "attempt.deadlineAtMs") + observed_at = self.number(observed.get("atMs"), "runtime_deadline.atMs") + if not 0 < remaining <= configured: + self.fail( + "runtime remaining time must be positive and within the configured timeout", + "INCOMPLETE", + ) + # A runtime deadline can precede wrapper-entry + configured timeout: + # native dispatch/setup has already consumed some invocation lifetime. + # One millisecond allows canonical timestamp rounding, not extra cleanup. + if deadline <= observed_at or not math.isclose( + deadline, observed_at + remaining, rel_tol=0, abs_tol=1 + ): + self.fail( + "attempt deadline does not match its observed runtime remaining time", + "INCOMPLETE", + ) + if ( + victim.get("outcome") != "TIMED_OUT" + or timeout.get("atMs", 0) < deadline + or blocked.get("atMs", deadline) >= deadline + ): + self.fail( + "fixture did not cross the selected request's actual platform deadline with work held" + ) + cutoff = deadline + self.budget("cleanupMs") + grace = self.budget("observationGraceMs") + if self.end < cutoff + grace: + self.fail( + "timeout collection ended before the stale-activity observation window", + "INCOMPLETE", + ) + gate_id = self.string(blocked.get("gateId"), "blocked.gateId") + measured_until = self.deadline_measurement_end(gate_id, deadline, cutoff, grace) + if self.late_deadline_activity(victim, cutoff, measured_until): + self.fail( + "original timed-out request continued side effects or checkpoint activity" + ) + released = self.released(gate_id) + if released is not None and released <= cutoff: + self.fail("fault release supplied the timeout cleanup being asserted") + return deadline + + def recovery(self) -> None: + victim = self.role("victim") + policy = self.profile.get("recoveryPolicy") + if policy not in ("reuse-worker", "replace-worker"): + self.fail( + "deadline recovery requires an explicit supported recoveryPolicy", + "INCOMPLETE", + ) + deadline = self.timeout(victim) + native = self.profile.get("nativeConcurrency") + if native == 2: + self.overlap(["victim", "peer"]) + peer = self.role("peer") + if peer.get("outcome") != "SUCCEEDED": + self.fail("healthy peer did not complete successfully") + roles = ["recovery"] + (["recovery-peer"] if native == 2 else []) + requests = self.overlap(roles, environment=victim.get("environmentId")) + bound = deadline + self.budget("recoveryMs") + release = self.released(self.one("blocked", victim).get("gateId", "")) + recovered = max( + (self.one("blocked", request).get("atMs", 0) for request in requests), + default=self.end, + ) + if release is not None and release <= recovered: + self.fail("driver release supplied the capacity recovery being asserted") + for request in requests: + blocked = self.one("blocked", request) + if ( + not deadline <= blocked.get("atMs", -1) <= bound + or request.get("outcome") != "SUCCEEDED" + ): + self.fail( + "full original-environment capacity did not recover within its declared budget" + ) + if self.profile.get("recoveryPolicy") == "reuse-worker" and not any( + request.get("processId") == victim.get("processId") + and request.get("workerId") == victim.get("workerId") + for request in requests + ): + self.fail("full-capacity recovery omitted the original worker", "PLACEMENT") + if self.profile.get("recoveryPolicy") == "replace-worker" and not any( + request.get("replacesWorkerId") == victim.get("workerId") + for request in requests + ): + self.fail( + "replacement capacity did not identify the retired original worker", + "PLACEMENT", + ) + + def replay(self, execution_arn: str) -> None: + requests = [ + item + for item in self.attempts.values() + if item.get("executionArn") == execution_arn + ] + pending = [item for item in requests if item.get("outcome") == "PENDING"] + completed = [item for item in requests if item.get("outcome") == "SUCCEEDED"] + if ( + len(requests) < 2 + or not pending + or not completed + or not any( + first.get("endedAtMs", 0) < last.get("startedAtMs", 0) + for first in pending + for last in completed + ) + ): + self.fail( + "replay needs a real PENDING request followed by a service resume of the same execution" + ) + if not requests: + return + original = min(requests, key=lambda item: item.get("startedAtMs", 0)) + marker = self.string(original.get("marker"), "attempt.marker") + if any(item.get("marker") != marker for item in requests): + self.fail("replay input marker changed between invocations") + ids = {item["requestId"] for item in requests} + for name, terminal in (("success", "SUCCEEDED"), ("failure", "FAILED")): + operation = self.operation(original, name, terminal) + bodies = [ + item + for item in self.select("body", operation=name) + if item.get("requestId") in ids + ] + if ( + len(bodies) != 1 + or bodies[0].get("value") != marker + or bodies[0].get("operationId") != operation.get("id") + ): + self.fail( + f"completed {name} body was repeated, omitted or misattributed" + ) + failures = [ + item + for item in self.select("stored_failure") + if item.get("requestId") in ids + ] + expected_error = {"type": "ValueError", "message": "expected:" + marker} + if len({item.get("requestId") for item in failures}) < 2 or any( + item.get("error") != expected_error for item in failures + ): + self.fail( + "stored failure was not rethrown with the original meaning on replay" + ) + retry = self.operation(original, "retry", "SUCCEEDED") + bodies = sorted( + [ + item + for item in self.select("body", operation="retry") + if item.get("requestId") in ids + ], + key=lambda item: item.get("atMs", 0), + ) + if ( + any(type(item.get("attempt")) is not int for item in bodies) + or [item.get("attempt") for item in bodies] != [1, 2] + or any( + item.get("value") != marker + or item.get("operationId") != retry.get("id") + for item in bodies + ) + ): + self.fail( + "retry fixture did not execute exactly its two configured attempts" + ) + operations = self.records( + self.history(original).get("operations"), "history.operations" + ) + for kind in ("WAIT", "CALLBACK", "CONTEXT"): + if not any( + item.get("type") == kind and item.get("status") == "SUCCEEDED" + for item in operations + ): + self.fail( + f"real replay history lacks a successful {kind} operation", + "INCOMPLETE", + ) + + def service_retry(self) -> None: + victim, retry = self.role("victim"), self.role("retry") + deadline = self.timeout(victim) + timeout = self.one("platform_timeout", victim, source="platform") + if victim.get("executionArn") != retry.get("executionArn") or retry.get( + "startedAtMs", 0 + ) <= timeout.get("atMs", deadline): + self.fail( + "timeout retry must be a later platform request of the same durable execution" + ) + resumed = self.one("service_retry", retry) + if ( + resumed.get("previousRequestId") != victim.get("requestId") + or resumed.get("source") != "platform" + ): + self.fail( + "driver-created invocation is not evidence of a service timeout retry" + ) + if retry.get("startedAtMs", self.end) > deadline + self.budget("retryMs"): + self.fail("service timeout retry exceeded its declared observation budget") + success = self.operation(victim, "success", "SUCCEEDED") + ids = { + item["requestId"] + for item in self.attempts.values() + if item.get("executionArn") == victim.get("executionArn") + } + bodies = [ + item + for item in self.select("body", operation="success") + if item.get("requestId") in ids + ] + if ( + len(bodies) != 1 + or bodies[0].get("requestId") != victim.get("requestId") + or bodies[0].get("operationId") != success.get("id") + or bodies[0].get("value") != victim.get("marker") + ): + self.fail( + "completed success was repeated or lost across the timeout service retry" + ) + + def peer_progress( + self, victim: Mapping[str, Any], boundary: Mapping[str, Any], gate_id: str + ) -> None: + if self.profile.get("nativeConcurrency") != 2: + return + self.overlap(["victim", "peer"], gate_ids={"victim": gate_id}) + peer = self.role("peer") + progress = self.select({"alive", "peer_progress"}, peer) + if not any( + boundary.get("atMs", 0) <= item.get("atMs", -1) < victim.get("endedAtMs", 0) + for item in progress + ): + self.fail( + "healthy peer did not progress while victim cleanup remained held" + ) + + def scope_settlement(self, victim: Mapping[str, Any], gate_id: str) -> None: + blocked = self.one("blocked", victim, gateId=gate_id) + exited = self.one("io_exit", victim, gateId=gate_id) + end = self.one(ENDINGS, victim) + release = self.released(gate_id) + if ( + not self.ordered(blocked, exited, end) + or release is None + or not blocked.get("atMs", 0) < release <= exited.get("atMs", 0) + ): + self.fail( + "wrapper abandoned held invocation-owned work before normal settlement" + ) + + def inflight(self) -> None: + victim = self.role("victim") + gate_id, selected = self.held_loser(victim) + root = self.one("user_exit", victim) + exited = self.one("io_exit", victim, gateId=gate_id) + if not self.ordered(selected, root, exited): + self.fail( + "root completion was not observed with losing work still in flight" + ) + self.peer_progress(victim, root, gate_id) + self.scope_settlement(victim, gate_id) + status = ( + "FAILED" + if self.evidence.get("scenario") == "failure-inflight" + else "SUCCEEDED" + ) + if victim.get("outcome") != status: + self.fail("inflight root outcome changed during scope settlement") + if status == "FAILED" and self.history(victim).get("error") != { + "type": "ValueError", + "message": "expected:" + victim.get("marker", ""), + }: + self.fail("settling inflight work replaced the original root failure") + + def pending_cleanup(self) -> None: + victim, resume = self.role("victim"), self.role("resume") + gate_id = self.gate(victim) + blocked = self.one("blocked", victim, gateId=gate_id) + self.peer_progress(victim, blocked, gate_id) + self.scope_settlement(victim, gate_id) + enter, exited = ( + self.one("cleanup_enter", victim), + self.one("cleanup_exit", victim), + ) + end = self.one(ENDINGS, victim) + io_exit = self.one("io_exit", victim, gateId=gate_id) + user_exit = self.one("user_exit", victim) + if ( + not self.ordered(enter, blocked, io_exit, exited, user_exit, end) + or victim.get("outcome") != "PENDING" + ): + self.fail("PENDING preceded completion of the original root finally block") + if ( + resume.get("executionArn") != victim.get("executionArn") + or resume.get("startedAtMs", 0) <= end.get("atMs", 0) + or resume.get("outcome") != "SUCCEEDED" + ): + self.fail( + "cleanup fixture did not really resume the original suspended execution" + ) + bodies = [ + item + for item in self.select("body", operation="success") + if item.get("executionArn") == victim.get("executionArn") + ] + if len(bodies) != 1 or bodies[0].get("value") != victim.get("marker"): + self.fail( + "completed side effect repeated after finally cleanup and suspension" + ) + if not any( + item.get("type") == "WAIT" and item.get("status") == "SUCCEEDED" + for item in self.records( + self.history(victim).get("operations"), "history.operations" + ) + ): + self.fail("cleanup replay did not use a real durable wait", "INCOMPLETE") + + def late_operation(self) -> None: + victim = self.role("victim") + gate_id, selected = self.held_loser(victim) + self.scope_settlement(victim, gate_id) + attempted, rejected = ( + self.one("late_attempt", victim), + self.one("late_rejected", victim), + ) + closed = self.one("scope_closed", victim, scopeId=attempted.get("scopeId")) + operations = self.records( + self.history(victim).get("operations"), "history.operations" + ) + if not any( + item.get("id") == attempted.get("scopeId") and item.get("type") == "CONTEXT" + for item in operations + ): + self.fail( + "late attempt is not bound to a real completed child scope", + "INCOMPLETE", + ) + if not self.ordered(self.one("io_exit", victim, gateId=gate_id), attempted): + self.fail( + "late operation fixture did not follow settlement of the held losing step" + ) + if ( + not self.ordered(closed, attempted, rejected) + or not self.ordered(selected, attempted) + or rejected.get("scopeId") != attempted.get("scopeId") + or rejected.get("error") != {"type": "OrphanedChildException"} + ): + self.fail( + "completed child scope did not reject the attempted late durable operation" + ) + if self.select("late_accepted", victim) or self.select( + "body", victim, operation="late-work" + ): + self.fail("abandoned child executed or admitted late user work") + if any( + item.get("name") == "late-work" + for item in self.records( + self.history(victim).get("operations"), "history.operations" + ) + ): + self.fail("late child operation reached the durable service") + for request in self.records(self.evidence.get("sdkRequests"), "sdkRequests"): + if ( + request.get("requestId") == victim.get("requestId") + and request.get("operation") == "late-work" + and request.get("outcome") == "DISPATCHED" + ): + self.fail("late child operation dispatched a checkpoint") + + def nested(self) -> None: + roles = ["victim"] + ( + ["peer"] if self.profile.get("nativeConcurrency") == 2 else [] + ) + requests = self.overlap(roles) + budget = self.budget("progressMs") + for request in requests: + begin, end = ( + self.one("progress_begin", request), + self.one("progress", request), + ) + if ( + not self.ordered(begin, end) + or end.get("atMs", self.end) - begin.get("atMs", 0) > budget + ): + self.fail( + "nested public operations starved beyond their progress budget" + ) + marker = request.get("marker", "") + if end.get("value") != [[marker + ":0", marker + ":1"], marker]: + self.fail("nested results crossed invocation boundaries") + for name in ("outer-parallel", "inner-map"): + operation = self.operation(request, name, "SUCCEEDED") + configuration = self.one( + "operation_config", + request, + operationId=operation.get("id"), + source="public-fixture", + ) + if ( + type(configuration.get("maxConcurrency")) is not int + or configuration.get("maxConcurrency") != 1 + ): + self.fail( + "nested fixture did not exercise configured single-lane branch pools", + "INCOMPLETE", + ) + self.operation(request, "progress-child", "SUCCEEDED") + + def warm(self) -> None: + sequence = self.records( + self.config.get("warmSequence"), "regression.warmSequence" + ) + if [item.get("fixture") for item in sequence] != [ + "success", + "failure", + "replay", + ] * 3: + self.fail( + "warm lifecycle requires the nine success/failure/replay fixture executions", + "INCOMPLETE", + ) + arns = [item.get("executionArn") for item in sequence] + if len(set(arns)) != len(arns): + self.fail( + "warm sequence substituted an execution for a later fixture", + "INCOMPLETE", + ) + previous_end = -1.0 + samples: dict[tuple[Any, Any], list[Mapping[str, Any]]] = {} + for item in sequence: + requests = [ + request + for request in self.attempts.values() + if request.get("executionArn") == item.get("executionArn") + ] + if not requests: + self.fail("warm fixture has no retained requests", "INCOMPLETE") + continue + if ( + any( + request.get("fixture") != item.get("fixture") + for request in requests + ) + or min(request.get("startedAtMs", 0) for request in requests) + <= previous_end + ): + self.fail( + "warm fixture routing changed or executions were not sequential" + ) + previous_end = max(request.get("endedAtMs", 0) for request in requests) + history = self.history(requests[0]) + expected = "FAILED" if item.get("fixture") == "failure" else "SUCCEEDED" + if history.get("status") != expected: + self.fail("warm fixture execution has the wrong terminal result") + if item.get("fixture") == "replay": + self.replay(item["executionArn"]) + for request in requests: + for end in self.select(ENDINGS, request): + samples.setdefault( + (request.get("environmentId"), request.get("processId")), [] + ).append(end) + reused = [ + sorted(values, key=lambda item: item.get("atMs", 0)) + for values in samples.values() + if len(values) >= 3 + ] + if not reused: + self.fail( + "warm resources lack at least three observed uses of one original process", + "PLACEMENT", + ) + growth = self.mapping(self.config.get("budgets"), "regression.budgets") + allowances = { + counter: self.number(growth.get(key), f"regression.budgets.{key}") + for counter, key in ( + ("threads", "maxThreadGrowth"), + ("fds", "maxFdGrowth"), + ("rssBytes", "maxRssGrowthBytes"), + ) + } + for values in reused: + baseline = self.resources(values[1]) + for sample in values[2:]: + current = self.resources(sample) + for counter, allowance in allowances.items(): + before = ( + len(baseline.get("threads", [])) + if counter == "threads" + else baseline.get(counter, 0) + ) + after = ( + len(current.get("threads", [])) + if counter == "threads" + else current.get(counter, 0) + ) + if after > before + allowance: + self.fail( + f"warm process {counter} grew beyond its declared post-cache allowance" + ) + + +def _run( + evidence: Mapping[str, Any], check: str, execution_arn: str | None = None +) -> list[str]: + if not isinstance(evidence, Mapping): + return ["INCOMPLETE: lifecycle evidence must be an object"] + observations: _Observations | None = None + try: + observations = _Observations(evidence) + if check == "events": + observations.lifecycle() + elif check == "replay": + observations.replay(execution_arn or "") + else: + observations.lifecycle() + scenario = evidence.get("scenario") + if evidence.get("coverageProfile") != "python-lifecycle": + observations.fail( + "Python regression semantics require the explicit coverageProfile", + "INCOMPLETE", + ) + elif scenario in OTEL_SCENARIOS: + if ( + evidence.get("requirementId") + != f"python-lmi-{OTEL_SCENARIOS[scenario]}" + ): + observations.fail( + "OTel scenario does not match its lifecycle requirement", + "INCOMPLETE", + ) + # The OTel extension validates carriers/providers/exports. Keep + # invocation-owned lifecycle checks for these six cases too. + elif ( + scenario not in SCENARIOS + or evidence.get("requirementId") != f"python-lmi-{SCENARIOS[scenario]}" + ): + observations.fail( + "scenario does not match a Python lifecycle requirement", + "INCOMPLETE", + ) + elif not observations.errors: + native = observations.profile.get("nativeConcurrency") + if native not in (1, 2): + observations.fail( + "Python regression matrix requires native concurrency 1 or 2", + "INCOMPLETE", + ) + elif scenario == "native-concurrency": + observations.overlap(["victim"] + (["peer"] if native == 2 else [])) + elif scenario.startswith("early-completion-"): + observations.early() + else: + checks = { + "deadline-recovery": observations.recovery, + "warm-lifecycle": observations.warm, + "checkpoint-settlement": observations.checkpoint, + "timeout-retry": observations.service_retry, + "late-operation": observations.late_operation, + "return-inflight": observations.inflight, + "failure-inflight": observations.inflight, + "pending-cleanup": observations.pending_cleanup, + "nested-progress": observations.nested, + } + if scenario == "replay-callback-retry": + observations.replay( + observations.role("victim").get("executionArn", "") + ) + else: + checks[scenario]() + except (KeyError, TypeError, ValueError, OverflowError) as exc: + error = f"INCOMPLETE: malformed lifecycle regression evidence: {exc}" + return [*(observations.errors if observations else []), error] + return observations.errors + + +def validate_lifecycle_events(evidence: Mapping[str, Any]) -> list[str]: + """Check settled wrapper/task/request/resource lifetimes, including OTel cases.""" + return _run(evidence, "events") + + +def validate_replay_evidence( + evidence: Mapping[str, Any], execution_arn: str +) -> list[str]: + """Check real wait/callback/child replay and configured retries for one execution.""" + return _run(evidence, "replay", execution_arn) + + +def validate_lifecycle_regressions( + evidence: Mapping[str, Any], *, phase: str = "final" +) -> list[str]: + """Validate one coverage cell; core common/retirement validation also applies. + + Both phases preserve behavioral observations. Final validation also inspects + any subsequently recorded release, so emergency release never repairs an + earlier failed assertion. The caller must retain observation-phase errors. + """ + if phase not in ("observations", "final"): + return ["INCOMPLETE: unknown lifecycle regression validation phase"] + return _run(evidence, "scenario") diff --git a/lmi_tests/python_otel.py b/lmi_tests/python_otel.py new file mode 100644 index 000000000..01da69dc3 --- /dev/null +++ b/lmi_tests/python_otel.py @@ -0,0 +1,740 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Mandatory OTel observations for the Python LMI regression matrix. + +The SDK adapter records actual public-plugin boundaries, exporter attempts and +runtime carriers. Canonical spans still use the ordinary backend normalization +and assertion engine. This profile describes Python's supported fixtures; it +does not require other runtimes to allocate identical plugin objects. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from aws_durable_execution_conformance_tests_otel.chained_invoke import ( + execution_trace_context, +) +from aws_durable_execution_conformance_tests_otel.lmi import ( + _identifier, + _number, + _records, + _service_execution_starts, + _span, + _view_errors, +) +from aws_durable_execution_conformance_tests_otel.model import ( + Span, + TelemetryQuery, + Trace, +) +from aws_durable_execution_conformance_tests_otel.validators import validate_trace + +SCENARIOS = ( + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + "otel-warm-lifecycle", + "otel-concurrent", + "export-error-recovery", + "held-export", + "deadline-recovery", + "timeout-retry", + "pending-cleanup", + "late-operation", + "return-inflight", + "failure-inflight", +) +_TERMINAL_SCENARIOS = frozenset(SCENARIOS[:6]) + + +class _Telemetry: + def __init__(self, evidence: Mapping[str, Any]) -> None: + self.evidence = evidence + self.errors: list[str] = [] + self.scenario = evidence.get("scenario") + self.view_name = str(evidence.get("otelView", "")) + self.attempts: dict[str, Mapping[str, Any]] = {} + self.events: list[Mapping[str, Any]] = [] + self.enters: dict[str, Mapping[str, Any]] = {} + self.exits: dict[str, Mapping[str, Any]] = {} + self.boundaries: dict[str, Mapping[str, Any]] = {} + self.carriers: dict[str, Mapping[str, Any]] = {} + self.view: Mapping[str, Any] = {} + + def fail(self, detail: str, kind: str = "ASSERTION") -> None: + self.errors.append(f"{kind}: {detail}") + + def records(self, request_id: str, kind: str) -> list[Mapping[str, Any]]: + return [ + event + for event in self.events + if event.get("requestId") == request_id and event.get("kind") == kind + ] + + def one(self, request_id: str, kind: str) -> Mapping[str, Any]: + records = self.records(request_id, kind) + if len(records) != 1: + self.fail(f"{request_id} needs exactly one observed {kind}", "INCOMPLETE") + return {} + return records[0] + + def selected(self, role: str) -> str: + request_id = self.evidence.get("selectedRequests", {}).get(role) + if not isinstance(request_id, str) or request_id not in self.attempts: + raise ValueError(f"missing original selected {role} request") + return request_id + + def common(self) -> bool: + if ( + self.evidence.get("coverageProfile") != "python-lifecycle" + or self.evidence.get("runtime") != "python" + ): + self.fail( + "Python OTel regressions require coverageProfile=python-lifecycle and runtime=python", + "INCOMPLETE", + ) + return False + if self.scenario not in SCENARIOS or self.view_name not in ( + "invocation", + "execution", + ): + self.fail("unknown OTel regression scenario or view", "INCOMPLETE") + return False + if not self.evidence.get("artifact", {}).get("plugin"): + self.fail("exact plugin artifact identity is required", "INCOMPLETE") + for attempt in _records(self.evidence.get("attempts"), "attempts"): + request_id = attempt.get("requestId") + if ( + not isinstance(request_id, str) + or not request_id + or request_id in self.attempts + ): + raise ValueError("attempts need unique request identities") + self.attempts[request_id] = attempt + if not self.attempts: + raise ValueError("no observed native requests") + self.events = _records(self.evidence.get("events"), "events") + sequences: set[tuple[str, int]] = set() + for event in self.events: + request_id = event.get("requestId") + if ( + request_id not in self.attempts + or event.get("ownerRequestId") != request_id + ): + self.fail("event does not retain original request ownership") + continue + if event.get("executionArn") != self.attempts[request_id]["executionArn"]: + self.fail("event does not retain original execution ownership") + sequence = event.get("sequence") + if ( + type(sequence) is not int + or sequence < 1 + or (request_id, sequence) in sequences + ): + self.fail( + "events require unique positive per-request sequence numbers", + "INCOMPLETE", + ) + else: + sequences.add((request_id, sequence)) + if ( + not _number(self.attempts[request_id]["startedAtMs"]) + <= _number(event.get("atMs")) + <= _number(self.evidence.get("observationEndedAtMs")) + ): + self.fail( + "event lies outside its original observation window", "INCOMPLETE" + ) + telemetry = self.evidence.get("telemetry", {}) + views = telemetry.get("views", {}) + if ( + not isinstance(views, Mapping) + or set(views) != {self.view_name} + or not isinstance(views[self.view_name], Mapping) + ): + raise ValueError( + "the matrix cell must collect exactly its declared otelView" + ) + self.view = views[self.view_name] + for carrier in _records(self.view.get("carriers"), "carriers"): + request_id = carrier.get("requestId") + if request_id not in self.attempts or request_id in self.carriers: + raise ValueError("carriers need one record per observed request") + self.carriers[request_id] = carrier + if set(self.carriers) != set(self.attempts): + raise ValueError("carriers must include every original and resumed request") + return not self.errors + + def lifecycle(self) -> None: + budget = _number( + self.evidence.get("regression", {}).get("budgets", {}).get("exportMs") + ) + if budget <= 0: + self.fail( + "regression.budgets.exportMs must be declared before measurement", + "INCOMPLETE", + ) + plugin_instances: set[str] = set() + providers: dict[tuple[str, str], tuple[str, str]] = {} + for request_id, attempt in self.attempts.items(): + enter = self.one(request_id, "otel_enter") + end = self.one(request_id, "otel_exit") + factory = self.one(request_id, "otel_factory") + self.enters[request_id], self.exits[request_id] = enter, end + boundary = self.records(request_id, "wrapper_return") + self.records( + request_id, "wrapper_raise" + ) + if len(boundary) != 1: + self.fail( + f"{request_id} is missing the actual wrapper completion boundary", + "INCOMPLETE", + ) + else: + self.boundaries[request_id] = boundary[0] + if ( + not _number(enter.get("atMs")) + <= _number(factory.get("atMs")) + <= _number(boundary[0].get("atMs")) + <= _number(end.get("atMs")) + ): + self.fail( + f"{request_id} plugin diagnostics contradict the actual SDK wrapper boundary" + ) + before, after = enter.get("contextBefore"), end.get("contextAfter") + for snapshot in (before, after): + if not isinstance(snapshot, Mapping) or set(snapshot) != { + "traceId", + "spanId", + }: + self.fail( + "context restoration needs actual before/after trace/span snapshots", + "INCOMPLETE", + ) + elif snapshot["traceId"] is None and snapshot["spanId"] is None: + pass + else: + _identifier(snapshot["traceId"], 32) + _identifier(snapshot["spanId"], 16) + if before != after: + self.fail(f"{request_id} did not restore its prior OTel context") + if ( + type(end.get("exporterShutdowns")) is not int + or end["exporterShutdowns"] != 0 + ): + self.fail( + f"{request_id} shut down or failed to observe the shared exporter" + ) + instance = factory.get("pluginInstanceId") + if ( + not isinstance(instance, str) + or not instance + or instance in plugin_instances + ): + self.fail( + "Python plugin instances must be distinct per native invocation" + ) + else: + plugin_instances.add(instance) + if factory.get("pluginExecutionArn") != attempt["executionArn"]: + self.fail( + "plugin factory did not receive the original durable execution identity" + ) + provider, exporter = enter.get("providerId"), enter.get("exporterId") + if ( + not isinstance(provider, str) + or not provider + or not isinstance(exporter, str) + or not exporter + ): + self.fail( + "provider/exporter identities must be observed at entry", + "INCOMPLETE", + ) + continue + process = (attempt["environmentId"], attempt["processId"]) + if providers.setdefault(process, (provider, exporter)) != ( + provider, + exporter, + ): + self.fail("warm process replaced its shared provider/exporter") + + def spans(self) -> None: + expected_sampled = self.scenario != "runtime-carrier-unsampled" + if self.scenario in { + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + "otel-concurrent", + "held-export", + } and any( + attempt.get("outcome") != "SUCCEEDED" for attempt in self.attempts.values() + ): + self.fail( + f"{self.scenario} requires the original successful handler outcome" + ) + for request_id, carrier in self.carriers.items(): + if carrier.get("expectedSampled") is not expected_sampled: + self.fail( + f"{self.scenario} requires explicit Sampled={int(expected_sampled)} for every request" + ) + if self.scenario in ( + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + ): + actual = execution_trace_context( + {"TraceHeader": {"XAmznTraceId": carrier.get("header")}} + ) + caller = execution_trace_context( + {"TraceHeader": {"XAmznTraceId": carrier.get("callerHeader")}} + ) + if ( + actual.trace_id != caller.trace_id + or actual.sampled != caller.sampled + ): + self.fail( + f"{request_id} runtime carrier did not preserve the caller Root and explicit sampling" + ) + records = _records(self.view.get("spans"), "spans") + first_observation: dict[tuple[str, str], tuple[Any, Span]] = {} + occurrences: list[tuple[Any, Span]] = [] + observed: list[Mapping[str, Any]] = [] + for record in records: + span = _span(record.get("span")) + key = (span.trace_id, span.span_id) + occurrence = (record.get("requestId"), span) + if first_observation.setdefault(key, occurrence) != occurrence: + self.fail( + "duplicate span identity has inconsistent export observations" + ) + # Collapse identical transport receipts only. Preserve contradictory + # data and every native owner for attribution checks and diagnostics. + if occurrence not in occurrences: + occurrences.append(occurrence) + observed.append(record) + if record.get("sampled") is not True: + self.fail("recording exporter span has no sampled SpanContext") + # Preserve raw duplicates, but assert logical span identity/counts rather + # than promising exactly-once transport delivery. + assertions: dict[str, dict[str, Any]] = {} + histories = { + item["executionArn"]: item + for item in _records(self.evidence.get("histories"), "histories") + } + for arn in {attempt["executionArn"] for attempt in self.attempts.values()}: + requests = [ + request_id + for request_id, attempt in self.attempts.items() + if attempt["executionArn"] == arn + ] + expected: dict[str, Any] = { + "minimum_spans": 1, + "require_execution_correlation": True, + "allowed_execution_arns": [arn], + "span_assertions": [], + } + if expected_sampled: + expected["span_assertions"].append( + { + "select": {"name": "Invocation"}, + "count": len(requests), + "expect": {}, + } + ) + for request_id in requests: + invocations = [ + record + for record in observed + if record.get("requestId") == request_id + and record["span"]["name"] == "Invocation" + ] + if len(invocations) != int(expected_sampled): + self.fail( + f"{request_id} must emit exactly {int(expected_sampled)} logical Invocation span" + ) + if expected_sampled and self.scenario in _TERMINAL_SCENARIOS: + terminal = max( + requests, + key=lambda request_id: self.attempts[request_id]["endedAtMs"], + ) + status = histories.get(arn, {}).get("status") + if status not in ("SUCCEEDED", "FAILED"): + self.fail( + f"{arn} lacks a successful or failed terminal service history", + "INCOMPLETE", + ) + expected["span_assertions"].append( + { + "select": {"name": "Workflow"}, + "expect": { + "status": "OK" if status == "SUCCEEDED" else "ERROR" + }, + } + ) + workflows = [ + record + for record in observed + if record["span"]["name"] == "Workflow" + and self.attempts.get(record.get("requestId", ""), {}).get( + "executionArn" + ) + == arn + ] + if len(workflows) != 1 or workflows[0].get("requestId") != terminal: + self.fail( + f"{arn} must emit its Workflow once at the terminal invocation, not during replay" + ) + assertions[arn] = expected + selected = [ + _span(record["span"]) + for record in observed + if record.get("requestId") in requests + ] + if selected and expected_sampled: + query = TelemetryQuery( + arn, + selected[0].service_name or "", + min(span.start_time for span in selected), + max(span.end_time for span in selected), + ) + self.errors.extend( + f"ASSERTION: {error}" + for error in validate_trace( + Trace(selected[0].trace_id, tuple(selected)), expected, query + ) + ) + strict_view = {**self.view, "spans": observed, "assertions": assertions} + self.errors.extend( + _view_errors( + self.view_name, + strict_view, + self.attempts, + "python", + _number(self.evidence.get("observationEndedAtMs")), + execution_starts=_service_execution_starts( + self.evidence.get("histories", []) + ), + operation_histories=_records( + self.evidence.get("histories", []), "histories" + ), + ) + ) + outcomes = { + item["requestId"]: item.get("status") + for item in _records(self.view.get("exportOutcomes"), "exportOutcomes") + } + victim = self.evidence.get("selectedRequests", {}).get("victim") + for request_id in self.attempts: + expected_outcome = ( + "unsampled" + if not expected_sampled + else "export-error" + if self.scenario == "export-error-recovery" and request_id == victim + else "delivered" + ) + if outcomes.get(request_id) != expected_outcome: + self.fail( + f"{self.scenario}/{request_id} requires export outcome {expected_outcome}" + ) + exports = self.records(request_id, "otel_export") + starts = self.records(request_id, "otel_export_started") + if {(event.get("traceId"), event.get("spanId")) for event in starts} != { + (event.get("traceId"), event.get("spanId")) for event in exports + }: + self.fail( + f"{request_id} export start/completion observations disagree", + "INCOMPLETE", + ) + emitted = { + (record["span"]["trace_id"], record["span"]["span_id"]) + for record in records + if record.get("requestId") == request_id + } + if { + (event.get("traceId"), event.get("spanId")) for event in exports + } != emitted: + self.fail( + f"{request_id} export observations omit or add canonical spans", + "INCOMPLETE", + ) + expected_event = ( + "failure" + if expected_outcome == "export-error" + else "held" + if self.scenario == "held-export" + else "success" + ) + for event in exports: + begun = [ + start + for start in starts + if (start.get("traceId"), start.get("spanId")) + == (event.get("traceId"), event.get("spanId")) + ] + if len(begun) != 1: + self.fail( + f"{request_id} needs a unique observed exporter start", + "INCOMPLETE", + ) + elif ( + not 0 + <= _number(event.get("atMs")) - _number(begun[0].get("atMs")) + <= _number(self.evidence["regression"]["budgets"]["exportMs"]) + ): + self.fail( + f"{request_id} exporter exceeded its immutable exportMs budget" + ) + if event.get("outcome") != expected_event: + self.fail( + f"{request_id} exporter outcome differs from the selected fixture" + ) + if ( + not _number(self.enters[request_id].get("atMs")) + <= _number(event.get("atMs")) + <= _number(self.boundaries[request_id].get("atMs")) + ): + self.fail( + f"{request_id} exporter activity escaped its OTel lifetime" + ) + + def warm(self) -> None: + sequence = _records( + self.evidence.get("regression", {}).get("warmSequence"), + "regression.warmSequence", + ) + if [item.get("fixture") for item in sequence] != [ + "success", + "failure", + "replay", + ] * 2: + self.fail( + "OTel warm coverage requires success/failure/replay repeated twice", + "INCOMPLETE", + ) + return + arns = [item.get("executionArn") for item in sequence] + if len(set(arns)) != 6 or set(arns) != { + attempt["executionArn"] for attempt in self.attempts.values() + }: + self.fail( + "warm sequence must identify all six distinct actual executions", + "INCOMPLETE", + ) + return + previous_end = -1.0 + reused: set[tuple[str, str]] = set() + saw_reuse = False + for item in sequence: + attempts = sorted( + ( + attempt + for attempt in self.attempts.values() + if attempt["executionArn"] == item["executionArn"] + ), + key=lambda attempt: attempt["startedAtMs"], + ) + if not attempts or any( + attempt.get("fixture") != item["fixture"] for attempt in attempts + ): + self.fail( + "warm fixtures must match observed invocation routing", "INCOMPLETE" + ) + continue + if _number(attempts[0]["startedAtMs"]) <= previous_end: + self.fail("warm executions were not sequential") + previous_end = _number(attempts[-1]["endedAtMs"]) + key = (attempts[0]["environmentId"], attempts[0]["processId"]) + saw_reuse |= key in reused + reused.add(key) + expected = "FAILED" if item["fixture"] == "failure" else "SUCCEEDED" + history: Mapping[str, Any] = next( + ( + history + for history in self.evidence["histories"] + if history["executionArn"] == item["executionArn"] + ), + {}, + ) + if attempts[-1]["outcome"] != expected or history.get("status") != expected: + self.fail( + f"warm {item['fixture']} has the wrong terminal invocation outcome" + ) + if item["fixture"] == "replay": + from lmi_tests.python_lifecycle import validate_replay_evidence + + self.errors.extend( + validate_replay_evidence(self.evidence, item["executionArn"]) + ) + if not saw_reuse: + self.fail("warm fixtures never reused an identified process", "PLACEMENT") + + def concurrency(self) -> None: + capacity = self.evidence["profile"]["nativeConcurrency"] + originals = [ + attempt for attempt in self.attempts.values() if attempt.get("attempt") == 1 + ] + if len(originals) != capacity or len(self.attempts) != capacity: + self.fail( + "concurrent telemetry must occupy exactly the configured native cohort", + "PLACEMENT", + ) + return + if ( + len({attempt["environmentId"] for attempt in originals}) != 1 + or len({attempt["processId"] for attempt in originals}) != capacity + ): + self.fail( + "concurrent Python requests need distinct processes in one original environment", + "PLACEMENT", + ) + ordered = sorted( + originals, + key=lambda attempt: self.exits[attempt["requestId"]].get("atMs", 0), + ) + for index, attempt in enumerate(ordered): + request_id = attempt["requestId"] + blocked = self.one(request_id, "blocked") + release = self.one(request_id, "control_released") + if blocked.get("gateId") != release.get("gateId") or not blocked.get( + "gateId" + ): + self.fail("cohort control release must identify its held gate") + if ( + not _number(blocked.get("atMs")) + < _number(release.get("atMs")) + <= _number(self.exits[request_id].get("atMs")) + ): + self.fail("invocation ended before its own barrier was released") + for peer in ordered[index + 1 :]: + peer_id = peer["requestId"] + boundary = _number(self.exits[request_id].get("atMs")) + peer_release = self.one(peer_id, "control_released") + if ( + not _number(self.enters[peer_id].get("atMs")) + < boundary + < _number(peer_release.get("atMs")) + ): + self.fail( + "ending one invocation did not leave the established peer held" + ) + if not any( + boundary + < _number(event.get("atMs")) + < _number(peer_release.get("atMs")) + for event in self.records(peer_id, "alive") + ): + self.fail( + "held peer made no observed progress after the other invocation ended" + ) + + def export_recovery(self) -> None: + victim, healthy = self.selected("victim"), self.selected("warm") + if any( + self.attempts[victim][key] != self.attempts[healthy][key] + for key in ("environmentId", "processId", "workerId") + ): + self.fail( + "export recovery must reuse the original worker/process", "PLACEMENT" + ) + if _number(self.attempts[healthy]["startedAtMs"]) <= _number( + self.exits[victim].get("atMs") + ): + self.fail("healthy export did not occur after the failed invocation ended") + if ( + self.attempts[victim].get("outcome") != "SUCCEEDED" + or self.attempts[healthy].get("outcome") != "SUCCEEDED" + ): + self.fail( + "export fixture failure must not change either successful handler outcome" + ) + + def held(self) -> None: + victim = self.selected("victim") + held = self.one(victim, "export_held") + release = self.one(victim, "export_released") + if not held.get("gateId") or held.get("gateId") != release.get("gateId"): + self.fail("held export and release must identify the same owned gate") + faults = _records(self.evidence.get("faultsCreated"), "faultsCreated") + owned = [ + fault + for fault in faults + if fault.get("id") == held.get("gateId") + and fault.get("ownerCaseId") == self.evidence.get("caseId") + ] + if ( + len(owned) != 1 + or owned[0].get("releasedAtMs") != release.get("atMs") + or release.get("mode") != "normal" + ): + self.fail( + "held-export release lacks case-owned normal measurement evidence", + "INCOMPLETE", + ) + if ( + not _number(held.get("atMs")) + < _number(release.get("atMs")) + <= _number(self.boundaries[victim].get("atMs")) + ): + self.fail( + "held export did not prevent SDK wrapper completion until normal release" + ) + markers = { + (event.get("traceId"), event.get("spanId")) + for event in self.records(victim, "otel_export") + } + if (held.get("traceId"), held.get("spanId")) not in markers: + self.fail( + "held export was not bound to an observed exporter span", "INCOMPLETE" + ) + else: + key = (held.get("traceId"), held.get("spanId")) + started = [ + event + for event in self.records(victim, "otel_export_started") + if (event.get("traceId"), event.get("spanId")) == key + ] + completed = [ + event + for event in self.records(victim, "otel_export") + if (event.get("traceId"), event.get("spanId")) == key + ] + if ( + len(started) != 1 + or not completed + or not _number(started[0].get("atMs")) + <= _number(held.get("atMs")) + < _number(release.get("atMs")) + <= min(_number(event.get("atMs")) for event in completed) + ): + self.fail("held-export gate was not inside the actual exporter call") + + +def validate_lifecycle_telemetry( + evidence: Mapping[str, Any], *, phase: str = "final" +) -> list[str]: + """Validate one mandatory view/configuration cell before releasing faults. + + Call the core lifecycle regression validator as well. Both phases preserve + observation checks; final resource retirement remains the core's concern. + """ + if phase not in ("observations", "final"): + return ["INCOMPLETE: unknown validation phase"] + validator = _Telemetry(evidence) + try: + if not validator.common(): + return validator.errors + validator.lifecycle() + if validator.errors: + return validator.errors + validator.spans() + if validator.scenario == "otel-warm-lifecycle": + validator.warm() + elif validator.scenario == "otel-concurrent": + validator.concurrency() + elif validator.scenario == "export-error-recovery": + validator.export_recovery() + elif validator.scenario == "held-export": + validator.held() + except (AttributeError, KeyError, TypeError, ValueError, OverflowError) as error: + validator.fail(f"malformed OTel regression evidence: {error}", "INCOMPLETE") + return validator.errors diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index 653631460..da94d45ed 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -1,119 +1,86 @@ -# Shared LMI conformance adapter +# Python-owned LMI regression suite ## Problem -Python's LMI regression harness and the shared conformance runner currently use -independent drivers, evidence formats and acceptance reports. Merging both test -PRs does not make the Python fixtures satisfy shared coverage or ownership rules. +Python-specific lifecycle requirements were mixed into the language-neutral +conformance catalog. That made Python exception, thread, cleanup and plugin +rules look like shared SDK requirements and coupled their evolution to the shared +package release. ## Cause -The SDK harness owns the actual public-API functions, deployment, S3 ledger and -service calls. The shared runner owns the multi-invocation scenario matrix, -resource leases, validators and strict reports. The existing ledger does not yet -record every operation, export and lifecycle boundary the shared schema requires. +The implementation combined two separate responsibilities: reusable execution, +coordination and evidence handling, and the SDK-specific rules defining what a +Python fixture must do. ## Options considered -1. Keep invoking pytest from the shared adapter. This reuses the old command but - does not expose placement, collection and retirement as separate phases, or - bind every native invocation to the shared ownership journal. -2. Copy the shared validators into this repository. This preserves two competing - acceptance implementations and makes version/replay comparisons unreliable. -3. Adapt the existing Cloud/fixture implementation and add the missing public-API - observations, while delegating scheduling and acceptance to a pinned shared - runner. This reuses the actual SDK behavior and eliminates the independent CI - acceptance path without fabricating missing evidence. +1. Keep a Python-only profile in the shared catalog. This leaves the ownership + problem in place. +2. Copy the whole coordinator and reporter into the SDK. This duplicates the + lease, journal, cancellation and retirement machinery. +3. Keep Python definitions and validators in this repository, using the shared + runner through its programmatic suite interface. ## Chosen design -Use option 3, following the requested adoption of conformance PR #186. Pin the -shared runner revision. Generate its `python-lifecycle` and `export-lifecycle` -matrices for the two persistent c1/c2 functions: 76 mapped lifecycle/OTel cells -plus 24 exporter-mode cells. Keep exporter mode independent of language and choose -immutable provider variants per invocation. The shared conformance package runs -on the driver only, not inside the deployed function bundle. - -The SDK-owned adapter exposes prepare/start/place/execute/collect/release/retire -and finish. It keeps a distinct Cloud/controller state for every case, journals -all direct launches and observed service resumes, and records exact artifact, -configuration and original request identities. Public plugin hooks and the -existing forwarding Lambda client provide actual operation/checkpoint metadata. -Missing observations remain incomplete; no timestamps, completion events or SDK -capabilities are synthesized to force shared validation to pass. - -Lifecycle and telemetry records preserve actual boundary timestamps separately -from later diagnostic I/O. Normalization retains raw source evidence. Synchronous -exports settle before wrapper return; asynchronous exports and flush work retain -ownership after a bounded wait until actual completion. Case cleanup releases only -its own controls and executions. Persistent ownership uses conditional S3 writes -outside the expiring run prefix and is retained after unproven retirement. S3 -stays private and uses native SigV4/IAM authentication. - -CI runs the shared cloud suite automatically for same-repository PRs from -non-Dependabot actors and retains manual dispatch. Both jobs use the existing -CodeBuild runner; fork and Dependabot PRs run only the GitHub-hosted harness. -SDK-local unit/regression tests remain available, while cloud acceptance uses the -shared runner. Reports and all attempted -case evidence are retained. The implementation and tests use the real public SDK -handler/plugin paths against the local service where practical; real-LMI -placement/recovery and deployed acceptance remain explicitly unverified until a -cloud run is recorded. - -## Removed or superseded work - -The old pytest cloud command is no longer the primary CI acceptance path. Its -SDK-specific fixture/regression tests remain available during migration. Shared -requirements and validators are not duplicated here, and this change does not -claim to fix the open runtime deadline/cleanup defects tracked by Python #741. - -## Reviewable decisions and limits - -- This PR targets current main, reuses the Python #743 harness, and does not modify that PR's branch. -- Conformance is pinned to PR #186 revision `86bcaff49829351b25a1b1ae76f6d6483d37df83`. -- The default target is the Python-specific 76-cell matrix plus the portable - 24-cell exporter matrix; it does not imply ordinary SDK/OTel suite acceptance - on LMI or acceptance against the separate standard profile. -- Fixture budgets are declared before a run and kept distinct from product SLAs. -- SDK bugs stay failures. No cloud success or bounded arbitrary user-I/O - termination is inferred from local harness tests. - -## Observation boundaries - -The public SDK does not emit a root user-function hook. The fixture observes its -actual root entry/finally using the root operation ID supplied by public invocation -metadata. Operation names may be null. A Workflow span may start at the real -service execution start, before the current native invocation begins; history -retains that timestamp separately from invocation admission. - -The source ledger reserves capture order at each actual SDK boundary independently -of later S3 publication. Export release receipts record when the SDK observed the -control; driver release intent and acknowledgement remain separate observations. -Native timeout and wrapper return are independent outcomes. Deadline measurements -are frozen before cleanup releases the held step; cleanup cannot repair failed -behavior assertions. The shared error expectations explicitly enumerate allowed -service statuses where a measurement may precede execution termination. - -## Retained cloud evidence corrections - -The shared reader selects the declared held-work gate when the Python fixture -also records a winner-admission handshake. Execution-view operation spans that -cross a resume need matching service operation identity, status and timestamps; -the adapter retains the first actual service start and final terminal timestamp. -Per-request export records remain intact even when one logical span is delivered -again. Contradictory span observations remain failures. - -Deadline snapshots are read from the persisted pre-release measurement and bound -to the original case, execution, selected requests and declared time budgets. -The full event ledger remains available. Measurement does not authorize new user -work or SDK dispatch after the deadline cutoff; completion of already dispatched -requests is accounted for separately from the controller's normal fault release. - -A failed phase can provide a read-only partial collection before release. Its -original errors and incomplete behavior evidence stay in the report. The adapter -also supplies a separate post-cleanup retirement snapshot, with the actual final -read boundary. The shared coordinator independently checks every journaled launch, -known resource and completed task/request/export before allowing reuse. A missing -required role remains a test failure, while an unidentifiable actual launch or -unsettled work retains quarantine. An adapter's boolean retirement result alone -cannot make a failed case safe or turn it into a passing case. +Use option 3. The Python SDK owns `python_catalog.py`, `python_lifecycle.py` and +`python_otel.py`, their regression tests, and the corresponding evidence rules. +The local `python-lifecycle` suite has 28 baseline lifecycle cells and 48 OTel +cells. It reports `python-lmi-*` IDs under the `python-lmi` report namespace. +These results do not claim acceptance of shared conformance requirements. + +The shared repository owns only the language-neutral `standard` requirements +1–8 and `export-lifecycle` requirement 16. Its exporter profile runs 24 cells +across both modes, both views and native c1/c2. The Python workflow continues to +run those 24 shared cells alongside the 76 SDK-owned cells, using the same two +persistent functions. The separate shared standard profile is not implicitly +accepted by this combination. + +`SuiteDefinition` supplies local requirement metadata, the exact matrix and the +local validator to the common programmatic runner. It does not register a new +profile in the shared CLI or package Python-specific requirement resources in +shared wheels. The common runner still owns bounded scheduling, immutable +manifest snapshots, exact launch journals, common evidence checks, resource +leases, independent retirement auditing and strict JSON/JUnit reporting. + +Python-specific applicability, recovery policies and nonnegative resource-growth +budgets are declared in the local suite. Both OTel views and native c1/c2 remain +independent required cells. Missing capabilities, missing evidence and SDK bugs +stay failures; migration does not remove assertions or make the SDK implement a +capability it lacks. + +## Evidence and cleanup + +The adapter uses real public SDK hooks and real service history. It preserves +actual service operation timestamps and checkpoint results, immutable invocation +identities, captured SDK boundaries, exporter ownership and raw observations. +The validator selects the fixture's declared work gate separately from its +winner-admission handshake. Execution-view historical spans require matching +service-operation evidence. Contradictory span records remain failures. + +A persisted deadline measurement records the original cohort and bounds before +normal fault release. The full event ledger remains available; new user work and +SDK dispatch after the cutoff cannot be excused as cleanup. Completion of already +dispatched I/O is accounted for separately. + +Failed phases can retain partial observations before release. A separate final +snapshot may prove physical retirement without changing the original failure. +Every actual launch and owned resource must be accounted for; unknown requests, +missing completion or unpublished observations retain quarantine. Ownership uses +conditional S3 writes outside the expiring run prefix, with private storage and +native IAM/SigV4 authentication. + +## Integration and removed coupling + +CI automatically runs on trusted same-repository PRs through CodeBuild and keeps +manual dispatch. Fork and Dependabot PRs only run the unprivileged harness. The +conformance dependency is pinned to an exact reviewed revision in +`shared_contract.py` and the workflow. Setup can install that dependency before +importing any SDK-owned suite that uses it. + +Python-only shared requirement files, catalog branches, validators and their +self-tests are moved out of the shared repository. Their behavior is retained +here under local IDs. Existing SDK-local regression entry points remain available. +The driver-only catalogs and validators are not included in the Lambda bundle. +No production SDK fix or real-LMI acceptance is implied by local harness tests. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 97a2c6610..60648fe63 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run the SDK-owned fixtures through the pinned shared conformance package.""" +"""Run Python LMI regressions and shared exporter conformance.""" from __future__ import annotations @@ -85,6 +85,7 @@ def make_manifest( deployment: Mapping[str, Any], coverage: str, directory: Path ) -> dict[str, Any]: from aws_durable_execution_conformance_tests.lmi.matrix import required_cases + from lmi_tests import python_catalog if coverage not in COVERAGE_PROFILES: raise ValueError("Unknown Python adapter coverage profile") @@ -115,7 +116,12 @@ def make_manifest( for key in ("c1", "c2") ] scenarios = [] - for required in required_cases("python", "lifecycle-otel", coverage): + required_cells = ( + python_catalog.required_cases("lifecycle-otel") + if coverage == "python-lifecycle" + else required_cases("python", "lifecycle-otel", coverage) + ) + for required in required_cells: case_id = f"{coverage}-{required.scenario}-c{required.native_concurrency}-{required.otel_view or 'baseline'}-{required.export_mode or 'sdk-default'}" scenarios.append( { @@ -162,6 +168,8 @@ def make_manifest( def run(deployment: Mapping[str, Any], directory: Path) -> int: + from lmi_tests import python_catalog + from aws_durable_execution_conformance_tests.lmi.app import ( build_report, coverage_inventory, @@ -190,17 +198,27 @@ def run(deployment: Mapping[str, Any], directory: Path) -> int: plan = make_manifest(deployment, coverage, output) path = output / "manifest.json" save(path, plan) - validated = load_manifest(path) + suite = ( + python_catalog.suite_definition(plan["capabilityProfile"]) + if coverage == "python-lifecycle" + else None + ) + validated = ( + python_catalog.load_manifest(path) + if suite is not None + else load_manifest(path) + ) results = asyncio.run( live_results( validated, "lmi_tests.shared_adapter:create_adapter", audit_dir=output, + suite=suite, ) ) - report = build_report(validated, results) + report = build_report(validated, results, suite=suite) save(output / "evidence.json", results) - save(output / "coverage.json", coverage_inventory(validated)) + save(output / "coverage.json", coverage_inventory(validated, suite=suite)) print(render_console(report)) for format_name in ("json", "junit"): write_report(report, format_name, str(output / "report")) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py index 367ef7fa6..a81bc766d 100644 --- a/lmi_tests/shared_contract.py +++ b/lmi_tests/shared_contract.py @@ -10,7 +10,7 @@ CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" # Updated only with the matching reviewed shared evidence contract. -CONFORMANCE_REVISION = "86bcaff49829351b25a1b1ae76f6d6483d37df83" +CONFORMANCE_REVISION = "59352e1097d92fe034c7cd55a9618837b95b905b" def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: diff --git a/lmi_tests/tests/e2e/test_shared_driver_local.py b/lmi_tests/tests/e2e/test_shared_driver_local.py index bd81e8923..7e51503c3 100644 --- a/lmi_tests/tests/e2e/test_shared_driver_local.py +++ b/lmi_tests/tests/e2e/test_shared_driver_local.py @@ -107,7 +107,7 @@ def test_shared_callback_driver_resumes_the_original_public_sdk_execution(): "local-run", Scenario( "callback-case", - "lmi-lifecycle-5", + "python-lmi-5", profile.profile_id, "replay-callback-retry", roles=("victim", "resume"), diff --git a/lmi_tests/tests/e2e/test_shared_evidence_local.py b/lmi_tests/tests/e2e/test_shared_evidence_local.py index 072ece0ef..05f020ce4 100644 --- a/lmi_tests/tests/e2e/test_shared_evidence_local.py +++ b/lmi_tests/tests/e2e/test_shared_evidence_local.py @@ -6,7 +6,7 @@ import time from pathlib import Path -from aws_durable_execution_conformance_tests.lmi.app import validate +from lmi_tests.python_catalog import validate from aws_durable_execution_conformance_tests.lmi.coordinator import ( Profile, Scenario, @@ -41,7 +41,7 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( "unit", Scenario( "case", - "lmi-lifecycle-5", + "python-lmi-5", "c1", "replay-callback-retry", roles=("victim", "resume"), diff --git a/lmi_tests/tests/unit/test_python_lifecycle.py b/lmi_tests/tests/unit/test_python_lifecycle.py new file mode 100644 index 000000000..c1e1c2f21 --- /dev/null +++ b/lmi_tests/tests/unit/test_python_lifecycle.py @@ -0,0 +1,1189 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Complete retained controls for every Python lifecycle coverage cell.""" + +from copy import deepcopy +from typing import Any + +import pytest + +from lmi_tests.python_lifecycle import ( + SCENARIOS, + validate_lifecycle_events, + validate_lifecycle_regressions, + validate_replay_evidence, +) + + +def resources() -> dict[str, Any]: + return { + "threads": [{"id": "runtime", "name": "MainThread"}], + "fds": 10, + "rssBytes": 1_000_000, + } + + +class _Builder: + def __init__(self, scenario: str, native: int) -> None: + self.data: dict[str, Any] = { + "schemaVersion": 1, + "coverageProfile": "python-lifecycle", + "scenario": scenario, + "runtime": "python", + "runId": "run", + "caseId": "case", + "cohortId": "cohort-1", + "requirementId": f"python-lmi-{SCENARIOS[scenario]}", + "artifact": { + "sdk": "sdk@sha256:tested", + "conformance": "conformance@sha256:tested", + }, + "profile": { + "functionArn": f"arn:function:c{native}", + "configurationDigest": "sha256:configuration", + "nativeConcurrency": native, + "timeoutMs": 500, + "recoveryPolicy": "reuse-worker", + }, + "attempts": [], + "events": [], + "selectedRequests": {}, + "histories": [], + "expectedHistories": {}, + "sdkRequests": [], + "faultsCreated": [], + "resourcesCreated": [], + "observationEndedAtMs": 20_000, + "collection": { + "watermarks": { + "lifecycle": 20_000, + "sdkRequests": 20_000, + "history": 20_000, + }, + "evidenceUris": ["s3://evidence/run/case/raw.json"], + }, + "retirement": {"retiredAtMs": 21_000, "resources": [], "faults": []}, + "regression": {"budgets": {"observationGraceMs": 5}}, + } + self.requests: dict[str, dict[str, Any]] = {} + self.histories: dict[str, dict[str, Any]] = {} + + def request( + self, + role: str, + start: int = 100, + end: int = 1_000, + *, + outcome: str = "SUCCEEDED", + process: int = 1, + execution: str | None = None, + fixture: str = "success", + ) -> dict[str, Any]: + arn = execution or f"arn:execution:{role}" + marker = f"marker:{arn}" + request = { + "requestId": f"request:{role}", + "executionArn": arn, + "environmentId": "environment-1", + "workerId": f"worker-{process}", + "processId": f"process-{process}", + "pid": process + 100, + "role": role, + "attempt": 1, + "startedAtMs": start, + "endedAtMs": end, + "outcome": outcome, + "marker": marker, + "fixture": fixture, + } + self.requests[role] = request + self.data["attempts"].append(request) + self.data["selectedRequests"][role] = request["requestId"] + self.histories.setdefault( + arn, + { + "executionArn": arn, + "ownerCaseId": "case", + "status": "SUCCEEDED", + "result": marker, + "operations": [], + }, + ) + self.event(role, "wrapper_enter", start, resources=resources()) + if outcome != "TIMED_OUT": + self.event(role, "user_exit", end - 2) + self.event( + role, "wrapper_return", end, outcome=outcome, resources=resources() + ) + else: + request["deadlineAtMs"] = start + 500 + self.event( + role, "runtime_deadline", start, source="runtime", remainingTimeMs=500 + ) + self.event(role, "platform_timeout", start + 500, source="platform") + self.event(role, "task_exit", start + 550, taskId="root") + self.op(role, "success", "SUCCEEDED") + if execution is None: + self.body(role, "success", start + 5) + if outcome == "FAILED": + self.histories[arn].pop("result", None) + self.histories[arn].update( + status="FAILED", + error={"type": "ValueError", "message": "expected:" + marker}, + ) + return request + + def event(self, role: str, kind: str, time: int, **extra: Any) -> dict[str, Any]: + request = self.requests[role] + item = { + "kind": kind, + "requestId": request["requestId"], + "ownerRequestId": request["requestId"], + "executionArn": request["executionArn"], + "atMs": time, + "sequence": 0, + **extra, + } + self.data["events"].append(item) + return item + + def op( + self, + role: str, + name: str, + status: str = "SUCCEEDED", + kind: str = "STEP", + **extra: Any, + ) -> dict[str, Any]: + history = self.histories[self.requests[role]["executionArn"]] + existing = next( + (item for item in history["operations"] if item["name"] == name), None + ) + if existing is not None: + return existing + operation = { + "id": f"{history['executionArn']}:{name}", + "name": name, + "type": kind, + "parentId": None, + "status": status, + **extra, + } + history["operations"].append(operation) + return operation + + def body(self, role: str, name: str, time: int, *, attempt: int = 1) -> None: + operation = self.op(role, name, "FAILED" if name == "failure" else "SUCCEEDED") + self.event( + role, + "body", + time, + operation=name, + operationId=operation["id"], + attempt=attempt, + value=self.requests[role]["marker"], + ) + + def block( + self, + role: str, + start: int, + end: int, + *, + gate: str | None = None, + release: int | None = None, + ) -> str: + gate = gate or f"gate:{role}" + self.data["faultsCreated"].append({"id": gate, "ownerCaseId": "case"}) + self.data["retirement"]["faults"].append( + { + "id": gate, + "ownerCaseId": "case", + "releasedAtMs": release or 20_500, + "mode": "normal", + } + ) + self.event(role, "io_start", start - 10, taskId=gate, gateId=gate) + self.event(role, "blocked", start, gateId=gate) + self.event(role, "effect", start + 20, gateId=gate) + self.event(role, "alive", start + 40, gateId=gate) + self.event(role, "io_exit", end, taskId=gate, gateId=gate) + if release is not None: + self.event(role, "fault_released", release, gateId=gate, mode="normal") + return gate + + def replay( + self, + role: str, + start: int, + *, + resume_role: str = "resume", + fixture: str = "replay", + ) -> None: + original = self.request( + role, start, start + 100, outcome="PENDING", fixture=fixture + ) + self.request( + resume_role, + start + 150, + start + 300, + execution=original["executionArn"], + fixture=fixture, + ) + self.body(role, "failure", start + 10) + self.body(role, "retry", start + 20, attempt=1) + self.body(role, "retry", start + 30, attempt=2) + error = {"type": "ValueError", "message": "expected:" + original["marker"]} + self.event(role, "stored_failure", start + 15, error=error) + self.event(resume_role, "stored_failure", start + 160, error=deepcopy(error)) + for name, kind in ( + ("pause", "WAIT"), + ("completion", "CALLBACK"), + ("nested", "CONTEXT"), + ): + self.op(role, name, kind=kind) + + def finish(self) -> dict[str, Any]: + for request in self.data["attempts"]: + local = sorted( + [ + item + for item in self.data["events"] + if item["requestId"] == request["requestId"] + ], + key=lambda item: item["atMs"], + ) + for index, item in enumerate(local): + item["sequence"] = index + self.data["resourcesCreated"].append( + { + "id": request["requestId"], + "kind": "invocation", + "ownerCaseId": "case", + } + ) + self.data["histories"] = list(self.histories.values()) + for arn, history in self.histories.items(): + self.data["expectedHistories"][arn] = { + key: deepcopy(history[key]) + for key in ("status", "result", "error") + if key in history + } + self.data["resourcesCreated"].append( + {"id": arn, "kind": "execution", "ownerCaseId": "case"} + ) + self.data["retirement"]["resources"] = [ + { + **item, + "settledAtMs": 20_900, + "evidenceUri": "s3://evidence/run/case/retired.json", + "outcome": "QUIESCENT" if item["kind"] == "invocation" else "SUCCEEDED", + } + for item in self.data["resourcesCreated"] + ] + self.data["sdkRequests"] = [ + {**item, "method": "CheckpointDurableExecution", "outcome": "DISPATCHED"} + for item in self.data["events"] + if item["kind"] == "checkpoint_call" + ] + return self.data + + +def build_evidence(scenario: str, native: int = 1) -> dict[str, Any]: + """Synthetic positive controls, never claims of deployed SDK acceptance.""" + b = _Builder(scenario, native) + budgets = b.data["regression"]["budgets"] + if scenario == "replay-callback-retry": + b.replay("victim", 100) + elif scenario == "warm-lifecycle": + budgets.update( + maxThreadGrowth=1, maxFdGrowth=8, maxRssGrowthBytes=32 * 1024 * 1024 + ) + sequence = [] + for index, fixture in enumerate(["success", "failure", "replay"] * 3): + role = "victim" if index == 0 else "warm" if index == 1 else f"warm-{index}" + start = 100 + index * 400 + if fixture == "replay": + b.replay(role, start, resume_role=f"{role}-resume") + else: + b.request( + role, + start, + start + 300, + outcome="FAILED" if fixture == "failure" else "SUCCEEDED", + fixture=fixture, + ) + sequence.append( + {"executionArn": b.requests[role]["executionArn"], "fixture": fixture} + ) + b.data["regression"]["warmSequence"] = sequence + elif scenario in ("deadline-recovery", "timeout-retry"): + budgets.update(cleanupMs=100) + victim = b.request("victim", end=600, outcome="TIMED_OUT") + b.block("victim", 200, 640) + if scenario == "timeout-retry": + budgets["retryMs"] = 500 + b.request("retry", 750, 1_000, execution=victim["executionArn"]) + b.event( + "retry", + "service_retry", + 751, + source="platform", + previousRequestId=victim["requestId"], + ) + else: + budgets["recoveryMs"] = 100 + if native == 2: + b.request("peer", 110, 500, process=2) + b.block("peer", 210, 450, release=440) + b.request("recovery", 650, 1_000) + b.block("recovery", 670, 800, release=790) + if native == 2: + b.request("recovery-peer", 660, 1_010, process=2) + b.block("recovery-peer", 680, 810, release=800) + elif scenario == "pending-cleanup": + original = b.request("victim", end=500, outcome="PENDING") + b.request("resume", 600, 900, execution=original["executionArn"]) + gate = b.block("victim", 220, 400, release=390) + b.data["regression"]["gateId"] = gate + b.event("victim", "cleanup_enter", 180) + b.event("victim", "cleanup_exit", 410) + b.op("victim", "pause", kind="WAIT") + if native == 2: + b.request("peer", 110, 1_000, process=2) + b.block("peer", 210, 800, release=790) + b.event("peer", "peer_progress", 300) + elif scenario == "checkpoint-settlement": + b.request("victim") + gate = b.block("victim", 180, 810, release=800) + b.data["regression"]["gateId"] = gate + b.event("victim", "checkpoint_call", 150, callId="checkpoint-1") + operation = b.op("victim", "checkpoint-loser") + pool = b.op("victim", "checkpoint-pool", kind="CONTEXT") + b.event( + "victim", + "checkpoint_ack", + 160, + callId="checkpoint-1", + source="service", + operationIds=[operation["id"]], + ) + b.event( + "victim", + "ack_held", + 170, + callId="checkpoint-1", + gateId=gate, + operationId=operation["id"], + ) + b.event("victim", "checkpoint_exit", 820, callId="checkpoint-1") + b.event("victim", "branch_join", 830, scopeId=pool["id"]) + elif scenario.startswith("early-completion-") or scenario in ( + "return-inflight", + "failure-inflight", + "late-operation", + ): + early = scenario.startswith("early-completion-") + b.request( + "victim", + end=400 if early else 1_000, + outcome="FAILED" if scenario == "failure-inflight" else "SUCCEEDED", + ) + gate = b.block( + "victim", 200, 350 if early else 810, release=None if early else 800 + ) + b.data["regression"]["gateId"] = gate + b.event("victim", "winner_ready", 250) + b.event("victim", "winner_selected", 260) + if early: + budgets["cleanupMs"] = 200 + names = ( + ("race-map",) + if scenario.endswith("map") + else ("outer-map", "nested-child", "race-parallel") + if scenario.endswith("nested") + else ("race-parallel",) + ) + for name in names: + b.op("victim", name, kind="CONTEXT") + elif scenario == "late-operation": + scope = b.op("victim", "losing-child", kind="CONTEXT")["id"] + b.event("victim", "scope_closed", 300, scopeId=scope) + b.event("victim", "late_attempt", 830, scopeId=scope) + b.event( + "victim", + "late_rejected", + 840, + scopeId=scope, + error={"type": "OrphanedChildException"}, + ) + else: + next(item for item in b.data["events"] if item["kind"] == "user_exit")[ + "atMs" + ] = 600 + if native == 2: + b.request("peer", 110, 1_100, process=2) + b.block("peer", 210, 900, release=890) + b.event("peer", "peer_progress", 700) + else: + b.request("victim") + b.block( + "victim", + 200, + 310 if scenario == "nested-progress" else 800, + release=300 if scenario == "nested-progress" else 790, + ) + if native == 2: + b.request("peer", 110, 1_010, process=2) + b.block( + "peer", + 210, + 320 if scenario == "nested-progress" else 810, + release=310 if scenario == "nested-progress" else 800, + ) + if scenario == "nested-progress": + budgets["progressMs"] = 500 + for role, request in b.requests.items(): + b.event(role, "progress_begin", 400) + b.event( + role, + "progress", + 700, + value=[ + [request["marker"] + ":0", request["marker"] + ":1"], + request["marker"], + ], + ) + for index, name in enumerate(("outer-parallel", "inner-map")): + operation = b.op(role, name, kind="CONTEXT") + b.event( + role, + "operation_config", + 410 + index, + operationId=operation["id"], + source="public-fixture", + maxConcurrency=1, + ) + b.op(role, "progress-child", kind="CONTEXT") + return b.finish() + + +def find(value: dict[str, Any], kind: str, role: str = "victim") -> dict[str, Any]: + return next( + item + for item in value["events"] + if item["kind"] == kind and item["requestId"] == value["selectedRequests"][role] + ) + + +@pytest.mark.parametrize("native", [1, 2]) +@pytest.mark.parametrize("scenario", sorted(SCENARIOS)) +def test_all_28_lifecycle_cells_have_positive_controls( + scenario: str, native: int +) -> None: + assert validate_lifecycle_regressions(build_evidence(scenario, native)) == [] + + +@pytest.mark.parametrize("scenario", sorted(SCENARIOS)) +def test_each_case_rejects_missing_wrapper_evidence(scenario: str) -> None: + value = build_evidence(scenario) + value["events"] = [ + item for item in value["events"] if item["kind"] != "wrapper_enter" + ] + assert validate_lifecycle_regressions(value) + + +@pytest.mark.parametrize( + "scenario", + ["early-completion-parallel", "early-completion-map", "early-completion-nested"], +) +def test_early_completion_cannot_be_supplied_by_release(scenario: str) -> None: + value = build_evidence(scenario) + value["retirement"]["faults"][0]["releasedAtMs"] = 300 + assert any( + "driver release" in item for item in validate_lifecycle_regressions(value) + ) + + +def test_loser_must_be_active_before_winner() -> None: + value = build_evidence("early-completion-parallel") + value["events"] = [item for item in value["events"] if item["kind"] != "effect"] + assert any( + "actually held" in item for item in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize( + "kind", ["body", "effect", "checkpoint_ack", "checkpoint_exit", "poll_exit"] +) +def test_late_activity_and_inflight_settlement_are_separate_assertions( + kind: str, +) -> None: + value = build_evidence("return-inflight") + event = deepcopy(find(value, "user_exit")) + event.update(kind=kind, sequence=100, atMs=1_001) + value["events"].append(event) + assert any( + "after wrapper exit" in item for item in validate_lifecycle_regressions(value) + ) + + +def test_held_checkpoint_requires_real_ack_and_ordered_exit() -> None: + value = build_evidence("checkpoint-settlement") + find(value, "checkpoint_ack")["source"] = "fixture" + assert validate_lifecycle_regressions(value) + value = build_evidence("checkpoint-settlement") + find(value, "checkpoint_exit")["atMs"] = 799 + find(value, "checkpoint_exit")["sequence"] = find(value, "io_exit")["sequence"] - 1 + assert any( + "held until" in item or "clock" in item + for item in validate_lifecycle_regressions(value) + ) + + +def test_real_timeout_is_not_step_retry_or_driver_replacement() -> None: + value = build_evidence("timeout-retry") + find(value, "service_retry", "retry")["source"] = "driver" + assert any( + "driver-created" in item for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("timeout-retry") + success = deepcopy(find(value, "body")) + retry = next(item for item in value["attempts"] if item["role"] == "retry") + success.update( + requestId=retry["requestId"], + ownerRequestId=retry["requestId"], + atMs=800, + sequence=2, + ) + value["events"].append(success) + assert validate_lifecycle_regressions(value) + + +def test_recovery_needs_original_environment_all_slots_and_bound() -> None: + value = build_evidence("deadline-recovery", 2) + next(item for item in value["attempts"] if item["role"] == "recovery-peer")[ + "environmentId" + ] = "replacement" + assert any( + "unrelated environment" in item + for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("deadline-recovery") + value["regression"]["budgets"]["recoveryMs"] = 10 + assert any( + "recover within" in item for item in validate_lifecycle_regressions(value) + ) + + +def test_pending_cannot_precede_finally() -> None: + value = build_evidence("pending-cleanup") + find(value, "cleanup_exit")["sequence"] = 100 + find(value, "cleanup_exit")["atMs"] = 501 + assert any("finally" in item for item in validate_lifecycle_regressions(value)) + + +def test_nested_progress_is_not_satisfied_by_unrelated_peer() -> None: + value = build_evidence("nested-progress", 2) + find(value, "progress", "peer")["value"] = find(value, "progress")["value"] + assert any( + "crossed invocation" in item for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("nested-progress") + find(value, "operation_config")["maxConcurrency"] = 2 + assert any("single-lane" in item for item in validate_lifecycle_regressions(value)) + + +@pytest.mark.parametrize("scenario", ["return-inflight", "failure-inflight"]) +def test_inflight_work_must_settle_before_actual_wrapper_exit(scenario: str) -> None: + value = build_evidence(scenario) + find(value, "io_exit").update(sequence=100, atMs=1_001) + assert any("unsettled" in item for item in validate_lifecycle_regressions(value)) + + +def test_failure_survives_scope_settlement() -> None: + value = build_evidence("failure-inflight") + value["histories"][0]["error"]["message"] = "cleanup replaced root failure" + assert any( + "original root failure" in item + for item in validate_lifecycle_regressions(value) + ) + + +def test_abandoned_child_is_checked_before_invocation_closure() -> None: + value = build_evidence("late-operation") + find(value, "late_rejected")["error"] = {"type": "UnrelatedError"} + assert any( + "completed child scope" in item + for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("late-operation") + value["histories"][0]["operations"].append( + { + "id": "late", + "name": "late-work", + "type": "STEP", + "parentId": "child", + "status": "SUCCEEDED", + } + ) + assert any( + "reached the durable service" in item + for item in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize("counter", ["fds", "rssBytes", "threads"]) +def test_warm_resource_growth_uses_post_cache_baseline(counter: str) -> None: + value = build_evidence("warm-lifecycle") + last = sorted( + [item for item in value["events"] if item["kind"] == "wrapper_return"], + key=lambda item: item["atMs"], + )[-1] + if counter == "threads": + last["resources"][counter] += [ + {"id": str(index), "name": "retained-cache-thread"} for index in range(10) + ] + else: + last["resources"][counter] += 100_000_000 + assert any( + "post-cache allowance" in item for item in validate_lifecycle_regressions(value) + ) + + +def test_replay_requires_real_callback_wait_context_and_stored_failure() -> None: + value = build_evidence("replay-callback-retry") + value["histories"][0]["operations"] = [ + item + for item in value["histories"][0]["operations"] + if item["type"] != "CALLBACK" + ] + assert any("CALLBACK" in item for item in validate_lifecycle_regressions(value)) + value = build_evidence("replay-callback-retry") + find(value, "stored_failure", "resume")["error"]["type"] = "ChangedError" + assert any("rethrow" in item for item in validate_lifecycle_regressions(value)) + + +def test_common_lifecycle_and_per_execution_replay_are_reusable_for_otel() -> None: + value = build_evidence("replay-callback-retry") + assert validate_lifecycle_events(value) == [] + assert validate_replay_evidence(value, value["attempts"][0]["executionArn"]) == [] + + +@pytest.mark.parametrize("invalid", [None, [], False, 1, float("nan"), "bad"]) +def test_malformed_regression_data_returns_errors(invalid: Any) -> None: + for key in ("regression", "attempts", "events", "profile", "histories"): + value = build_evidence("native-concurrency") + value[key] = invalid + assert validate_lifecycle_regressions(value) + + +@pytest.mark.parametrize( + "scenario,requirement", + [ + ("runtime-carrier-sampled", 7), + ("runtime-carrier-unsampled", 7), + ("otel-warm-lifecycle", 7), + ("otel-concurrent", 7), + ("export-error-recovery", 8), + ("held-export", 8), + ], +) +def test_otel_only_cells_keep_common_lifetimes_for_extension_validation( + scenario: str, requirement: int +) -> None: + value = build_evidence("native-concurrency") + value.update(scenario=scenario, requirementId=f"python-lmi-{requirement}") + assert validate_lifecycle_regressions(value) == [] + + +def test_unrelated_ack_and_missing_branch_join_cannot_prove_checkpoint_settlement() -> ( + None +): + value = build_evidence("checkpoint-settlement") + find(value, "checkpoint_ack")["operationIds"] = ["unrelated-operation"] + assert any( + "selected real checkpoint" in item + for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("checkpoint-settlement") + value["events"] = [ + item for item in value["events"] if item["kind"] != "branch_join" + ] + assert any("branch_join" in item for item in validate_lifecycle_regressions(value)) + + +def test_static_body_names_may_repeat_in_distinct_service_contexts() -> None: + value = build_evidence("native-concurrency") + body = deepcopy(find(value, "body")) + value["histories"][0]["operations"].append( + { + "id": "child-success", + "name": "success", + "parentId": "child", + "status": "SUCCEEDED", + "type": "STEP", + } + ) + body.update(operationId="child-success", sequence=50, atMs=900) + value["events"].append(body) + for sequence, event in enumerate( + sorted(value["events"], key=lambda item: item["atMs"]) + ): + event["sequence"] = sequence + assert validate_lifecycle_events(value) == [] + + +@pytest.mark.parametrize("policy", [None, "undefined", False]) +def test_recovery_cannot_bypass_original_capacity_with_an_unknown_policy( + policy: Any, +) -> None: + value = build_evidence("deadline-recovery") + if policy is None: + value["profile"].pop("recoveryPolicy") + else: + value["profile"]["recoveryPolicy"] = policy + recovery = next(item for item in value["attempts"] if item["role"] == "recovery") + recovery.update(processId="replacement", workerId="replacement", pid=999) + assert any( + "supported recoveryPolicy" in item + for item in validate_lifecycle_regressions(value) + ) + + +def test_boolean_retry_attempt_and_lane_count_are_not_integers() -> None: + value = build_evidence("replay-callback-retry") + next( + item + for item in value["events"] + if item["kind"] == "body" and item["operation"] == "retry" + )["attempt"] = True + assert any( + "two configured attempts" in item + for item in validate_lifecycle_regressions(value) + ) + value = build_evidence("nested-progress") + find(value, "operation_config")["maxConcurrency"] = True + assert any("single-lane" in item for item in validate_lifecycle_regressions(value)) + + +def test_runtime_remaining_time_accounts_for_dispatch_time_before_wrapper_entry() -> ( + None +): + value = build_evidence("timeout-retry") + value["profile"]["timeoutMs"] = 1_000 + # The configured timer began 500ms before the observed SDK wrapper entry. + assert value["attempts"][0]["deadlineAtMs"] == 600 + assert find(value, "runtime_deadline")["remainingTimeMs"] == 500 + assert validate_lifecycle_regressions(value) == [] + + +@pytest.mark.parametrize("missing", ["deadlineAtMs", "runtime_deadline"]) +def test_actual_runtime_deadline_evidence_is_mandatory(missing: str) -> None: + value = build_evidence("deadline-recovery") + if missing == "deadlineAtMs": + value["attempts"][0].pop(missing) + else: + value["events"] = [item for item in value["events"] if item["kind"] != missing] + assert validate_lifecycle_regressions(value) + + +@pytest.mark.parametrize("change", ["source", "deadline", "remaining", "boolean"]) +def test_counterfeit_or_inconsistent_runtime_deadline_is_rejected(change: str) -> None: + value = build_evidence("timeout-retry") + observation = find(value, "runtime_deadline") + if change == "source": + observation["source"] = "input" + elif change == "deadline": + value["attempts"][0]["deadlineAtMs"] += 5 + elif change == "remaining": + observation["remainingTimeMs"] = 501 + else: + observation["remainingTimeMs"] = True + assert validate_lifecycle_regressions(value) + + +def _resequence(value: dict[str, Any]) -> None: + for request in value["attempts"]: + events = sorted( + ( + item + for item in value["events"] + if item["requestId"] == request["requestId"] + ), + key=lambda item: item["atMs"], + ) + for sequence, item in enumerate(events): + item["sequence"] = sequence + + +def _timeout_wrapper( + outcome: str = "FAILED", kind: str = "wrapper_raise" +) -> dict[str, Any]: + value = build_evidence("timeout-retry") + owner = { + key: value["events"][0][key] + for key in ("requestId", "ownerRequestId", "executionArn") + } + value["events"].extend( + [ + {**owner, "kind": "task_start", "atMs": 101, "taskId": "root"}, + {**owner, "kind": "user_exit", "atMs": 645}, + { + **owner, + "kind": kind, + "atMs": 660, + "outcome": outcome, + "resources": resources(), + }, + ] + ) + _resequence(value) + return value + + +@pytest.mark.parametrize( + "outcome,kind", + [ + ("FAILED", "wrapper_raise"), + ("SUCCEEDED", "wrapper_return"), + ("PENDING", "wrapper_return"), + ], +) +def test_actual_sdk_wrapper_outcome_remains_independent_after_platform_timeout( + outcome: str, kind: str +) -> None: + value = _timeout_wrapper(outcome, kind) + assert value["attempts"][0]["outcome"] == "TIMED_OUT" + assert validate_lifecycle_regressions(value) == [] + assert find(value, kind)["outcome"] == outcome + + +def test_timeout_wrapper_exception_does_not_erase_task_settlement_checks() -> None: + value = _timeout_wrapper() + find(value, "io_exit")["atMs"] = 670 + _resequence(value) + assert any("unsettled" in error for error in validate_lifecycle_regressions(value)) + + +def test_timeout_wrapper_does_not_erase_platform_or_late_activity_evidence() -> None: + value = _timeout_wrapper() + value["events"] = [ + item for item in value["events"] if item["kind"] != "platform_timeout" + ] + assert any( + "platform_timeout" in error for error in validate_lifecycle_regressions(value) + ) + value = _timeout_wrapper() + find(value, "body")["atMs"] = 680 + _resequence(value) + assert any( + "after wrapper exit" in error for error in validate_lifecycle_regressions(value) + ) + + +def test_wrapper_outcomes_remain_required_and_exact_without_platform_timeout() -> None: + value = _timeout_wrapper() + find(value, "wrapper_raise")["outcome"] = "invented" + assert any( + "unrecognized" in error for error in validate_lifecycle_regressions(value) + ) + value = build_evidence("native-concurrency") + find(value, "wrapper_return")["outcome"] = "FAILED" + assert any("differs" in error for error in validate_lifecycle_regressions(value)) + + +def test_public_operation_end_can_close_parent_before_winner_result_returns() -> None: + value = build_evidence("late-operation") + find(value, "scope_closed")["atMs"] = 255 + assert find(value, "winner_selected")["atMs"] == 260 + _resequence(value) + assert validate_lifecycle_regressions(value) == [] + + +@pytest.mark.parametrize("boundary", ["scope_closed", "winner_selected"]) +def test_both_parent_closure_and_winner_selection_must_precede_late_attempt( + boundary: str, +) -> None: + value = build_evidence("late-operation") + find(value, boundary)["atMs"] = 835 + _resequence(value) + assert any( + "completed child scope" in error + for error in validate_lifecycle_regressions(value) + ) + + +def _add_handshake_gate( + value: dict[str, Any], role: str = "victim", *, before: bool = False +) -> None: + """Real fixtures observe losing I/O and a separate winner-admission gate.""" + blocked = find(value, "blocked", role) + gate = blocked["gateId"] + "-handshake" + start = blocked["atMs"] + (-10 if before else 10) + for offset, kind in ( + (-1, "io_start"), + (0, "blocked"), + (1, "alive"), + (20, "fault_released"), + (21, "io_exit"), + ): + value["events"].append( + { + **blocked, + "kind": kind, + "atMs": start + offset, + "gateId": gate, + "taskId": gate, + "mode": "normal", + } + ) + value["faultsCreated"].append({"id": gate, "ownerCaseId": value["caseId"]}) + value["retirement"]["faults"].append( + { + "id": gate, + "ownerCaseId": value["caseId"], + "releasedAtMs": start + 20, + "mode": "normal", + } + ) + _resequence(value) + + +@pytest.mark.parametrize("scenario", ["return-inflight", "failure-inflight"]) +@pytest.mark.parametrize("handshake_first", [False, True]) +def test_peer_overlap_selects_declared_losing_work_gate( + scenario: str, handshake_first: bool +) -> None: + value = build_evidence(scenario, 2) + _add_handshake_gate(value, before=handshake_first) + assert validate_lifecycle_regressions(value) == [] + + +@pytest.mark.parametrize( + "damage", ["missing", "duplicate", "wrong-gate-liveness", "early-exit"] +) +def test_handshake_cannot_replace_losing_work_overlap(damage: str) -> None: + value = build_evidence("return-inflight", 2) + gate = value["regression"]["gateId"] + _add_handshake_gate(value) + if damage == "missing": + value["events"] = [ + event + for event in value["events"] + if not (event["kind"] == "blocked" and event.get("gateId") == gate) + ] + expected = "exactly one blocked" + elif damage == "duplicate": + value["events"].append(deepcopy(find(value, "blocked"))) + expected = "exactly one blocked" + elif damage == "wrong-gate-liveness": + pivot = find(value, "blocked", "peer")["atMs"] + for event in value["events"]: + if event.get("gateId") == gate and event["kind"] in {"alive", "effect"}: + event["atMs"] = pivot - 1 + expected = "native overlap lacks positive continued liveness" + else: + find(value, "io_exit")["atMs"] = find(value, "blocked", "peer")["atMs"] - 1 + expected = "native capacity stopped before overlap" + _resequence(value) + assert any(expected in error for error in validate_lifecycle_regressions(value)) + + +@pytest.mark.parametrize( + "scenario,role", + [ + ("return-inflight", "peer"), + ("native-concurrency", "victim"), + ("nested-progress", "peer"), + ("deadline-recovery", "recovery"), + ], +) +def test_undeclared_cohort_gate_ambiguity_stays_incomplete( + scenario: str, role: str +) -> None: + value = build_evidence(scenario, 2) + _add_handshake_gate(value, role) + assert any( + "exactly one blocked" in error + for error in validate_lifecycle_regressions(value) + ) + + +def _measured_timeout() -> dict[str, Any]: + value = _timeout_wrapper() + find(value, "wrapper_raise")["atMs"] = 900 + find(value, "user_exit")["atMs"] = 895 + find(value, "task_exit")["atMs"] = 898 + owner = { + key: value["events"][0][key] + for key in ("requestId", "ownerRequestId", "executionArn") + } + for offset, kind in enumerate( + ("checkpoint_call", "checkpoint_ack", "checkpoint_exit") + ): + value["events"].append( + { + **owner, + "kind": kind, + "atMs": 690 if offset == 0 else 800 + offset, + "callId": "cleanup-checkpoint", + } + ) + fault = value["faultsCreated"][0] + fault.update( + releaseRequestedAtMs=750, releaseAcknowledgedAtMs=751, releasedAtMs=752 + ) + value["retirement"]["faults"][0]["releasedAtMs"] = 752 + uri = "s3://evidence/run/case/measurements/deadline.json" + value["collection"]["evidenceUris"].append(uri) + value["regression"]["deadlineMeasurement"] = { + "selectedRequests": deepcopy(value["selectedRequests"]), + "deadlineAtMs": 600, + "cleanupCutoffAtMs": 700, + "observationCutoffAtMs": 705, + "observedAtMs": 720, + "sourceEvidenceUri": uri, + } + _resequence(value) + return value + + +def test_normal_postmeasurement_cleanup_is_not_deadline_measurement_activity() -> None: + value = _measured_timeout() + assert validate_lifecycle_regressions(value) == [] + assert any( + event["kind"] == "checkpoint_exit" and event["atMs"] == 802 + for event in value["events"] + ) + + +def test_frozen_measurement_keeps_real_prerelease_late_effect_failures() -> None: + value = _measured_timeout() + effect = deepcopy(find(value, "effect")) + effect["atMs"] = 710 + value["events"].append(effect) + _resequence(value) + assert any( + "continued side effects" in error + for error in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize( + "damage", + [ + "owner", + "deadline", + "cleanup", + "grace", + "short-window", + "after-release", + "unknown-source", + "missing-release", + "malformed", + ], +) +def test_deadline_measurement_cannot_shorten_or_substitute_the_original_window( + damage: str, +) -> None: + value = _measured_timeout() + measurement = value["regression"]["deadlineMeasurement"] + if damage == "owner": + measurement["selectedRequests"]["victim"] = value["selectedRequests"]["retry"] + elif damage == "deadline": + measurement["deadlineAtMs"] = 650 + elif damage == "cleanup": + measurement["cleanupCutoffAtMs"] = 750 + elif damage == "grace": + measurement["observationCutoffAtMs"] = 700 + elif damage == "short-window": + measurement["observedAtMs"] = 704 + elif damage == "after-release": + measurement["observedAtMs"] = 760 + elif damage == "unknown-source": + measurement["sourceEvidenceUri"] = "s3://uncollected/measurement" + elif damage == "missing-release": + value["faultsCreated"][0].pop("releaseRequestedAtMs") + else: + value["regression"]["deadlineMeasurement"] = [] + assert any( + "deadline measurement" in error + for error in validate_lifecycle_regressions(value) + ) + + +def test_legacy_timeout_evidence_keeps_full_observation_activity_checks() -> None: + value = _measured_timeout() + value["regression"].pop("deadlineMeasurement") + assert any( + "continued side effects" in error + for error in validate_lifecycle_regressions(value) + ) + + +def test_deadline_measurement_does_not_hide_work_after_wrapper_exit() -> None: + value = _measured_timeout() + find(value, "checkpoint_exit")["atMs"] = 901 + _resequence(value) + assert any( + "after wrapper exit" in error or "unsettled" in error + for error in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize("when", [710, 730, 750]) +def test_effects_until_controller_release_remain_deadline_failures(when: int) -> None: + value = _measured_timeout() + effect = deepcopy(find(value, "effect")) + effect["atMs"] = when + value["events"].append(effect) + _resequence(value) + assert any( + "continued side effects" in error + for error in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize("field", ["receipt", "acknowledgement"]) +def test_deadline_measurement_rejects_release_evidence_before_request( + field: str, +) -> None: + value = _measured_timeout() + if field == "receipt": + value["faultsCreated"][0]["releasedAtMs"] = 710 + value["retirement"]["faults"][0]["releasedAtMs"] = 710 + else: + value["faultsCreated"][0]["releaseAcknowledgedAtMs"] = 710 + assert any( + "deadline measurement" in error + for error in validate_lifecycle_regressions(value) + ) + + +@pytest.mark.parametrize( + "kind", ["body", "effect", "checkpoint_call", "poll_call", "unpaired-response"] +) +def test_postrelease_cleanup_cannot_admit_new_user_work(kind: str) -> None: + value = _measured_timeout() + if kind in {"body", "effect"}: + body = deepcopy(find(value, kind)) + body["atMs"] = 810 + value["events"].append(body) + elif kind == "unpaired-response": + value["events"] = [ + event for event in value["events"] if event["kind"] != "checkpoint_call" + ] + else: + call = find(value, "checkpoint_call") + call["atMs"] = 800 + if kind == "poll_call": + call["kind"] = kind + value["events"].remove(find(value, "checkpoint_ack")) + find(value, "checkpoint_exit")["kind"] = "poll_exit" + _resequence(value) + assert any( + "continued side effects" in error + for error in validate_lifecycle_regressions(value) + ) diff --git a/lmi_tests/tests/unit/test_python_matrix.py b/lmi_tests/tests/unit/test_python_matrix.py new file mode 100644 index 000000000..4a561dd4f --- /dev/null +++ b/lmi_tests/tests/unit/test_python_matrix.py @@ -0,0 +1,248 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""The Python regression profile cannot pass with missing configuration slots.""" + +from dataclasses import replace +from pathlib import Path +from runpy import run_path + +import pytest + +from lmi_tests.python_catalog import requirements, required_cases +from aws_durable_execution_conformance_tests.lmi.matrix import ( + describe_case, + duplicate_coverage, + missing_cases, +) + +BASE_SCENARIOS = { + "native-concurrency", + "replay-callback-retry", + "warm-lifecycle", + "early-completion-parallel", + "early-completion-map", + "early-completion-nested", + "checkpoint-settlement", + "deadline-recovery", + "timeout-retry", + "pending-cleanup", + "nested-progress", + "return-inflight", + "failure-inflight", + "late-operation", +} +OTEL_SCENARIOS = { + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + "otel-warm-lifecycle", + "otel-concurrent", + "export-error-recovery", + "held-export", + "deadline-recovery", + "timeout-retry", + "pending-cleanup", + "late-operation", + "return-inflight", + "failure-inflight", +} + + +def test_python_lifecycle_has_fourteen_variants_on_both_native_configurations(): + cases = required_cases("lifecycle") + assert len(cases) == 28 + assert {case.scenario for case in cases} == BASE_SCENARIOS + assert { + (case.scenario, case.native_concurrency, case.otel_view) for case in cases + } == { + (scenario, concurrency, None) + for scenario in BASE_SCENARIOS + for concurrency in (1, 2) + } + assert not {"python-lmi-2", "python-lmi-3", "python-lmi-7", "python-lmi-8"} & { + case.requirement_id for case in cases + } + + +def test_python_otel_adds_twelve_variants_in_both_views_without_replacing_baseline(): + cases = required_cases("lifecycle-otel") + baseline = required_cases("lifecycle") + instrumented = [case for case in cases if case.otel_view is not None] + assert len(cases) == 76 + assert len(instrumented) == 48 + assert {case.key for case in baseline} <= {case.key for case in cases} + assert { + (case.scenario, case.native_concurrency, case.otel_view) + for case in instrumented + } == { + (scenario, concurrency, view) + for scenario in OTEL_SCENARIOS + for concurrency in (1, 2) + for view in ("invocation", "execution") + } + assert missing_cases(cases, [case.key for case in cases]) == [] + assert duplicate_coverage(case.key for case in cases) == set() + + +def test_declared_but_unexercised_c1_cannot_be_filled_with_more_c2_results(): + cases = required_cases("lifecycle") + c2 = [case.key for case in cases if case.native_concurrency == 2] + missing = missing_cases(cases, c2 + c2) + assert len(missing) == 14 + assert {case.native_concurrency for case in missing} == {1} + assert duplicate_coverage(c2 + c2) == set(c2) + + +def test_relabeling_native_concurrency_or_view_does_not_cover_the_original_slot(): + cases = required_cases("lifecycle-otel") + selected = next( + case + for case in cases + if case.scenario == "runtime-carrier-sampled" + and case.native_concurrency == 1 + and case.otel_view == "invocation" + ) + observed = [case.key for case in cases if case != selected] + for invalid in ( + replace(selected, native_concurrency=2), + replace(selected, native_concurrency=8), + replace(selected, otel_view="execution"), + replace(selected, otel_view=None), + replace(selected, scenario="runtime-carrier-unsampled"), + replace(selected, requirement_id="python-lmi-8"), + ): + assert missing_cases(cases, [*observed, invalid.key]) == [selected] + + +def test_otel_coverage_cannot_replace_uninstrumented_lifecycle_or_other_views(): + cases = required_cases("lifecycle-otel") + observed = [case.key for case in cases if case.otel_view == "invocation"] + missing = missing_cases(cases, observed) + assert len(missing) == 52 + assert {case.otel_view for case in missing} == {None, "execution"} + + +def test_peer_roles_follow_real_python_worker_configuration(): + cases = required_cases("lifecycle") + indexed = {(case.scenario, case.native_concurrency): case for case in cases} + for name in ("native-concurrency", "nested-progress"): + assert indexed[name, 1].roles == ("victim",) + assert indexed[name, 2].roles == ("victim", "peer") + assert indexed[name, 1].minimum_native_concurrency == 1 + assert indexed[name, 2].minimum_native_concurrency == 2 + assert indexed["deadline-recovery", 1].roles == ("victim", "recovery") + assert set(indexed["deadline-recovery", 2].roles) == { + "victim", + "peer", + "recovery", + "recovery-peer", + } + assert "peer" not in indexed["pending-cleanup", 1].roles + assert "peer" in indexed["pending-cleanup", 2].roles + assert indexed["replay-callback-retry", 2].minimum_native_concurrency == 1 + + +def test_blocked_faults_and_resource_baselines_require_exclusive_windows(): + cases = required_cases("lifecycle-otel") + shared = { + "replay-callback-retry", + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + } + assert all(case.exclusive == (case.scenario not in shared) for case in cases) + for case in cases: + assert case.native_concurrency is not None + assert case.minimum_native_concurrency <= case.native_concurrency + assert case.source_tests + assert all( + test.startswith("lmi_tests/tests/e2e/") for test in case.source_tests + ) + deadline = next(case for case in cases if case.scenario == "deadline-recovery") + assert deadline.budget_names == ("observationGraceMs", "cleanupMs", "recoveryMs") + assert describe_case(deadline) == "python-lmi-4/deadline-recovery/native-1/no-otel" + + +def test_expanded_catalog_exposes_selector_union_and_python_owned_ids(): + catalog = requirements("lifecycle-otel") + assert len(catalog) == 13 + assert set(catalog["python-lmi-7"]["requiredScenarios"]) == { + "runtime-carrier-sampled", + "runtime-carrier-unsampled", + "otel-warm-lifecycle", + "otel-concurrent", + } + for definition in catalog.values(): + assert definition["runtimes"] == ["python"] + assert definition["id"].startswith("python-lmi-") + + +def test_every_expanded_slot_declares_the_validator_behavior_budget_names(): + cases = required_cases("lifecycle-otel") + for case in cases: + assert "observationGraceMs" in case.budget_names + assert not { + "cleanupGraceMs", + "placementTimeoutMs", + "retryTimeoutMs", + "progressTimeoutMs", + } & set(case.budget_names) + if case.scenario == "warm-lifecycle": + assert set(case.budget_names) == { + "observationGraceMs", + "maxThreadGrowth", + "maxFdGrowth", + "maxRssGrowthBytes", + } + if case.scenario.startswith("early-completion-"): + assert set(case.budget_names) == {"observationGraceMs", "cleanupMs"} + if case.scenario == "timeout-retry": + assert set(case.budget_names) - {"exportMs"} == { + "observationGraceMs", + "cleanupMs", + "retryMs", + } + assert ("exportMs" in case.budget_names) == (case.otel_view is not None) + + +@pytest.mark.parametrize("native", [1, 2]) +def test_warm_lifecycle_positive_control_fits_the_complete_default_role_plan(native): + build = run_path(str(Path(__file__).with_name("test_python_lifecycle.py")))[ + "build_evidence" + ] + evidence = build("warm-lifecycle", native) + required = next( + case + for case in required_cases("lifecycle") + if case.scenario == "warm-lifecycle" and case.native_concurrency == native + ) + assert len(required.roles) == 12 + assert set(evidence["selectedRequests"]) == set(required.roles) + assert {attempt["role"] for attempt in evidence["attempts"]} == set(required.roles) + + +def test_otel_warm_defaults_plan_all_six_executions_and_two_native_resumes(): + expected = { + "victim", + "warm-1", + "warm-2", + "warm-2-resume", + "warm-3", + "warm-4", + "warm", + "warm-resume", + } + cells = [ + case + for case in required_cases("lifecycle-otel") + if case.scenario == "otel-warm-lifecycle" + ] + assert len(cells) == 4 + assert all(set(case.roles) == expected for case in cells) + + +@pytest.mark.parametrize("capability", ["unknown", "", None]) +def test_unknown_python_capability_is_rejected(capability): + with pytest.raises(ValueError): + requirements(capability) + with pytest.raises(ValueError): + required_cases(capability) diff --git a/lmi_tests/tests/unit/test_python_otel.py b/lmi_tests/tests/unit/test_python_otel.py new file mode 100644 index 000000000..33e234fb9 --- /dev/null +++ b/lmi_tests/tests/unit/test_python_otel.py @@ -0,0 +1,1157 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""Native-profile OTel cells and deliberate regressions in observed evidence.""" + +from __future__ import annotations + +from copy import deepcopy +from datetime import UTC, datetime +from typing import Any + +import pytest + +from lmi_tests.python_otel import SCENARIOS, validate_lifecycle_telemetry +from aws_durable_execution_conformance_tests_otel.model import Span, span_to_dict + + +def build_evidence( + scenario: str, native: int = 2, view: str = "invocation" +) -> dict[str, Any]: + """Synthetic observations, never a claim of deployment or cloud acceptance.""" + if scenario in SCENARIOS[6:]: + from lmi_tests.tests.unit.test_python_lifecycle import ( + build_evidence as lifecycle_evidence, + ) + + baseline = lifecycle_evidence(scenario, native) + return add_telemetry(baseline, view) + attempts: list[dict[str, Any]] = [] + sequence = [] + + def add( + role: str, + start: int, + end: int, + *, + fixture: str = "success", + process: str = "process-1", + arn: str | None = None, + outcome: str = "SUCCEEDED", + ordinal: int = 1, + ) -> dict[str, Any]: + execution = arn or f"arn:execution:{role}" + attempt = { + "requestId": f"request-{role}", + "executionArn": execution, + "role": role, + "attempt": ordinal, + "fixture": fixture, + "marker": execution, + "environmentId": "environment-1", + "processId": process, + "workerId": process, + "startedAtMs": start, + "endedAtMs": end, + "outcome": outcome, + } + attempts.append(attempt) + return attempt + + if scenario == "otel-warm-lifecycle": + for index, fixture in enumerate(["success", "failure", "replay"] * 2): + role = "victim" if index == 0 else "warm" if index == 5 else f"warm-{index}" + start = 100 + index * 200 + original = add( + role, + start, + start + (60 if fixture == "replay" else 100), + fixture=fixture, + outcome="FAILED" + if fixture == "failure" + else "PENDING" + if fixture == "replay" + else "SUCCEEDED", + ) + sequence.append( + {"executionArn": original["executionArn"], "fixture": fixture} + ) + if fixture == "replay": + add( + f"{role}-resume", + start + 80, + start + 140, + fixture=fixture, + arn=original["executionArn"], + ordinal=1, + ) + else: + add("victim", 100, 200) + if scenario == "otel-concurrent" and native == 2: + add("peer", 105, 230, process="process-2") + if scenario == "export-error-recovery": + add("warm", 300, 400) + selected = { + attempt["role"]: attempt["requestId"] + for attempt in attempts + if attempt["attempt"] == 1 + } + end = max(attempt["endedAtMs"] for attempt in attempts) + 100 + events: list[dict[str, Any]] = [] + + def emit(owner: dict[str, Any], kind: str, at: int, **details: Any) -> None: + events.append( + { + "kind": kind, + "atMs": at, + "requestId": owner["requestId"], + "ownerRequestId": owner["requestId"], + "executionArn": owner["executionArn"], + **details, + } + ) + + sampled = scenario != "runtime-carrier-unsampled" + carriers = [] + spans = [] + export_outcomes = [] + histories = [] + expected_histories = {} + executions = list(dict.fromkeys(attempt["executionArn"] for attempt in attempts)) + for arn in executions: + members = [attempt for attempt in attempts if attempt["executionArn"] == arn] + terminal = members[-1] + expectation: dict[str, Any] = {"status": terminal["outcome"]} + if terminal["outcome"] == "FAILED": + expectation["error"] = {"type": "ValueError", "message": f"expected:{arn}"} + else: + expectation["result"] = arn + operations = [] + if members[0]["fixture"] == "replay": + for name, op_type in ( + ("success", "STEP"), + ("failure", "STEP"), + ("retry", "STEP"), + ("pause", "WAIT"), + ("completion", "CALLBACK"), + ("nested", "CONTEXT"), + ): + operation: dict[str, Any] = { + "id": name, + "name": name, + "type": op_type, + "parentId": None, + "status": "FAILED" if name == "failure" else "SUCCEEDED", + } + operation["error" if name == "failure" else "result"] = ( + {"type": "ValueError", "message": f"expected:{arn}"} + if name == "failure" + else arn + ) + operations.append(operation) + expected_histories[arn] = expectation + histories.append( + { + "executionArn": arn, + "ownerCaseId": "case-1", + "operations": operations, + **expectation, + } + ) + faults = [] + for attempt in attempts: + request_id, start, finish = ( + attempt["requestId"], + attempt["startedAtMs"], + attempt["endedAtMs"], + ) + trace_id = f"{executions.index(attempt['executionArn']) + 1:032x}" + parent_id = f"{attempts.index(attempt) + 100:016x}" + carrier = f"Root=1-{trace_id[:8]}-{trace_id[8:]};Parent={parent_id};Sampled={int(sampled)}" + carriers.append( + { + "requestId": request_id, + "environmentId": attempt["environmentId"], + "processId": attempt["processId"], + "workerId": attempt["workerId"], + "source": "process-environment", + "header": carrier, + "callerHeader": carrier, + "expectedSampled": sampled, + } + ) + snapshot = { + "threads": [{"id": "main", "name": "main"}], + "fds": 3, + "rssBytes": 1000, + } + emit(attempt, "wrapper_enter", start, resources=deepcopy(snapshot)) + emit( + attempt, + "otel_factory", + start + 2, + pluginInstanceId=f"plugin-{request_id}", + pluginExecutionArn=attempt["executionArn"], + ) + emit( + attempt, + "otel_enter", + start + 1, + providerId=f"provider-{attempt['processId']}", + exporterId=f"exporter-{attempt['processId']}", + contextBefore={"traceId": None, "spanId": None}, + ) + emit(attempt, "user_exit", finish - 30) + if attempt["fixture"] == "replay": + if attempt is next( + item + for item in attempts + if item["executionArn"] == attempt["executionArn"] + ): + emit( + attempt, + "body", + start + 4, + operation="success", + operationId="success", + attempt=1, + value=attempt["marker"], + ) + emit( + attempt, + "body", + start + 5, + operation="failure", + operationId="failure", + attempt=1, + value=attempt["marker"], + ) + emit( + attempt, + "body", + start + 6, + operation="retry", + operationId="retry", + attempt=1, + value=attempt["marker"], + ) + else: + emit( + attempt, + "body", + start + 6, + operation="retry", + operationId="retry", + attempt=2, + value=attempt["marker"], + ) + emit( + attempt, + "stored_failure", + start + 7, + error={ + "type": "ValueError", + "message": f"expected:{attempt['marker']}", + }, + ) + emit( + attempt, + "otel_exit", + finish + 1, + contextAfter={"traceId": None, "spanId": None}, + exporterShutdowns=0, + ) + emit( + attempt, + "wrapper_raise" if attempt["outcome"] == "FAILED" else "wrapper_return", + finish, + outcome=attempt["outcome"], + resources=deepcopy(snapshot), + ) + failure = scenario == "export-error-recovery" and attempt["role"] == "victim" + status = ( + "unsampled" if not sampled else "export-error" if failure else "delivered" + ) + outcome: dict[str, Any] = {"requestId": request_id, "status": status} + if failure: + outcome.update( + error="observed exporter failure", fixture="bounded-export-error" + ) + export_outcomes.append(outcome) + if sampled: + members = [ + item + for item in attempts + if item["executionArn"] == attempt["executionArn"] + ] + names = ["Invocation"] + (["Workflow"] if attempt is members[-1] else []) + for index, name in enumerate(names): + span_id = f"{attempts.index(attempt) * 2 + index + 1:016x}" + span_start = ( + members[0]["startedAtMs"] + 2 if name == "Workflow" else start + 2 + ) + parent = ( + next( + item["header"].split("Parent=")[1].split(";")[0] + for item in carriers + if item["requestId"] == members[0]["requestId"] + ) + if name == "Workflow" + else parent_id + ) + span = Span( + trace_id=trace_id, + span_id=span_id, + name=name, + start_time=datetime.fromtimestamp(span_start / 1000, UTC), + end_time=datetime.fromtimestamp((finish - 25) / 1000, UTC), + parent_span_id=parent, + status="ERROR" if attempt["outcome"] == "FAILED" else "OK", + attributes={ + "durable.execution.arn": attempt["executionArn"], + "durable.invocation.first": attempt is members[0], + }, + ) + spans.append( + { + "requestId": request_id, + "sampled": True, + "span": span_to_dict(span), + } + ) + emit( + attempt, + "otel_export_started", + finish - 24, + traceId=trace_id, + spanId=span_id, + ) + emit( + attempt, + "otel_export", + finish - 9, + traceId=trace_id, + spanId=span_id, + outcome="failure" + if failure + else "held" + if scenario == "held-export" + else "success", + ) + if scenario == "held-export" and index == 0: + emit( + attempt, + "export_held", + finish - 20, + gateId="export-gate", + traceId=trace_id, + spanId=span_id, + ) + emit( + attempt, + "export_released", + finish - 10, + gateId="export-gate", + mode="normal", + ) + faults.append( + { + "id": "export-gate", + "ownerCaseId": "case-1", + "releasedAtMs": finish - 10, + } + ) + if scenario == "otel-concurrent": + emit(attempt, "blocked", 120, gateId=f"gate-{request_id}") + emit(attempt, "control_released", finish - 10, gateId=f"gate-{request_id}") + if attempt["role"] == "peer": + emit(attempt, "alive", 205) + for attempt in attempts: + owned = sorted( + (event for event in events if event["requestId"] == attempt["requestId"]), + key=lambda event: event["atMs"], + ) + for index, event in enumerate(owned, start=1): + event["sequence"] = index + resources = [ + {"id": attempt["requestId"], "kind": "invocation", "ownerCaseId": "case-1"} + for attempt in attempts + ] + resources.extend( + {"id": arn, "kind": "execution", "ownerCaseId": "case-1"} for arn in executions + ) + return { + "schemaVersion": 1, + "coverageProfile": "python-lifecycle", + "runtime": "python", + "runId": "run-1", + "caseId": "case-1", + "cohortId": "cohort-1", + "requirementId": "python-lmi-8" + if scenario in {"export-error-recovery", "held-export"} + else "python-lmi-7", + "scenario": scenario, + "otelView": view, + "artifact": { + "sdk": "sdk-commit", + "conformance": "conformance-commit", + "plugin": "plugin-commit", + }, + "profile": { + "functionArn": f"arn:function:c{native}", + "configurationDigest": f"config-{native}", + "nativeConcurrency": native, + "recoveryPolicy": "reuse-worker", + "timeoutMs": 100, + }, + "observationEndedAtMs": end, + "attempts": attempts, + "selectedRequests": selected, + "events": events, + "sdkRequests": [], + "histories": histories, + "expectedHistories": expected_histories, + "collection": { + "watermarks": {"lifecycle": end, "sdkRequests": end, "history": end}, + "evidenceUris": ["s3://example/run-1/case-1/raw"], + }, + "resourcesCreated": resources, + "faultsCreated": faults, + "regression": { + "warmSequence": sequence, + "budgets": {"observationGraceMs": 10, "exportMs": 30}, + }, + "retirement": { + "retiredAtMs": end + 10, + "resources": [ + { + **item, + "settledAtMs": end + 1, + "evidenceUri": "s3://example/run-1/case-1/retired", + "outcome": "QUIESCENT" + if item["kind"] == "invocation" + else "SUCCEEDED", + } + for item in resources + ], + "faults": [{**item, "mode": "normal"} for item in faults], + }, + "telemetry": { + "views": { + view: { + "collection": { + "status": "complete", + "scope": "cohort", + "requestIds": [attempt["requestId"] for attempt in attempts], + }, + "carriers": carriers, + "spans": spans, + "exportOutcomes": export_outcomes, + } + } + }, + } + + +@pytest.mark.parametrize("scenario", SCENARIOS) +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("native", [1, 2]) +def test_all_48_otel_cells_require_real_observations( + scenario: str, view: str, native: int +) -> None: + assert validate_lifecycle_telemetry(build_evidence(scenario, native, view)) == [] + + +@pytest.mark.parametrize( + "scenario", ["runtime-carrier-sampled", "runtime-carrier-unsampled"] +) +def test_sampled_and_unsampled_are_distinct_mandatory_cases(scenario: str) -> None: + value = build_evidence(scenario) + carrier = value["telemetry"]["views"]["invocation"]["carriers"][0] + carrier["expectedSampled"] = not carrier["expectedSampled"] + assert any( + "requires explicit Sampled" in error + for error in validate_lifecycle_telemetry(value) + ) + + +@pytest.mark.parametrize( + "header", + [ + None, + "", + "malformed", + "Root=1-11111111-111111111111111111111111;Parent=1111111111111111;Sampled=1;Sampled=0", + ], +) +def test_injected_bad_carriers_do_not_satisfy_runtime_cases(header: Any) -> None: + value = build_evidence("runtime-carrier-sampled") + value["telemetry"]["views"]["invocation"]["carriers"][0]["header"] = header + assert validate_lifecycle_telemetry(value) + + +def test_runtime_root_must_match_actual_caller_propagation() -> None: + value = build_evidence("runtime-carrier-sampled") + carrier = value["telemetry"]["views"]["invocation"]["carriers"][0] + carrier["callerHeader"] = carrier["callerHeader"].replace( + "000000000000000000000001", "000000000000000000000002" + ) + assert any("caller Root" in error for error in validate_lifecycle_telemetry(value)) + + +@pytest.mark.parametrize( + "mutation", ["missing-exit", "context-leak", "exporter-shutdown", "shared-instance"] +) +def test_actual_plugin_lifetime_is_not_inferred_from_final_trace(mutation: str) -> None: + value = build_evidence("export-error-recovery") + exits = [event for event in value["events"] if event["kind"] == "otel_exit"] + if mutation == "missing-exit": + value["events"].remove(exits[0]) + elif mutation == "context-leak": + exits[0]["contextAfter"] = {"traceId": "a" * 32, "spanId": "b" * 16} + elif mutation == "exporter-shutdown": + exits[0]["exporterShutdowns"] = 1 + else: + factories = [ + event for event in value["events"] if event["kind"] == "otel_factory" + ] + factories[1]["pluginInstanceId"] = factories[0]["pluginInstanceId"] + assert validate_lifecycle_telemetry(value) + + +def test_failure_fixture_cannot_use_all_successful_exports() -> None: + value = build_evidence("export-error-recovery") + value["telemetry"]["views"]["invocation"]["exportOutcomes"][0]["status"] = ( + "delivered" + ) + assert any( + "requires export outcome export-error" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_export_recovery_cannot_use_a_replacement_process() -> None: + value = build_evidence("export-error-recovery") + value["attempts"][1]["workerId"] = "replacement" + assert any( + "original worker/process" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_held_export_prevents_otel_and_wrapper_completion() -> None: + value = build_evidence("held-export") + next(event for event in value["events"] if event["kind"] == "export_released")[ + "atMs" + ] = 200.5 + value["faultsCreated"][0]["releasedAtMs"] = 200.5 + assert any( + "prevent SDK wrapper completion" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_emergency_release_cannot_establish_held_export_contract() -> None: + value = build_evidence("held-export") + next(event for event in value["events"] if event["kind"] == "export_released")[ + "mode" + ] = "emergency" + assert any( + "normal measurement" in error for error in validate_lifecycle_telemetry(value) + ) + + +def test_ending_victim_must_leave_live_peer_observed() -> None: + value = build_evidence("otel-concurrent", 2) + value["events"] = [event for event in value["events"] if event["kind"] != "alive"] + assert any( + "no observed progress" in error for error in validate_lifecycle_telemetry(value) + ) + + +def test_weak_supplied_assertions_cannot_skip_workflow_status() -> None: + value = build_evidence("runtime-carrier-sampled") + view = value["telemetry"]["views"]["invocation"] + view["assertions"] = {"arn:execution:victim": {"minimum_spans": 0}} + next(record for record in view["spans"] if record["span"]["name"] == "Workflow")[ + "span" + ]["status"] = "ERROR" + assert any("status" in error for error in validate_lifecycle_telemetry(value)) + + +def test_every_boundary_case_runs_plugin_validation() -> None: + value = build_evidence("pending-cleanup") + from lmi_tests.python_catalog import validate + + assert value["requirementId"] == "python-lmi-14" + assert validate(value) == [] + value["events"] = [ + event for event in value["events"] if event["kind"] != "otel_exit" + ] + assert validate(value) + + +def add_telemetry(baseline: dict[str, Any], view: str) -> dict[str, Any]: + """Attach real-shape plugin observations to a complete core positive control.""" + value = deepcopy(baseline) + value["otelView"] = view + value["artifact"]["plugin"] = "plugin-commit" + value["regression"]["budgets"]["exportMs"] = 30 + attempts = value["attempts"] + events = value["events"] + arns = list(dict.fromkeys(attempt["executionArn"] for attempt in attempts)) + carriers, spans, exports = [], [], [] + + def emit(owner: dict[str, Any], kind: str, when: float, **extra: Any) -> None: + events.append( + { + "requestId": owner["requestId"], + "ownerRequestId": owner["requestId"], + "executionArn": owner["executionArn"], + "kind": kind, + "atMs": when, + **extra, + } + ) + + for index, attempt in enumerate(attempts): + request_id, arn = attempt["requestId"], attempt["executionArn"] + starts = [ + event + for event in events + if event["requestId"] == request_id and event["kind"] == "wrapper_enter" + ] + endings = [ + event + for event in events + if event["requestId"] == request_id + and event["kind"] in {"wrapper_return", "wrapper_raise"} + ] + start = starts[0]["atMs"] + if not endings: + # Positive cooperative-timeout observation. The platform result and + # later actual wrapper cleanup are separate retained boundaries. + root_exit = next( + event + for event in events + if event["requestId"] == request_id + and event["kind"] == "task_exit" + and event.get("taskId") == "root" + ) + finish = root_exit["atMs"] + emit(attempt, "user_exit", finish - 3) + emit( + attempt, + "wrapper_raise", + finish, + outcome=attempt["outcome"], + resources=deepcopy(starts[0]["resources"]), + ) + else: + finish = endings[0]["atMs"] + trace_id, parent_id = f"{arns.index(arn) + 1:032x}", f"{index + 100:016x}" + header = f"Root=1-{trace_id[:8]}-{trace_id[8:]};Parent={parent_id};Sampled=1" + carriers.append( + { + "requestId": request_id, + "environmentId": attempt["environmentId"], + "workerId": attempt["workerId"], + "processId": attempt["processId"], + "source": "process-environment", + "header": header, + "expectedSampled": True, + } + ) + emit( + attempt, + "otel_factory", + start + 0.2, + pluginInstanceId=f"plugin-{request_id}", + pluginExecutionArn=arn, + ) + emit( + attempt, + "otel_enter", + start + 0.1, + providerId=f"provider-{attempt['processId']}", + exporterId=f"exporter-{attempt['processId']}", + contextBefore={"traceId": None, "spanId": None}, + ) + emit( + attempt, + "otel_exit", + finish + 0.1, + exporterShutdowns=0, + contextAfter={"traceId": None, "spanId": None}, + ) + members = [member for member in attempts if member["executionArn"] == arn] + for offset, name in enumerate( + ["Invocation"] + + ( + ["Workflow"] + if attempt is members[-1] + and attempt["outcome"] in {"SUCCEEDED", "FAILED"} + else [] + ) + ): + span_id = f"{2 * index + offset + 1:016x}" + original = members[0] + span_start = ( + original["startedAtMs"] + 0.2 if name == "Workflow" else start + 0.2 + ) + parent = ( + next( + carrier["header"].split("Parent=")[1].split(";")[0] + for carrier in carriers + if carrier["requestId"] == original["requestId"] + ) + if name == "Workflow" + else parent_id + ) + span = Span( + trace_id, + span_id, + name, + datetime.fromtimestamp(span_start / 1000, UTC), + datetime.fromtimestamp((finish - 0.9) / 1000, UTC), + parent_span_id=parent, + status="ERROR" + if attempt["outcome"] in {"FAILED", "TIMED_OUT"} + else "OK", + attributes={ + "durable.execution.arn": arn, + "durable.invocation.first": attempt is original, + }, + ) + spans.append( + {"requestId": request_id, "sampled": True, "span": span_to_dict(span)} + ) + emit( + attempt, + "otel_export_started", + finish - 0.8, + traceId=trace_id, + spanId=span_id, + ) + emit( + attempt, + "otel_export", + finish - 0.2, + traceId=trace_id, + spanId=span_id, + outcome="success", + ) + exports.append({"requestId": request_id, "status": "delivered"}) + for attempt in attempts: + local = sorted( + (event for event in events if event["requestId"] == attempt["requestId"]), + key=lambda event: event["atMs"], + ) + for index, event in enumerate(local, start=1): + event["sequence"] = index + value["telemetry"] = { + "views": { + view: { + "collection": { + "status": "complete", + "scope": "cohort", + "requestIds": [attempt["requestId"] for attempt in attempts], + }, + "carriers": carriers, + "spans": spans, + "exportOutcomes": exports, + } + } + } + return value + + +@pytest.mark.parametrize("scenario", SCENARIOS[6:]) +@pytest.mark.parametrize("view", ["invocation", "execution"]) +@pytest.mark.parametrize("native", [1, 2]) +def test_boundary_overlay_passes_both_actual_lifecycle_and_otel_validators( + scenario: str, view: str, native: int +) -> None: + from lmi_tests.python_lifecycle import validate_lifecycle_regressions + + value = build_evidence(scenario, native, view) + assert validate_lifecycle_regressions(value) == [] + assert validate_lifecycle_telemetry(value) == [] + + +@pytest.mark.parametrize("scenario", SCENARIOS[:6]) +def test_otel_only_cells_preserve_actual_wrapper_lifecycle_checks( + scenario: str, +) -> None: + from lmi_tests.python_lifecycle import validate_lifecycle_events + + assert validate_lifecycle_events(build_evidence(scenario)) == [] + + +def test_export_budget_cannot_be_chosen_by_the_export_observation() -> None: + value = build_evidence("held-export") + value["regression"]["budgets"]["exportMs"] = 5 + assert any( + "immutable exportMs" in error for error in validate_lifecycle_telemetry(value) + ) + + +def test_export_start_is_required_even_when_final_trace_is_complete() -> None: + value = build_evidence("runtime-carrier-sampled") + value["events"] = [ + event for event in value["events"] if event["kind"] != "otel_export_started" + ] + assert any( + "export start" in error or "exporter start" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_factory_runs_inside_sdk_call_after_observation_entry() -> None: + value = build_evidence("runtime-carrier-sampled") + next(event for event in value["events"] if event["kind"] == "otel_factory")[ + "atMs" + ] = 100 + assert any( + "actual SDK wrapper boundary" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_post_call_diagnostic_does_not_hide_export_after_sdk_return() -> None: + value = build_evidence("runtime-carrier-sampled") + event = next(event for event in value["events"] if event["kind"] == "otel_export") + event["atMs"] = 200.5 # Actual SDK return is 200; post-call OTEL_EXIT is 201. + assert any( + "escaped its OTel lifetime" in error + for error in validate_lifecycle_telemetry(value) + ) + + +@pytest.mark.parametrize("missing", ["failure", "replay"]) +def test_warm_coverage_cannot_replace_failure_or_replay_with_success( + missing: str, +) -> None: + value = build_evidence("otel-warm-lifecycle") + next( + item + for item in value["regression"]["warmSequence"] + if item["fixture"] == missing + )["fixture"] = "success" + assert any( + "success/failure/replay" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_workflow_is_not_exported_from_an_earlier_pending_invocation() -> None: + value = build_evidence("otel-warm-lifecycle") + spans = value["telemetry"]["views"]["invocation"]["spans"] + workflow = next( + record + for record in spans + if record["span"]["name"] == "Workflow" + and record["requestId"] == "request-warm-2-resume" + ) + workflow["requestId"] = "request-warm-2" + assert any( + "terminal invocation" in error for error in validate_lifecycle_telemetry(value) + ) + + +def test_same_logical_span_reexport_does_not_claim_exactly_once_transport() -> None: + value = build_evidence("runtime-carrier-sampled") + spans = value["telemetry"]["views"]["invocation"]["spans"] + spans.append(deepcopy(spans[0])) + assert validate_lifecycle_telemetry(value) == [] + + +@pytest.mark.parametrize("view", ["invocation", "execution"]) +def test_replayed_workflow_uses_its_original_service_execution_start(view: str) -> None: + value = build_evidence("otel-warm-lifecycle", view=view) + workflow = next( + record + for record in value["telemetry"]["views"][view]["spans"] + if record["span"]["name"] == "Workflow" + and record["requestId"] == "request-warm-2-resume" + ) + original = next( + attempt + for attempt in value["attempts"] + if attempt["requestId"] == "request-warm-2" + ) + service_start = original["startedAtMs"] - 25 + next( + history + for history in value["histories"] + if history["executionArn"] == original["executionArn"] + )["startedAtMs"] = service_start + workflow["span"]["start_time"] = datetime.fromtimestamp( + service_start / 1000, UTC + ).isoformat() + assert validate_lifecycle_telemetry(value) == [] + workflow["span"]["start_time"] = datetime.fromtimestamp( + (service_start - 1) / 1000, UTC + ).isoformat() + assert any( + "outside its lifecycle observation window" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def resumed_operation_evidence(*, duplicate: bool = False) -> dict[str, Any]: + """Minimal, sanitized versions of run 43331709913's execution-view spans. + + The wait finishes before resume but is exported on resume. The timeout case + re-exports one logical step identity from two native requests, changing its + start and invocation link. Neither case re-runs the completed step body. + """ + value = build_evidence( + "timeout-retry" if duplicate else "pending-cleanup", 2, "execution" + ) + owner = value["selectedRequests"]["retry" if duplicate else "resume"] + original = value["selectedRequests"]["victim"] + attempt = next(item for item in value["attempts"] if item["requestId"] == owner) + history = next( + item + for item in value["histories"] + if item["executionArn"] == attempt["executionArn"] + ) + operation = next( + item + for item in history["operations"] + if item["name"] == ("success" if duplicate else "pause") + ) + operation.update( + startedAtMs=110 if duplicate else 250, endedAtMs=130 if duplicate else 550 + ) + view = value["telemetry"]["views"]["execution"] + workflow = next( + item + for item in view["spans"] + if item["requestId"] == owner and item["span"]["name"] == "Workflow" + ) + + def append(request_id: str, start: float, exported: float) -> None: + record = deepcopy(workflow) + record["requestId"] = request_id + invocation = next( + item["span"] + for item in view["spans"] + if item["requestId"] == request_id and item["span"]["name"] == "Invocation" + ) + record["span"].update( + name=operation["name"], + span_id="f" * 16, + parent_span_id=workflow["span"]["span_id"], + start_time=datetime.fromtimestamp(start / 1000, UTC).isoformat(), + end_time=datetime.fromtimestamp( + operation["endedAtMs"] / 1000, UTC + ).isoformat(), + attributes={ + "durable.execution.arn": attempt["executionArn"], + "durable.operation.id": operation["id"], + "durable.operation.name": operation["name"], + "durable.operation.type": operation["type"], + "durable.operation.status": "SUCCEEDED", + }, + links=[ + {"trace_id": invocation["trace_id"], "span_id": invocation["span_id"]} + ], + ) + if duplicate: + record["span"]["attributes"]["durable.attempt.number"] = ( + 1 # A logical STEP may carry this attribute. + ) + view["spans"].append(record) + for kind, when in ( + ("otel_export_started", exported), + ("otel_export", exported + 1), + ): + value["events"].append( + { + "requestId": request_id, + "ownerRequestId": request_id, + "executionArn": attempt["executionArn"], + "kind": kind, + "atMs": when, + "traceId": invocation["trace_id"], + "spanId": "f" * 16, + "outcome": "success", + } + ) + + if duplicate: + append(original, 105, 350) + append(owner, operation["startedAtMs"], 800) + for request in value["attempts"]: + events = sorted( + ( + item + for item in value["events"] + if item["requestId"] == request["requestId"] + ), + key=lambda item: item["atMs"], + ) + for sequence, event in enumerate(events, 1): + event["sequence"] = sequence + return value + + +def test_resumed_logical_wait_uses_its_actual_service_operation_interval() -> None: + assert validate_lifecycle_telemetry(resumed_operation_evidence()) == [] + + +@pytest.mark.parametrize( + "change", + [ + "missing-start", + "missing-end", + "wrong-id", + "wrong-type", + "wrong-name", + "wrong-status", + "open", + "reversed", + "nonfinite", + "duplicate", + ], +) +def test_early_logical_span_requires_matching_complete_operation_history( + change: str, +) -> None: + value = resumed_operation_evidence() + history = next( + item + for item in value["histories"] + if any(op["name"] == "pause" for op in item["operations"]) + ) + operation = next(item for item in history["operations"] if item["name"] == "pause") + if change == "missing-start": + del operation["startedAtMs"] + elif change == "missing-end": + del operation["endedAtMs"] + elif change == "wrong-id": + operation["id"] = "different-operation" + elif change == "wrong-type": + operation["type"] = "STEP" + elif change == "wrong-name": + operation["name"] = "other-wait" + elif change == "wrong-status": + operation["status"] = "FAILED" + elif change == "open": + operation["status"] = "STARTED" + value["telemetry"]["views"]["execution"]["spans"][-1]["span"]["attributes"][ + "durable.operation.status" + ] = "STARTED" + elif change == "reversed": + operation["endedAtMs"] = 200 + elif change == "nonfinite": + operation["startedAtMs"] = float("nan") + else: + history["operations"].append(deepcopy(operation)) + assert validate_lifecycle_telemetry(value) + + +@pytest.mark.parametrize( + "change", + [ + "invocation-view", + "attempt", + "user-span", + "too-early", + "too-late", + "wrong-execution", + ], +) +def test_operation_history_does_not_relax_unrelated_span_windows(change: str) -> None: + value = resumed_operation_evidence() + view = value["telemetry"]["views"]["execution"] + span = view["spans"][-1]["span"] + if change == "invocation-view": + value["otelView"] = "invocation" + value["telemetry"]["views"] = {"invocation": view} + elif change == "attempt": + span["attributes"]["durable.attempt.outcome"] = "SUCCEEDED" + elif change == "user-span": + del span["attributes"]["durable.operation.id"] + elif change == "too-early": + span["start_time"] = datetime.fromtimestamp(0.249, UTC).isoformat() + elif change == "too-late": + span["end_time"] = datetime.fromtimestamp(0.551, UTC).isoformat() + else: + history = next( + item + for item in value["histories"] + if any(op["name"] == "pause" for op in item["operations"]) + ) + history["executionArn"] = "arn:execution:unrelated" + assert any( + "observation window" in error for error in validate_lifecycle_telemetry(value) + ) + + +def test_inconsistent_cross_request_span_does_not_corrupt_either_export_ledger() -> ( + None +): + value = resumed_operation_evidence(duplicate=True) + before = deepcopy(value) + errors = validate_lifecycle_telemetry(value) + assert any("duplicate span identity" in error for error in errors) + assert not any("omit or add canonical spans" in error for error in errors) + assert not any("observation window" in error for error in errors) + assert value == before + + +def test_per_request_ledger_still_rejects_an_actually_missing_canonical_span() -> None: + value = resumed_operation_evidence(duplicate=True) + view = value["telemetry"]["views"]["execution"] + original = value["selectedRequests"]["victim"] + view["spans"] = [ + item + for item in view["spans"] + if not (item["requestId"] == original and item["span"]["span_id"] == "f" * 16) + ] + assert any( + "omit or add canonical spans" in error + for error in validate_lifecycle_telemetry(value) + ) + + +def test_identical_same_request_reexport_may_have_another_transport_receipt() -> None: + value = build_evidence("runtime-carrier-sampled", 1, "execution") + spans = value["telemetry"]["views"]["execution"]["spans"] + repeated = {**deepcopy(spans[0]), "exportId": "another-observed-export-receipt"} + spans.append(repeated) + assert validate_lifecycle_telemetry(value) == [] + + +def test_historical_span_proof_does_not_hide_sdk_exporter_shutdown_failure() -> None: + value = resumed_operation_evidence() + next(event for event in value["events"] if event["kind"] == "otel_exit")[ + "exporterShutdowns" + ] = 1 + assert any("shut down" in error for error in validate_lifecycle_telemetry(value)) + + +@pytest.mark.parametrize("name_missing", [False, True]) +def test_historical_unnamed_operation_uses_its_actual_id_without_inventing_a_name( + name_missing: bool, +) -> None: + value = resumed_operation_evidence() + history = next( + item + for item in value["histories"] + if any(op["name"] == "pause" for op in item["operations"]) + ) + operation = next(item for item in history["operations"] if item["name"] == "pause") + operation.update(name=None, type="CONTEXT") + span = value["telemetry"]["views"]["execution"]["spans"][-1]["span"] + span["name"] = operation["id"] + span["attributes"].pop("durable.operation.name") + span["attributes"]["durable.operation.type"] = "CONTEXT" + if name_missing: + del operation["name"] + assert validate_lifecycle_telemetry(value) + else: + assert validate_lifecycle_telemetry(value) == [] diff --git a/lmi_tests/tests/unit/test_python_profile.py b/lmi_tests/tests/unit/test_python_profile.py new file mode 100644 index 000000000..5d376e586 --- /dev/null +++ b/lmi_tests/tests/unit/test_python_profile.py @@ -0,0 +1,319 @@ +# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. +# +# SPDX-License-Identifier: Apache-2.0 +"""The Python profile must exercise every declared native/view/behavior slot.""" + +import json +from functools import lru_cache +from pathlib import Path +from runpy import run_path +from copy import deepcopy +from unittest.mock import patch + +import pytest + +from aws_durable_execution_conformance_tests.lmi.app import ( + _identity_errors as shared_identity_errors, + build_report as shared_build_report, + coverage_inventory as shared_coverage_inventory, +) +from lmi_tests.python_catalog import ( + load_manifest, + required_cases, + suite_definition, + validate, +) +from aws_durable_execution_conformance_tests.report import ReportStatus + + +def build_report(plan, rows): + return shared_build_report( + plan, rows, suite=suite_definition(plan["capabilityProfile"]) + ) + + +def coverage_inventory(plan): + return shared_coverage_inventory( + plan, suite=suite_definition(plan["capabilityProfile"]) + ) + + +def _identity_errors(plan, case, evidence): + return shared_identity_errors( + plan, case, evidence, suite=suite_definition(plan["capabilityProfile"]) + ) + + +def manifest(capability="lifecycle-otel"): + plan = { + "schemaVersion": 1, + "runId": "run-1", + "runtime": "python", + "profiles": [ + { + "profile_id": f"native-{n}", + "resource_id": f"arn:function:{n}", + "capacity": n, + "native_concurrency": n, + "artifact_id": "sdk-commit", + "configuration_id": f"config-{n}", + } + for n in (1, 2) + ], + } + plan.update(coverageProfile="python-lifecycle", capabilityProfile=capability) + for profile in plan["profiles"]: + profile["recovery_policy"] = "reuse-worker" + plan["scenarios"] = [ + { + "case_id": f"case-{index}", + "requirement_id": case.requirement_id, + "scenario": case.scenario, + "profile_id": f"native-{case.native_concurrency}", + "behavior_budgets": { + name: (0 if name.startswith("max") else 1000) + for name in case.budget_names + }, + **({"otel_view": case.otel_view} if case.otel_view else {}), + } + for index, case in enumerate(required_cases(capability)) + ] + return plan + + +def results(plan): + return [ + { + "case_id": case["case_id"], + "requirement_id": case["requirement_id"], + "profile_id": case["profile_id"], + "status": "PASSED", + "errors": [], + } + for case in plan["scenarios"] + ] + + +@pytest.mark.parametrize( + "capability,count", [("lifecycle", 28), ("lifecycle-otel", 76)] +) +def test_full_python_matrix_is_explicit_and_reported(tmp_path, capability, count): + plan = manifest(capability) + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + assert load_manifest(path) == plan + assert len(coverage_inventory(plan)["required"]) == count + assert coverage_inventory(plan)["missing"] == [] + report = build_report(plan, results(plan)) + assert report.exit_code() == 0 + assert report.run.suites == ["python-lmi"] + assert all(entry.id.startswith("python-lmi-") for entry in report.entries) + + +@pytest.mark.parametrize("slot", range(76)) +def test_removing_any_required_matrix_slot_blocks_acceptance(slot): + plan = manifest() + removed = plan["scenarios"].pop(slot) + report = build_report(plan, results(plan)) + assert report.exit_code() == 1 + entry = next( + entry for entry in report.entries if entry.id == removed["requirement_id"] + ) + assert entry.status == ReportStatus.UNCOVERED + assert any("Missing mandatory matrix slot" in error for error in entry.errors) + + +@pytest.mark.parametrize( + "predicate", + [ + lambda case: case["profile_id"] == "native-1", + lambda case: case.get("otel_view") == "execution", + lambda case: case["scenario"] == "runtime-carrier-unsampled", + lambda case: case["scenario"] == "export-error-recovery", + lambda case: case["scenario"] == "held-export", + lambda case: not case.get("otel_view"), + ], +) +def test_declared_but_unexercised_profiles_or_paths_never_pass(predicate): + plan = manifest() + plan["scenarios"] = [case for case in plan["scenarios"] if not predicate(case)] + assert build_report(plan, results(plan)).exit_code() == 1 + + +def test_duplicate_c2_cases_cannot_fill_c1_coverage(): + plan = manifest() + plan["scenarios"] = [ + case for case in plan["scenarios"] if case["profile_id"] == "native-2" + ] + copies = deepcopy(plan["scenarios"]) + for case in copies: + case["case_id"] += "-duplicate" + plan["scenarios"].extend(copies) + assert len(plan["scenarios"]) == 76 + assert build_report(plan, results(plan)).exit_code() == 1 + + +def test_every_gap_and_failed_variant_stays_blocking(): + plan = manifest() + passed = results(plan) + for status in ( + "FAILED", + "NOT_IMPLEMENTED", + "INCOMPLETE_EVIDENCE", + "INFRASTRUCTURE_ERROR", + ): + current = deepcopy(passed) + current[0].update(status=status, errors=["original fixture did not pass"]) + assert build_report(plan, current).exit_code() == 1 + assert build_report(plan, passed[1:]).exit_code() == 1 + + +@pytest.mark.parametrize( + "mutation", + [ + lambda plan: plan.update(runtime="java"), + lambda plan: plan.update(coverageProfile="python-ish"), + lambda plan: plan["scenarios"][0].update(scenario="made-up-selector"), + lambda plan: plan["scenarios"][0].update(otel_view="wrong-view"), + lambda plan: plan["scenarios"][0].update(coverage_profile="standard"), + ], +) +def test_invalid_profile_is_rejected_before_admission(tmp_path, mutation): + plan = manifest() + mutation(plan) + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + with pytest.raises(ValueError): + load_manifest(path) + + +def test_evidence_identity_includes_behavior_and_view(): + plan = manifest() + case = next(case for case in plan["scenarios"] if case.get("otel_view")) + evidence = { + "coverageProfile": "standard", + "scenario": "wrong-behavior", + "otelView": "wrong-view", + } + errors = _identity_errors(plan, case, evidence) + assert any("coverageProfile" in error for error in errors) + assert any("scenario" in error for error in errors) + assert any("OTel view" in error for error in errors) + + +def test_otel_validates_lifecycle_requirements_too(): + evidence = { + "coverageProfile": "python-lifecycle", + "runtime": "python", + "requirementId": "python-lmi-11", + "scenario": "timeout-retry", + "otelView": "execution", + "profile": {"recoveryPolicy": "reuse-worker"}, + } + with ( + patch( + "aws_durable_execution_conformance_tests.lmi.evidence.validate_common_evidence", + return_value=[], + ), + patch( + "lmi_tests.python_lifecycle.validate_lifecycle_regressions", + return_value=[], + ), + patch( + "lmi_tests.python_otel.validate_lifecycle_telemetry", + return_value=["ASSERTION: missing replay Workflow"], + ) as telemetry, + ): + assert validate(evidence) == ["ASSERTION: missing replay Workflow"] + telemetry.assert_called_once_with(evidence, phase="final") + + +def test_observed_budgets_cannot_be_inflated_after_the_run(): + plan = manifest() + case = plan["scenarios"][0] + evidence = { + "coverageProfile": "python-lifecycle", + "scenario": case["scenario"], + "regression": {"budgets": {"observationGraceMs": 999999}}, + } + assert any("budgets" in error for error in _identity_errors(plan, case, evidence)) + + +@pytest.mark.parametrize( + "value", + [ + None, + {}, + {"observationGraceMs": True}, + {"observationGraceMs": float("inf")}, + {"observationGraceMs": 0}, + {"observationGraceMs": -1}, + ], +) +def test_behavior_budget_configuration_is_required_and_finite(tmp_path, value): + plan = manifest() + plan["scenarios"][0]["behavior_budgets"] = value + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + with pytest.raises(ValueError): + load_manifest(path) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda rows: rows[0].update(profile_id="wrong-profile"), + lambda rows: rows[0].update(errors=["unresolved assertion"]), + lambda rows: rows.append(deepcopy(rows[0])), + lambda rows: rows[0].update(case_id="unadmitted-case"), + ], +) +def test_strict_report_rejects_misbound_or_contradictory_results(mutation): + plan = manifest() + rows = results(plan) + mutation(rows) + assert build_report(plan, rows).exit_code() == 1 + + +@pytest.mark.parametrize("policy", [None, "unknown", True]) +def test_recovery_policy_must_be_declared_before_execution(tmp_path, policy): + plan = manifest() + plan["profiles"][0]["recovery_policy"] = policy + path = tmp_path / "plan.json" + path.write_text(json.dumps(plan)) + with pytest.raises(ValueError): + load_manifest(path) + + +@lru_cache(maxsize=2) +def _fixture(telemetry): + filename = "test_python_otel.py" if telemetry else "test_python_lifecycle.py" + return run_path(str(Path(__file__).with_name(filename)))["build_evidence"] + + +@pytest.mark.parametrize("cell", required_cases()) +def test_each_python_matrix_cell_composes_shared_evidence_with_local_contracts(cell): + build = _fixture(cell.otel_view is not None) + evidence = ( + build(cell.scenario, native=cell.native_concurrency, view=cell.otel_view) + if cell.otel_view + else build(cell.scenario, native=cell.native_concurrency) + ) + assert evidence["requirementId"] == cell.requirement_id + assert set(cell.roles) == set(evidence["selectedRequests"]) + assert validate(evidence, phase="observations") == [] + assert validate(evidence, phase="final") == [] + + +@pytest.mark.parametrize("runtime", ["java", "javascript", None]) +def test_local_validator_does_not_apply_python_contracts_to_another_runtime(runtime): + evidence = _fixture(False)("native-concurrency") + evidence["runtime"] = runtime + assert any("only to the Python SDK" in error for error in validate(evidence)) + + +@pytest.mark.parametrize("policy", [None, "unknown", []]) +def test_local_validator_requires_an_explicit_supported_recovery_policy(policy): + evidence = _fixture(False)("native-concurrency") + evidence["profile"]["recoveryPolicy"] = policy + assert any("recoveryPolicy" in error for error in validate(evidence)) diff --git a/lmi_tests/tests/unit/test_shared_adapter.py b/lmi_tests/tests/unit/test_shared_adapter.py index 0d955e6da..0a2e54306 100644 --- a/lmi_tests/tests/unit/test_shared_adapter.py +++ b/lmi_tests/tests/unit/test_shared_adapter.py @@ -42,7 +42,7 @@ def context( "run", Scenario( case_id, - "lmi-lifecycle-5", + "lmi-lifecycle-16" if mode else "python-lmi-5", profile.profile_id, scenario, roles=roles, @@ -502,11 +502,11 @@ def test_missed_original_placement_is_not_retried_as_another_victim(): def test_warm_sequence_uses_real_callback_resumes_not_extra_driver_invocations( scenario, count ): - from aws_durable_execution_conformance_tests.lmi.matrix import required_cases + from lmi_tests.python_catalog import required_cases cell = next( case - for case in required_cases("python", "lifecycle-otel", "python-lifecycle") + for case in required_cases("lifecycle-otel") if case.scenario == scenario and case.native_concurrency == 1 ) cloud = ScriptCloud() diff --git a/lmi_tests/tests/unit/test_shared_deadline_measurement.py b/lmi_tests/tests/unit/test_shared_deadline_measurement.py index d5f768653..02a028593 100644 --- a/lmi_tests/tests/unit/test_shared_deadline_measurement.py +++ b/lmi_tests/tests/unit/test_shared_deadline_measurement.py @@ -110,7 +110,7 @@ def measured_driver( "run", Scenario( "case", - "lmi-lifecycle-11", + "python-lmi-11", profile.profile_id, scenario, roles=roles, diff --git a/lmi_tests/tests/unit/test_shared_evidence_fixes.py b/lmi_tests/tests/unit/test_shared_evidence_fixes.py index b66e74ac9..88f6571a7 100644 --- a/lmi_tests/tests/unit/test_shared_evidence_fixes.py +++ b/lmi_tests/tests/unit/test_shared_evidence_fixes.py @@ -184,7 +184,7 @@ def test_assembler_keeps_post_measurement_events_in_the_main_evidence(measuremen case_id="case", cohort_id="cohort-1", scenario="timeout-retry", - requirement_id="lmi-lifecycle-11", + requirement_id="python-lmi-11", coverage_profile="python-lifecycle", otel_view=None, export_mode=None, diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py index 35f33143f..28cb816d6 100644 --- a/lmi_tests/tests/unit/test_shared_manifest.py +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -9,6 +9,7 @@ from aws_durable_execution_conformance_tests.lmi.app import load_manifest from lmi_tests.shared import make_manifest, verify_checkout +from lmi_tests.python_catalog import suite_definition from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity @@ -53,11 +54,21 @@ def test_generated_manifest_covers_every_required_slot(tmp_path, profile, count) source = deployment() plan = make_manifest(source, profile, tmp_path) assert len(plan["scenarios"]) == count + identifiers = {case["requirement_id"] for case in plan["scenarios"]} + if profile == "python-lifecycle": + assert all(identity.startswith("python-lmi-") for identity in identifiers) + else: + assert identifiers == {"lmi-lifecycle-16"} assert {case["profile_id"] for case in plan["scenarios"]} == {"c1", "c2"} assert plan["sdk"]["deployment"]["conformanceRevision"] == CONFORMANCE_REVISION path = tmp_path / "plan.json" path.write_text(json.dumps(plan)) - assert load_manifest(path) == plan + assert ( + load_manifest( + path, suite=suite_definition() if profile == "python-lifecycle" else None + ) + == plan + ) plan["sdk"]["deployment"]["commit"] = "mutated" assert source["commit"] == "sdk-commit" diff --git a/lmi_tests/tests/unit/test_shared_partial_evidence.py b/lmi_tests/tests/unit/test_shared_partial_evidence.py index 3aa3ea5d0..a3267b74d 100644 --- a/lmi_tests/tests/unit/test_shared_partial_evidence.py +++ b/lmi_tests/tests/unit/test_shared_partial_evidence.py @@ -23,7 +23,7 @@ def context(): "run", Scenario( "case", - "lmi-lifecycle-11", + "python-lmi-11", "c1", "timeout-retry", roles=("victim", "retry"), @@ -113,7 +113,7 @@ def test_final_read_bound_alone_does_not_invent_quiescence(tmp_path): ctx = context() ctx.scenario = Scenario( "case", - "lmi-lifecycle-1", + "python-lmi-1", "c1", "native-concurrency", roles=("victim",), diff --git a/lmi_tests/tests/unit/test_shared_workflow.py b/lmi_tests/tests/unit/test_shared_workflow.py index 135301fe2..b7751a67b 100644 --- a/lmi_tests/tests/unit/test_shared_workflow.py +++ b/lmi_tests/tests/unit/test_shared_workflow.py @@ -100,3 +100,37 @@ def test_cloud_runs_automatically_for_trusted_prs_and_supports_manual_dispatch() assert cloud["needs"] == "harness" assert cloud["concurrency"]["group"] == "lmi-e2e-shared-capacity-provider" assert cloud["concurrency"]["cancel-in-progress"] is False + + +def test_shared_setup_bootstraps_before_conformance_is_installed(): + import subprocess + import sys + import textwrap + + script = textwrap.dedent(""" + import importlib.abc + import sys + from pathlib import Path + from unittest.mock import patch + + class NoConformance(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith('aws_durable_execution_conformance_tests'): + raise AssertionError('bootstrap tried to import uninstalled conformance') + return None + + sys.meta_path.insert(0, NoConformance()) + from lmi_tests import shared + with patch.object(shared, 'verify_checkout') as verify, patch.object(shared.subprocess, 'run') as install: + shared.setup(Path('.conformance')) + verify.assert_called_once_with(Path('.conformance')) + assert install.call_args.args[0][1:4] == ['-m', 'pip', 'install'] + """) + completed = subprocess.run( + [sys.executable, "-c", script], + cwd=ROOT, + capture_output=True, + text=True, + timeout=20, + ) + assert completed.returncode == 0, completed.stdout + completed.stderr From fda7daaeaa516baf534c4240430c449c1162247d Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:10:49 -0700 Subject: [PATCH 20/32] test: adopt shared LMI lifecycle suite --- .github/workflows/lmi-e2e-tests.yml | 8 +- lmi_tests/README.md | 513 ++----- lmi_tests/cloud.py | 54 +- lmi_tests/deploy.py | 16 +- lmi_tests/evidence.py | 246 ---- lmi_tests/fixture.py | 395 +++--- lmi_tests/python-lifecycle-schema.md | 155 --- lmi_tests/python-otel.md | 67 - lmi_tests/python_catalog.py | 612 --------- lmi_tests/python_lifecycle.py | 1224 ----------------- lmi_tests/python_otel.py | 740 ---------- lmi_tests/shared-suite-design.md | 114 +- lmi_tests/shared-telemetry.md | 80 -- lmi_tests/shared.py | 65 +- lmi_tests/shared_adapter.py | 4 +- lmi_tests/shared_contract.py | 2 +- lmi_tests/shared_evidence.py | 353 +++-- lmi_tests/shared_exporter.py | 489 ------- lmi_tests/shared_observations.py | 3 +- lmi_tests/shared_scenarios.py | 450 ++---- lmi_tests/shared_telemetry.py | 207 --- lmi_tests/summary.py | 61 +- lmi_tests/telemetry.py | 246 ---- lmi_tests/telemetry_evidence.py | 134 -- lmi_tests/tests/e2e/conftest.py | 64 - lmi_tests/tests/e2e/test_lifecycle.py | 301 ---- lmi_tests/tests/e2e/test_observability.py | 140 -- .../tests/e2e/test_shared_driver_local.py | 5 +- .../tests/e2e/test_shared_evidence_local.py | 94 +- .../tests/e2e/test_shared_exports_local.py | 239 ---- .../e2e/test_shared_observations_local.py | 101 +- .../tests/e2e/test_shared_telemetry_local.py | 112 -- lmi_tests/tests/regressions/conftest.py | 11 - lmi_tests/tests/regressions/test_issue741.py | 168 --- .../tests/regressions/test_late_operation.py | 9 - lmi_tests/tests/unit/conftest.py | 29 - lmi_tests/tests/unit/support.py | 80 ++ lmi_tests/tests/unit/test_cloud.py | 123 +- lmi_tests/tests/unit/test_control.py | 59 +- lmi_tests/tests/unit/test_evidence.py | 241 ---- lmi_tests/tests/unit/test_fixture.py | 16 +- lmi_tests/tests/unit/test_java_parity.py | 237 ---- .../tests/unit/test_lifecycle_mapping.py | 90 ++ lmi_tests/tests/unit/test_python_lifecycle.py | 1189 ---------------- lmi_tests/tests/unit/test_python_matrix.py | 248 ---- lmi_tests/tests/unit/test_python_otel.py | 1157 ---------------- lmi_tests/tests/unit/test_python_profile.py | 319 ----- lmi_tests/tests/unit/test_shared_adapter.py | 210 +-- lmi_tests/tests/unit/test_shared_cleanup.py | 2 +- .../unit/test_shared_deadline_measurement.py | 263 ---- .../tests/unit/test_shared_evidence_fixes.py | 245 ---- lmi_tests/tests/unit/test_shared_manifest.py | 40 +- .../tests/unit/test_shared_observations.py | 9 +- .../unit/test_shared_partial_evidence.py | 12 +- lmi_tests/tests/unit/test_shared_telemetry.py | 646 --------- lmi_tests/tests/unit/test_telemetry.py | 271 ---- pyproject.toml | 10 +- 57 files changed, 1019 insertions(+), 11959 deletions(-) delete mode 100644 lmi_tests/python-lifecycle-schema.md delete mode 100644 lmi_tests/python-otel.md delete mode 100644 lmi_tests/python_catalog.py delete mode 100644 lmi_tests/python_lifecycle.py delete mode 100644 lmi_tests/python_otel.py delete mode 100644 lmi_tests/shared-telemetry.md delete mode 100644 lmi_tests/shared_exporter.py delete mode 100644 lmi_tests/shared_telemetry.py delete mode 100644 lmi_tests/telemetry.py delete mode 100644 lmi_tests/telemetry_evidence.py delete mode 100644 lmi_tests/tests/e2e/conftest.py delete mode 100644 lmi_tests/tests/e2e/test_lifecycle.py delete mode 100644 lmi_tests/tests/e2e/test_observability.py delete mode 100644 lmi_tests/tests/e2e/test_shared_exports_local.py delete mode 100644 lmi_tests/tests/e2e/test_shared_telemetry_local.py delete mode 100644 lmi_tests/tests/regressions/conftest.py delete mode 100644 lmi_tests/tests/regressions/test_issue741.py delete mode 100644 lmi_tests/tests/regressions/test_late_operation.py delete mode 100644 lmi_tests/tests/unit/conftest.py create mode 100644 lmi_tests/tests/unit/support.py delete mode 100644 lmi_tests/tests/unit/test_java_parity.py create mode 100644 lmi_tests/tests/unit/test_lifecycle_mapping.py delete mode 100644 lmi_tests/tests/unit/test_python_lifecycle.py delete mode 100644 lmi_tests/tests/unit/test_python_matrix.py delete mode 100644 lmi_tests/tests/unit/test_python_otel.py delete mode 100644 lmi_tests/tests/unit/test_python_profile.py delete mode 100644 lmi_tests/tests/unit/test_shared_deadline_measurement.py delete mode 100644 lmi_tests/tests/unit/test_shared_evidence_fixes.py delete mode 100644 lmi_tests/tests/unit/test_shared_telemetry.py delete mode 100644 lmi_tests/tests/unit/test_telemetry.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 82ab581bf..f0fa10e93 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -19,7 +19,7 @@ env: # Ignore the CodeBuild image's shared pip target and Python search path. PIP_CONFIG_FILE: /dev/null PYTHONPATH: "" - CONFORMANCE_REVISION: 59352e1097d92fe034c7cd55a9618837b95b905b + CONFORMANCE_REVISION: cc1eae152d9d0a49ecb777db140339f290b43567 LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} jobs: @@ -63,8 +63,8 @@ jobs: cloud: name: cloud (Python 3.14, concurrency 1 and 2) needs: harness - # Trusted PRs and manual runs exercise the full suite. The #741 regressions - # remain strict failures; fork and Dependabot PRs run only the harness. + # Trusted PRs and manual runs exercise the shared lifecycle suite. SDK + # failures remain strict; fork and Dependabot PRs run only the harness. if: github.actor != 'dependabot[bot]' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) runs-on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }} timeout-minutes: 120 @@ -128,7 +128,7 @@ jobs: role-session-name: python-lmi-assertions aws-region: ${{ env.AWS_REGION }} allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Run Python lifecycle regressions and shared exporter conformance + - name: Run seven shared lifecycle requirements timeout-minutes: 60 run: hatch run lmi:shared - name: Refresh credentials for evidence and cleanup diff --git a/lmi_tests/README.md b/lmi_tests/README.md index a81343d8c..921e3c93f 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -1,405 +1,128 @@ -# Python LMI shared conformance adapter - -The cloud entry point runs two separately owned suites against the same persistent -c1/c2 functions. The Python SDK owns `python-lifecycle`: 28 lifecycle cells plus -48 instrumented cells, using `python-lmi-*` requirement IDs. Their definitions, -matrices and validators live in `python_catalog.py`, `python_lifecycle.py`, and -`python_otel.py`. They are Python regressions, not shared conformance requirements. - -The language-neutral `export-lifecycle` profile contributes another 24 cells from -[conformance PR #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186), -pinned to `59352e1097d92fe034c7cd55a9618837b95b905b`. Both suites reuse its generic -coordinator, launch journals, common evidence/retirement audit, and JSON/JUnit -reporting. Only the 24 exporter cells count toward that shared profile; these runs -do not claim acceptance of the separate shared `standard` profile. - -This suite implements [#742](https://github.com/aws/aws-durable-execution-sdk-python/issues/742) -and asserts the invocation-lifetime behavior requested by -[#741](https://github.com/aws/aws-durable-execution-sdk-python/issues/741). -It follows the separate cloud-suite approach in -[Java PR #728](https://github.com/aws/aws-durable-execution-sdk-java/pull/728). -It changes no production SDK code. The deadline and early-completion regressions -are **expected to fail on the current SDK**. They are ordinary failing assertions, -with no `xfail`, swallowed failure, or expected-failure success status. CI runs the -non-cloud harness automatically on PR updates and main pushes. Same-repository -PRs from non-Dependabot actors also run the complete cloud suite automatically, -and `workflow_dispatch` remains available. The #741 assertions remain strict -failures in those PR runs. -Local cloud runs and local red regressions also require explicit commands; the -ordinary local test suite remains usable while the fix is designed. - -## Run locally - -Use Python 3.14 and Hatch from the repository root: - -```sh -# Check out the exact shared runner revision first. -gh repo clone aws/aws-durable-execution-conformance-tests .conformance -git -C .conformance checkout 59352e1097d92fe034c7cd55a9618837b95b905b +# Python LMI lifecycle conformance + +This PR adapts the Python SDK to the seven shared lifecycle requirements in +[conformance #186](https://github.com/aws/aws-durable-execution-conformance-tests/pull/186). +The exact dependency is pinned in `shared_contract.py` and the workflow. Each +shared requirement declares one scalar `scenario`; the Python adapter reads the +shared matrix directly rather than maintaining a second requirement catalog. + +| Requirement | Scenario | Target | Invocation roles | +| --- | --- | --- | --- | +| `lmi-lifecycle-1` | `native-overlap` | c2 | victim, peer | +| `lmi-lifecycle-2` | `normal-return` | c1 | victim | +| `lmi-lifecycle-3` | `deadline-during-step` | c1 | victim | +| `lmi-lifecycle-4` | `peer-recovery` | c2 | victim, peer, recovery | +| `lmi-lifecycle-5` | `suspension-retry` | c1 | victim, resume | +| `lmi-lifecycle-6` | `warm-reuse` | c1 | victim, warm | +| `lmi-lifecycle-7` | `normal-failure` | c1 | victim | + +The shared runner owns assertions, strict reports, placement identity, leases, +launch journals and independent retirement auditing. This repository owns the +real public SDK handlers, AWS transport, source observations and deployment. +OTel/exporter tests and the previous Python-specific matrix are deferred. +There is no local conformance profile or validator that replaces a shared rule. + +## Local validation + +Install Hatch and Python 3.14. Clone the shared repository into `.conformance` +and check out the exact `CONFORMANCE_REVISION` from `shared_contract.py`: + +```bash +git clone https://github.com/aws/aws-durable-execution-conformance-tests.git .conformance +git -C .conformance checkout --detach cc1eae152d9d0a49ecb777db140339f290b43567 hatch run lmi:shared-setup -hatch run lmi:shared-tests hatch run lmi:lint -hatch run lmi:regressions --junitxml=lmi_tests/artifacts/regressions.xml +hatch run lmi:shared-tests +hatch run lmi:build ``` -The last command asserts five fixed behaviors against the real decorator/step/map/ -parallel APIs with an in-memory checkpoint service: an expired invocation starts no -step, and an early result does not pin a parallel, map, or nested-map wrapper past -its invocation budget. A further regression checks that an abandoned child cannot -start another step in its `finally` block. These local tests have final releases and bounded fixture -waits. They establish the SDK defects, not real LMI worker recovery. - -The green harness tests exercise negative evidence controls, process-shared marker -initialization, real public-API replay/callback/retry with the local runner, -checkpoint response/error forwarding, persistent create/update, ownership checks, -and run-scoped cleanup. -They also exercise the real parallel/map/nested fixtures with controlled I/O and -reject an early-completion pass when the losing branch failed before the winner. - -## Cloud prerequisites and configuration - -Use a **test account** and region supporting durable functions on LMI with Python -3.14 and arm64. Provision an existing test-owned capacity provider with an -explicit 2–128 vCPU maximum. Its `VpcConfig` supplies subnet/security-group -configuration; change networking on that provider before the run. Workers need -access to Lambda's checkpoint API, S3, and CloudWatch Logs through NAT or endpoints. -The suite saves the provider configuration, but never creates, updates, or deletes -the provider. Its owner remains responsible for its capacity costs and lifecycle. - -Set: - -| Variable | Purpose | -| --- | --- | -| `AWS_REGION` | Region matching the provider | -| `TEST_ACCOUNT_ID` | Required authenticated account check | -| `CAPACITY_PROVIDER_ARN` | Existing test provider with bounded capacity | -| `TEST_LAMBDA_EXECUTION_ROLE_ARN` | Lambda role with LMI/durable execution and logging permissions | - -The deployment identity needs CloudFormation stack operations, S3 bucket/policy/ -object operations, Lambda deployment/configuration/invoke/history/list/stop, -`iam:PassRole` for the supplied role, CloudWatch Logs read/create/delete, and -read access to the provider. The stack's private bucket policy grants that function -role access **only** to the suite's `runs/` event and control objects. The SDK and dependencies -come from a wheel built from the checked-out core package; no published SDK wheel -is substituted. The artifact hash, SDK wheel version, and commit are retained. - -```sh +`shared-setup` verifies the commit and rejects edited shared package sources. +It installs only the shared core runner. The LMI environment contains the checked +out core SDK and testing library; the Lambda bundle contains the core SDK and +its dependencies plus fixture/observation modules. It does not include the OTel +SDK, an OTel plugin, local catalogs, validators or the shared runner itself. + +Unit tests cover manifest completeness, case ownership, cancellation, source +normalization, driver ordering and cleanup. Local e2e tests use the actual public +SDK and its in-process test service, replacing only AWS transport and evidence +storage. They verify checkpoint/replay and preserve genuine admission failures. +These controls do not establish real LMI placement or deployed acceptance. + +## Cloud execution + +Use a dedicated test account with the configured LMI capacity provider and Lambda +execution role. The workflow validates the account before deployment. The +persistent functions use Python 3.14, arm64, 2048 MiB and native concurrency 1 and +2, with one environment each. SDK artifact and configuration digests are retained +in every manifest and checked before invoking either target. + +```bash +export AWS_REGION=us-west-2 +export LMI_SHARED_RUN_ID=your-unique-run-id +# Set CAPACITY_PROVIDER_ARN, TEST_ACCOUNT_ID and TEST_LAMBDA_EXECUTION_ROLE_ARN. hatch run lmi:build hatch run lmi:python -m lmi_tests.deploy deploy \ - --stack-name python-lmi-e2e --run-id local-20260923-a --runtime python3.14 + --stack-name python-lmi-e2e --run-id "$LMI_SHARED_RUN_ID" --runtime python3.14 hatch run lmi:shared -# Run both commands below even when a scenario fails: -hatch run lmi:python -m lmi_tests.deploy collect hatch run lmi:shared-cleanup -python -m lmi_tests.summary ``` -For unattended local runs, install a shell `EXIT`/`INT`/`TERM` trap that calls -`collect` followed by `shared-cleanup`; the workflow already has unconditional steps. -Use a fresh run ID for each run and reuse the same stack name. Before build or -deployment preflight, the driver removes prior run metadata and reports from -`lmi_tests/artifacts`, preserving `build.json` for artifact verification. Archive -any evidence you need before starting another run. The run manifest is invalidated -first, so a failed build, invalid configuration, or account/provider check cannot -make an exit trap collect or clean up the preceding run. This deployment supplies both native -environment-concurrency settings, `1` and `2`, on Python 3.14. The single cloud -command runs 76 Python-owned regression cells and 24 shared exporter cells. The shared coordinator reserves native -capacity for each case and serializes exclusive cases, without redeploying or -changing function configuration between cases. Independent SDK -branch-concurrency settings include 1, 2, and 3. - -To launch the same cloud workflow from an SDK branch: - -```sh -gh workflow run lmi-e2e-tests.yml --repo aws/aws-durable-execution-sdk-python \ - --ref YOUR_BRANCH -f region=us-west-2 -``` - -CI runs same-repository and manually dispatched jobs on the existing -`github-actions-runner` CodeBuild project. Fork and Dependabot PR harness checks -use GitHub-hosted runners. Python 3.14 comes from `setup-uv`; the runner image's -shared pip target and Python search path are disabled so Hatch environments stay -isolated. CI first runs the local harness, then assumes the repository's configured -test role and deploys the checked-out SDK and OTel wheels. It uploads the raw S3/service -records, suite manifests, attempt journals, coverage inventories, and JSON/JUnit -reports under `lmi_tests/artifacts/shared/`. Missing placement or source evidence -is a failure, not an accepted skip. Local integration checks exercise the real -public SDK and local checkpoint service; they do not establish LMI recovery. - -A non-expiring conditional S3 ownership record under `ownership//` fences -updates to the shared functions. Case controls live under separate run/case -prefixes. Normal completion releases ownership only when both profiles prove -quiescence. A failed case retains its original observations and errors. Available partial -observations are captured before fault release; a separate final snapshot can -prove that every actual launch and owned task/request/export has settled. Only -that independent audit permits later cases to reuse the target. Interrupted or -unproven retirement retains ownership for reconciliation; `shared-cleanup` -releases only that run's recorded controls and executions and cannot turn an -unknown retirement into success. Do not delete the record merely -to bypass a blocked deployment. - -The fixture's asynchronous processor waits up to five seconds through the real -provider flush API; shared acceptance allows ten seconds for that wait boundary. -Export completion is tracked separately and remains owned after wrapper return. -These are declared test budgets, not SDK SLAs. See -[shared telemetry observations](shared-telemetry.md) for capture/publication -ordering, [Python lifecycle rules](python-lifecycle-schema.md) and -[Python OTel rules](python-otel.md) for SDK-owned assertions, and -[the adapter design](shared-suite-design.md) for suite ownership details. - -The first run creates the persistent `python-lmi-e2e` stack, one private bucket, -and two functions/log groups (`python-lmi-e2e-c1`, `python-lmi-e2e-c2`). Later runs -update these same resources and preserve their names/ARNs. Select another dedicated -stack with `--stack-name`; its bucket name is stable for the account/region/stack. -All scenarios, including deadlines and healthy-peer probes, -reuse the function for their concurrency setting and the same handler artifact. -Both functions use a 60-second invocation timeout. Deadline tests wait for the -real platform timeout, leaving ordinary scenarios their existing invocation -budget. A single fixed function configuration cannot exercise both native -concurrency limits; merely sending one request to `c2` would not establish `c1`. - -Both functions declare native scaling limits of exactly one environment and use -2 GiB / 1 vCPU. LMI automatically publishes -`$LATEST.PUBLISHED`; every invocation uses that qualified target. Creating an extra -numbered version would allocate another set of environments. Before assertions, -the suite checks Active state, runtime/architecture, durability, timeout, provider, -process concurrency, applied scaling limits, commit/run identity, and code hash. -Unsupported configurations, insufficient permissions, and unavailable capacity fail -provisioning. There is no fallback to standard Lambda and no passing cloud skip. - -The handler mappings are saved under `c1` and `c2` in -`artifacts/function-name-map.json`; each entry has the existing -`PYTEST_FUNCTION_NAME_MAP` shape. The test driver reuses -`DurableFunctionCloudTestRunner` for async invocation, callback completion, and -waiting for results, with finite boto transport timeouts. - -## SDK fixture and regression contracts - -| Scenario | Evidence required | -| --- | --- | -| Replay/callback/retry | Real service histories; one body execution for checkpointed success and failure; stable failure type/message; retry attempts 1 and 2; nested child/map, parallel, wait, callback completion; PENDING and resumed requests | -| Environment concurrency | Held external barriers overlap in one shared `/tmp` environment marker, with distinct request IDs, execution ARNs, process UUIDs, and PIDs; c1 is the baseline, c2 must prove two processes | -| Warm cleanup | Repeated success, failure, and suspension/resume; explicit reuse of the same environment/process; no residual SDK threads after wrapper return; bounded current RSS and FD growth after warming | -| Early completion (#741) | `first_successful` parallel, `min_successful=1` map, and nested pools; loser blocked inside a step; separate winner selection, user result, user exit, wrapper exit; no late effects after grace | -| Checkpoint settlement | Real synchronous service checkpoint is acknowledged but response delivery to the SDK waiter is externally held; wrapper cannot return before settlement; releasing the response must unblock branch cleanup | -| Invocation deadline (#741) | A real short function timeout, service invocation-completed error for the exact request, external side effects, full original-environment worker capacity recovery, and healthy concurrent work on another process for c2 | -| Timeout/retry | A second service invocation of the **same** durable execution, completed step skipped, original attempt effects cease; repeated interrupted at-least-once work is recorded separately and is not called exactly-once | - -The following sections describe the retained SDK-local fixtures and legacy -pytest regression entry points. The automated run uses the Python-owned and shared exporter suites described -above. Java-specific executor, handler-owner and Context/MDC -assertions remain the Java SDK's responsibility. - -### Java lifecycle coverage in Python - -The analogous cases from Java PR #728 are mapped to Python's public APIs: - -| Java case | Python coverage | -| --- | --- | -| Root `finally` before PENDING, with a healthy peer | `test_pending_waits_for_root_finally_and_then_replays`: externally hold root cleanup, require cleanup exit before PENDING, verify actual wait/resume and no repeated step body; c2 also observes a healthy worker progressing during cleanup | -| Two roots on a fixed executor | Local subprocess test invokes the same decorated handler concurrently through the real local runner, with distinct inputs/checkpoints and a bounded rendezvous | -| Fixed-pool nested child/map/parallel progress | `test_nested_single_lane_pools_progress_for_all_runtime_workers`: saturate runtime lanes, release a shared barrier, and finish nested child/map/parallel work with each SDK branch pool limited to one worker; results and a 15-second progress budget are asserted | -| Return/failure with an in-flight step | `test_root_return_or_failure_settles_inflight_work`: `first_successful` leaves a controlled losing branch in flight, then the root returns or raises; the wrapper must settle that work before returning SUCCEEDED/FAILED, retaining the original failure; c2 keeps a healthy peer active | -| Non-cooperative residual child attempts later SDK work | `test_abandoned_child_rejects_late_durable_operation`: after parent completion, explicitly attempt another step in the losing branch's `finally`; rejection must precede any body execution or service checkpoint | -| Deadline, worker recovery, repeated warm batches | Existing deadline/isolation/retry and warm-resource tests remain enabled | - -There is no Python equivalent of Java's public shared-executor injection or -`stepAsync`. The Python cases use separate invocation scopes, public child/map/ -parallel APIs, and LMI's distinct Python worker processes. They do not inject a -Java-style executor into Python or manufacture an async step API. - -The root-finally gate is test-only lifecycle instrumentation after a durable wait -has requested suspension; it does not schedule another durable operation or change -the handler's result/operation sequence. Ordinary orchestration delays still use -`context.wait`. Local shared-root tests run under a subprocess watchdog, so an -executor-starvation regression cannot hang the test process. Cloud progress tests -have bounded driver budgets and post-assertion release/settlement of run-owned work. - -Healthy-peer cases checkpoint an admission decision before running fault work. -If an environment rotation places the victim elsewhere, it returns without -entering the scenario and the driver replaces the pair, up to three attempts. -Once admitted, lifecycle assertions are final and are never retried to get a pass. - -The late-operation guard currently exposes an additional #741 scope-lifetime gap: -a newly created operation can escape the SDK's set of already-known orphaned -operation IDs. Its local regression is explicitly enabled with `lmi:regressions`; -its real-cloud assertion always runs. The green harness validates fixture behavior -and negative evidence controls without treating the SDK defect as a passing result. - -The current SDK has no public cooperative cancellation API. The fault fixture -therefore deliberately remains blocked until interrupted/retired by the eventual -SDK/runtime policy or released externally. Its 150-second emergency limit always -invalidates a regression pass. It does not implement deadline cancellation in test -code, use `shutdown(wait=False)` as a fix, kill the process itself, or manufacture -successful checkpoints. Selecting a bounded retirement strategy belongs to #741. - -The initial **test acceptance budget** is five seconds after invocation deadline -or early winner selection. This is not a published SDK cancellation contract; -review/update it together with the #741 fix. A platform-supported replacement of -an affected Python worker in the same environment is allowed. Whole-environment -replacement is reported as a placement failure until a supported policy and -corresponding evidence are agreed; scaling elsewhere cannot hide a pinned worker. -The local regressions use a one-second invocation budget and two-second observation. - -All trace records include run, commit, marker, execution ARN, request, environment, -process initialization UUID, PID, sequence, timestamp, and phase. Body/effect records -include operation/attempt where relevant. A file lock safely initializes the shared -environment UUID across Python processes. Test-only wrapper/client diagnostics do -not control durable branches. All barrier and side-effect I/O in user code lives -inside steps; operation names stay static. Infrastructure diagnostics can repeat on -replay and are explicitly separate from the body/effect ledger. - -S3 records remain readable when a wrapper is stuck. CloudWatch logs and real history -provide independent runtime/service evidence. Recovery probes are queued at the -deadline, before waiting for eventually consistent history. A client timeout or a -logical execution's `TIMED_OUT` status alone cannot satisfy invocation-timeout -assertions. Missing service retry evidence is a collection/precondition failure, -never a passing retry test. No explicit second logical execution is mislabeled as -a service retry. - -Control objects are created with `hold` before invocation and updated to `release` -explicitly. Fixtures read their content with `GetObject`; missing objects, 403s, -and invalid states emit `CONTROL_ERROR` and fail as test infrastructure errors. -This avoids relying on `HeadObject` returning 404 for missing keys when the -function role has no `s3:ListBucket` permission. The winner is released only after -the losing step has validated its controls and emitted an actual side-effect -record. A loser that exits before `WINNER_READY` cannot satisfy the regression. - -Lifecycle objects are partitioned by case marker and request ID. Live polling -reads only the current case's prefixes with a bounded pool of readers; final -collection retrieves the complete run. Earlier cases therefore do not consume a -later case's short observation budget. Checkpoint holds and returns are correlated -to the same request, and stale-attempt effects are checked before waiting for a -retry that might itself be unable to start on a pinned worker. - -Per-case teardown collects only that case's executions. Complete-run histories -and CloudWatch logs are collected once by the final workflow step. Both runner -and artifact history reads are paced at one request per second, with at most five -throttling attempts within a 20-second retry budget (in addition to the finite -transport timeout). Invocation and callback writes are not retried by this layer. - -## Independent budgets and results - -| Budget | Default | -| --- | --- | -| Lambda invocation | 60 s for all scenarios, including deadlines | -| Durable logical execution | 240 s | -| Driver result polling | 120 s | -| Cleanup acceptance grace | 5 s | -| Fault-fixture emergency I/O release | 150 s | -| Per-case retirement after assertions | 30 s | -| Workflow cloud assertions (76 Python + 24 shared exporter cells) | 60 min | -| Waiting for prior executions before updating | 270 s | -| Single cloud job including deployment/run cleanup | 120 min | - -Artifacts contain JUnit, configuration/provider readbacks, qualified targets, -commit/wheel/code hash, invocation inputs/ARNs, all lifecycle and side-effect records, -execution histories/results, logs, setup/collection errors, and cleanup confirmation. -Tokens are omitted from client tracing and redacted from structured histories. -JUnit `lmi_outcome` properties and the Actions summary distinguish -`ProvisioningError`, `PlacementError`, `CollectionError`, and `RegressionAssertion`. -An unexecuted scenario is never summarized as passing. - -## CI and resource ownership - -The dedicated workflow runs the non-cloud harness when a PR is opened, -updated, or reopened (including Draft PRs), and on every push to `main`. There -are no path filters or ready-for-review requirements. After the harness passes, -same-repository PRs from non-Dependabot actors automatically run the cloud job. -Manual `workflow_dispatch` runs also execute both jobs; main pushes run only the -harness. There are no scheduled jobs. The #741 regressions stay ordinary failures, -so an affected SDK can produce a failing PR check. - -Each eligible PR or manual workflow run has one harness job and one cloud job -using the persistent resources. All cloud jobs share the repository-wide -`lmi-e2e-shared-capacity-provider` concurrency group. Only one cloud job can -deploy, test, or clean up at a time; the slot is held until the entire job finishes. -Harness jobs can run -in parallel. The capacity provider's configuration is not changed. - -`cancel-in-progress: false` preserves running jobs, and -[`queue: max`](https://docs.github.com/en/actions/how-tos/write-workflows/choose-when-workflows-run/control-workflow-concurrency#example-queueing-multiple-pending-runs) -retains up to 100 pending jobs instead of replacing an older pending job on each update. GitHub -cancels additional arrivals when that queue is full. This repository-wide group -does not coordinate manual local deployments or jobs in other repositories using -the same provider; those consumers must also leave sufficient capacity available. - -Fork and Dependabot PRs run only the harness on GitHub-hosted runners. Other PRs -and manual runs use CodeBuild. Cloud jobs reuse -`TEST_ROLE_ARN`, `TEST_ACCOUNT_ID`, and `TEST_LAMBDA_EXECUTION_ROLE_ARN` with OIDC. -The #741 regression assertions remain visibly failing until its fix lands; they -are not skipped or converted into expected-success results to keep CI green. - -After each case's assertions, teardown releases that case's controls, stops any -remaining logical executions to prevent retries, and waits for every observed -wrapper to return or raise. A terminal service status alone does not prove worker -retirement. If retirement fails, the next case must first retire that pending work -before invoking the shared function. Collection remains scoped to the case, and -the final collector gathers the entire run. These post-assertion releases and -stops cannot satisfy a deadline, cleanup-grace, or capacity-recovery assertion. - -Before updating, the driver checks persistent ownership tags and waits for earlier -durable executions on both functions to finish. Incomplete execution listings, -active executions past the wait budget, and failed/busy stacks block updates. -The driver never deletes and recreates a failed stack; CloudFormation's normal -deployment rollback behavior still applies. Every run updates `LMI_RUN_ID` and -`LMI_COMMIT` in the function configuration, refreshing runtime state while keeping -function names and ARNs stable, as in Java PR #728. Readback and trace identity must -match the tested run and commit. No code/configuration changes occur during tests. - -The workflow collects evidence before final run cleanup, even after test failure or -ordinary cancellation. `cleanup` releases `runs/RUN_ID/control/release-all`, discovers -only the current run's invocations from its ledger/artifacts, stops unfinished -executions, and waits for observed wrappers to exit. It never deletes functions, -the stack, bucket, or code artifacts. A terminal service status alone does not prove -that LMI worker code stopped. Other runs' controls and executions are not released -or stopped by this cleanup. - -Events and controls use `runs/RUN_ID/events/` and `runs/RUN_ID/control/`; only `runs/` -objects expire after one day. Content-addressed `code/` artifacts are retained for -updates and CloudFormation rollback. Logs and durable histories retain one day, -and GitHub artifacts retain seven days. There is no automatic infrastructure deletion -or janitor. The test-account owner manages final retirement of the persistent stack, -functions, bucket, and provider. A forced runner termination can interrupt run -cleanup; inspect artifacts and let prior executions settle before the next update. - -## OpenTelemetry acceptance - -The same two functions now also run the observability scope in #742. Both OTel -views use the real public factory registration and a process-owned provider. -The exporter writes the unmodified span identity, attributes, timing and status -to the existing private S3 evidence ledger. A separate invocation identity in -each record lets the assertions detect wrong attribution rather than deriving -the expected owner from the span itself. - -The cloud suite adds native X-Ray sampled/unsampled cases, sequential -success/failure/replay with proven process reuse, concurrent-worker attribution, -export failure/recovery, held export, and the original deadline, retry, PENDING, -late-operation and in-flight-return contracts with each OTel view enabled. -It still runs the original plugin-free lifecycle cases. The cloud assertion -budget is 60 minutes within a 120-minute job; provider capacity and the two -persistent functions are unchanged. - -An instrumented test caller supplies `X-Amzn-Trace-Id` on the real Invoke HTTP -request. The function records the actual runtime `_X_AMZN_TRACE_ID`; it never -substitutes a payload header into the runtime. Native acceptance requires the -observed runtime carrier and actual exported spans. Missing/invalid header -fallbacks are separately labeled local public-wrapper fixtures. OTel's explicit -`Sampled=0` decision can remain authoritative even when the trace ID is invalid. - -`hatch run lmi:unit` includes public-wrapper sampling, process-owned provider -reuse, real local suspension/replay, exporter failures, and negative evidence -controls for wrong execution/trace/parent attribution, missing/duplicate spans, -and leaked context. On non-Linux development hosts, unit-only resource snapshots -retain real thread identity while leaving RSS/FD measurement to the deployed -Linux fixture. - -`lmi:build` builds and installs both the checkout's core and OTel wheels with -the existing standalone OTel dependencies. `build.json` and the deployment -manifest retain both wheel hashes; `build/dependencies.json` records dependency -resolution. No published core/plugin substitutes for the checkout. - -The new tests do not close #742 without a fresh deployed acceptance run. -Plugin-enabled lifecycle cases still expose unresolved #741 behavior; local -green telemetry tests do not imply bounded non-cooperative cleanup or guaranteed -export after a runtime timeout. +`shared plan` writes the seven-case manifest without invoking functions. `shared +run` writes under `lmi_tests/artifacts/shared/standard/`: the manifest, coverage, +JSON/JUnit reports, launch journal and retained case evidence. Each invocation +uses a case-owned control/evidence prefix. A service resume is journaled under +the same durable execution; it is not another driver Invoke. + +Trusted same-repository pull requests and manual workflow runs exercise the cloud +suite. Fork and Dependabot pull requests run only the unprivileged local harness. +Runs using the same capacity provider are serialized through cleanup. A conditional +S3 run lease also prevents overlapping deployments or reuse of quarantined work. + +## Actual observations and failure handling + +`fixture.py` uses public durable APIs for the scenarios. Its instrumentation +observer captures actual operation IDs, parents, stored outcomes and user-task +lifetimes. The service-client wrapper forwards every SDK request and response; +it never supplies synthetic checkpoints or changes SDK cancellation flags. +The instrumentation observer is a measurement tool, not a plugin conformance test. + +For normal return/failure, the adapter records the actual durable wrapper boundary +and probes the original context after that boundary. The operation probe uses +`DurableContext.step`; the checkpoint probe calls the SDK's +`context.state.create_checkpoint(is_sync=False)` admission path without issuing a +raw Lambda request. Only an actual `OrphanedChildException` is recorded as +rejection. An accepted checkpoint, another error, or a still-pending operation +remains visible. Probe tasks are tracked separately; a timeout cannot make them +quiescent. Current SDK gaps are failures, not expected passes or exclusions. + +The deadline fixture holds real I/O across the runtime deadline and cooperatively +exits after it. A runtime clock alone never supplies a `platform_timeout` event: +that event requires the original request's service `InvocationCompleted` history. +The recovery driver keeps the original healthy peer held while an independent +recovery request establishes capacity in the original environment. Releasing a +victim fault cannot establish the preceding cleanup or recovery assertion. + +The warm fixture creates invocation-owned late work inside a step. A later +invocation must find it in the same process; it observes the old callback using +the original request identity. Failure to reuse that process is a placement or +collection failure, not permission to select a replacement victim. + +`shared_evidence.py` normalizes retained observations into the shared schema. +Service payload wrappers are decoded consistently with checkpoint snapshots; +truncated results are rejected. Replay compares a real resume's initial +checkpoints to the final service history and counts actual user-function entries. +Missing observations are never filled from expected answers. + +The shared behavior schema asks for one scenario-owned `task_exit`. Other user +and probe tasks remain in the same evidence as `observed_task_start/exit`, with +their original `sourceKind`. The independent retirement snapshot restores all +original task kinds, so a pending task cannot disappear from the audit. +Coordination barriers retain their complete `controlRecords` journal separately +from injected victim `faultsCreated`; releasing a healthy peer is not victim +fault recovery. + +Cleanup releases only the current run's controls and stops only its identified +executions. It does not erase assertion errors or clear ownership simply because +the runner process ended. A failed case can release its target only after all +actual launches and tasks have independently settled. Unknown invocations, +missing evidence and unresolved late work retain quarantine for reconciliation. diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 09a76d030..e1b2f735d 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -27,28 +27,14 @@ class RunnerClient: """Share paced history reads with the public runner; other APIs are unchanged.""" - def __init__(self, cloud, trace_header=None, *, before_call=None): + def __init__(self, cloud, *, before_call=None): self.cloud = cloud - self.trace_header = trace_header self.before_call = before_call def invoke(self, **kwargs): if self.before_call is not None: self.before_call() - if self.trace_header is None: - return self.cloud.lam.invoke(**kwargs) - # An instrumented caller's HTTP carrier goes through the real Lambda - # service. Never inject a synthetic header into the function payload. - isolated = client("lambda") - - def add_carrier(params, **_kwargs): - params["headers"]["X-Amzn-Trace-Id"] = self.trace_header - - isolated.meta.events.register("before-call.lambda.Invoke", add_carrier) - try: - return isolated.invoke(**kwargs) - finally: - isolated.close() + return self.cloud.lam.invoke(**kwargs) def __getattr__(self, name): operation = getattr(self.cloud.lam, name) @@ -208,9 +194,6 @@ def start( gate=None, target_environment=None, *, - otel_view=None, - trace_header=None, - otel_export="success", role=None, resume_roles=(), dispatcher_payload=None, @@ -229,6 +212,10 @@ def start( payload["sharedScenario"] = dispatcher_payload["scenario"] payload.update(scenario=scenario, marker=marker, run=self.manifest["run"]) if self.case_context is not None: + payload["coverageProfile"] = "standard" + payload["behaviorBudgets"] = dict( + self.case_context.scenario.behavior_budgets + ) payload["controlPrefix"] = self.control_prefix if "release-all" not in self.gates: self.hold("release-all") @@ -239,48 +226,22 @@ def start( for resume in resume_roles ): raise ValueError("Resume role is not part of the admitted scenario") - if otel_view is not None: - if otel_view not in {"invocation", "execution"}: - raise ValueError("Unknown OTel view") - if otel_export not in {"success", "failure", "held"}: - raise ValueError("Unknown exporter fault") - payload.update(otel_view=otel_view, otel_export=otel_export) - if otel_export == "held": - payload["otel_export_gate"] = marker + "-export" - self.hold(payload["otel_export_gate"]) if target_environment is not None: payload["target_environment"] = target_environment if gate: payload["gate"] = gate self.hold(gate) - if scenario in { - "parallel", - "map", - "nested", - "return-inflight", - "failure-inflight", - "late-operation", - }: - self.hold(marker + "-loser") - self.hold(marker + "-loser-started") - elif scenario == "checkpoint": - self.hold(marker + "-checkpoint") - elif scenario == "suspend-cleanup": - self.hold(marker + "-cleanup") runner = DurableFunctionCloudTestRunner( self.manifest["functions"][self.configuration], region=self.manifest["region"], ) - runner.lambda_client = RunnerClient( - self, trace_header, before_call=self.check_active - ) + runner.lambda_client = RunnerClient(self, before_call=self.check_active) item = { "marker": marker, "arn": None, "configuration": self.configuration, "runner": runner, "started": time.time(), - "trace_header": trace_header, "payload": payload, "role": role, "resume_roles": tuple(resume_roles), @@ -320,7 +281,6 @@ def start( "arn": arn, "function": runner.function_name, "started": item["started"], - "trace_header": trace_header, }, ) return item diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index cb82f7461..37648c796 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -84,12 +84,8 @@ def build(): if directory.exists(): shutil.rmtree(directory) directory.mkdir() - # Both SDK artifacts come from this checkout; pip resolves only their - # dependencies. A published core/plugin must not replace either artifact. - for package in ( - "aws-durable-execution-sdk-python", - "aws-durable-execution-sdk-python-otel", - ): + # The tested core SDK comes from this checkout; pip resolves its dependencies. + for package in ("aws-durable-execution-sdk-python",): subprocess.run( ["hatch", "build", "-t", "wheel", str(directory)], cwd=ROOT / "packages" / package, @@ -97,8 +93,8 @@ def build(): check=True, ) wheels = sorted(directory.glob("*.whl")) - if len(wheels) != 2: - raise ProvisioningError("Expected exactly the checked-out core and OTel wheels") + if len(wheels) != 1: + raise ProvisioningError("Expected exactly the checked-out core SDK wheel") wheel = next(directory.glob("aws_durable_execution_sdk_python-*.whl")) target = directory / "package" subprocess.run( @@ -124,8 +120,6 @@ def build(): "--report", str(directory / "dependencies.json"), *map(str, wheels), - "opentelemetry-sdk>=1.20.0", - "opentelemetry-propagator-aws-xray", ], check=True, ) @@ -133,9 +127,7 @@ def build(): for name in ( "__init__.py", "fixture.py", - "telemetry.py", "shared_observations.py", - "shared_exporter.py", ): shutil.copy(ROOT / "lmi_tests" / name, target / "lmi_tests" / name) artifact = directory / "function.zip" diff --git a/lmi_tests/evidence.py b/lmi_tests/evidence.py index 38c9c0abd..6439741fd 100644 --- a/lmi_tests/evidence.py +++ b/lmi_tests/evidence.py @@ -30,126 +30,6 @@ def check_controls(events): raise CollectionError(f"Control {error['gate']} failed: {error['error']}") -def held_loser(events): - """Establish the fault before asserting SDK cleanup; a failed loser is invalid.""" - check_controls(events) - ready = select(events, "WINNER_READY") - if not ready: - raise CollectionError( - "Missing winner-ready evidence for the blocked-loser fixture" - ) - cutoff = ready[0] - gate = cutoff["marker"] + "-loser" - local = [ - e for e in events if e["request"] == cutoff["request"] and e.get("gate") == gate - ] - if not any( - e["phase"] == "EFFECT" and e["sequence"] < cutoff["sequence"] for e in local - ) or any( - e["phase"] == "IO_EXIT" and e["sequence"] < cutoff["sequence"] for e in local - ): - raise CollectionError("Losing branch was not held when the winner became ready") - - -def checkpoint_held(events, gate): - check_controls(events) - blocked = [e for e in select(events, "BLOCKED") if e.get("gate") == gate] - if not blocked: - raise CollectionError("Checkpoint response was not held by the control gate") - held = blocked[0] - local = [e for e in events if e["request"] == held["request"]] - assert any(e["sequence"] < held["sequence"] for e in select(local, "ACK_HELD")), ( - "No real acknowledged checkpoint was held" - ) - assert not [ - e - for e in local - if e["sequence"] > held["sequence"] - and e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE", "CHECKPOINT_EXIT"} - ], "Wrapper/checkpoint exited before the held response settled" - - -def scope_exit(events, gate, status=None): - """A result or failure must not abandon live user work (Java in-flight cases).""" - check_controls(events) - assert not select(events, "ESCAPE"), "Fixture emergency exit is not SDK cleanup" - starts = [e for e in select(events, "BLOCKED") if e.get("gate") == gate] - if not starts: - raise CollectionError("Missing in-flight work/cleanup precondition") - request = starts[0]["request"] - local = [e for e in events if e["request"] == request] - ends = select(local, "WRAPPER_RETURN") + select(local, "WRAPPER_RAISE") - for end in ends: - exits = [e for e in select(local, "IO_EXIT") if e.get("gate") == gate] - assert exits and exits[0]["sequence"] < end["sequence"], ( - "Wrapper exited while invocation-owned work was still active" - ) - if status is not None: - assert any(e.get("status") == status for e in ends), ( - f"No {status} return for the original request" - ) - lifecycle(events) - - -def suspension_cleanup(events, history): - check_controls(events) - pending = [e for e in select(events, "WRAPPER_RETURN") if e["status"] == "PENDING"] - assert pending, "No real PENDING invocation observed" - assert len({e["request"] for e in select(events, "WRAPPER_ENTER")}) >= 2, ( - "No real resume" - ) - for end in pending: - local = [e for e in events if e["request"] == end["request"]] - enters, exits = select(local, "CLEANUP_ENTER"), select(local, "CLEANUP_EXIT") - assert ( - enters - and exits - and enters[0]["sequence"] < exits[0]["sequence"] < end["sequence"] - ), "PENDING preceded root finally cleanup" - bodies = [e for e in select(events, "BODY") if e["operation"] == "success"] - assert len(bodies) == 1, "Completed body repeated after cleanup/suspension" - assert any(e.get("EventType") == "WaitSucceeded" for e in history) - lifecycle(events) - - -def progress(events, marker, seconds=15): - check_controls(events) - entered, exited = select(events, "PROGRESS_BEGIN"), select(events, "PROGRESS") - assert entered and exited, ( - "Nested work did not progress with single-lane branch pools" - ) - assert exited[0]["time"] - entered[0]["time"] <= seconds, ( - "Nested orchestration exhausted its progress budget" - ) - assert exited[0]["value"] == [[marker + ":0", marker + ":1"], marker], ( - "Nested results crossed invocation boundaries" - ) - lifecycle(events) - - -def late_operation(events, history): - held_loser(events) - ready = select(events, "WINNER_READY")[0] - assert select(events, "WINNER_SELECTED"), "Parent completion was not observed" - attempts, rejected = select(events, "LATE_ATTEMPT"), select(events, "LATE_REJECTED") - assert attempts and attempts[0]["sequence"] > ready["sequence"], ( - "Late operation was not attempted after winner readiness" - ) - assert not [e for e in select(events, "BODY") if e["operation"] == "late-work"], ( - "#741: abandoned child executed a late user side effect" - ) - assert rejected and rejected[0]["error"] == "OrphanedChildException", ( - "#741: completed child scope did not reject subsequent work" - ) - assert not select(events, "LATE_ACCEPTED"), ( - "#741: completed child scope accepted a late operation" - ) - assert not [e for e in history if e.get("Name") == "late-work"], ( - "#741: late operation reached the service" - ) - lifecycle(events) - - def overlap(events, markers, count, environment=None): blocked = [e for e in select(events, "BLOCKED") if e["marker"] in markers] for start in blocked: @@ -191,131 +71,5 @@ def overlap(events, markers, count, environment=None): return env raise PlacementError(f"No {count}-process overlap in one execution environment") - -def lifecycle(events): - returned = select(events, "WRAPPER_RETURN") + select(events, "WRAPPER_RAISE") - assert returned, "No wrapper exit observed" - assert not select(events, "ESCAPE"), "Test emergency timeout released SDK work" - for end in returned: - local = [e for e in events if e["request"] == end["request"]] - begins = select(local, "WRAPPER_ENTER") - assert len(begins) == 1, "Missing/duplicate wrapper entry" - baseline = {t["id"] for t in begins[0]["resources"]["threads"]} - residual = [ - t - for t in end["resources"]["threads"] - if t["id"] not in baseline - and ( - t["name"].startswith("dex-") - or t["name"].startswith("ThreadPoolExecutor") - ) - ] - assert not residual, ( - f"Invocation-owned workers survived wrapper exit: {residual}" - ) - assert any( - e["sequence"] < end["sequence"] for e in select(local, "USER_EXIT") - ), "User code still running" - assert not [ - e - for e in local - if e["sequence"] > end["sequence"] - and e["phase"] - in { - "BODY", - "EFFECT", - "CHECKPOINT_CALL", - "CHECKPOINT_ACK", - "CHECKPOINT_EXIT", - "POLL_CALL", - "POLL_EXIT", - } - ], "Late invocation activity" - - -def replay(events, history, marker): - assert len({e["request"] for e in select(events, "WRAPPER_ENTER")}) >= 2, ( - "No real replay" - ) - assert any(e["status"] == "PENDING" for e in select(events, "WRAPPER_RETURN")), ( - "No suspension" - ) - for name, terminal in [("success", "StepSucceeded"), ("failure", "StepFailed")]: - bodies = [e for e in select(events, "BODY") if e["operation"] == name] - assert len(bodies) == 1 and bodies[0]["value"] == marker, ( - f"Completed {name} repeated/contaminated" - ) - entries = [e for e in history if e.get("Name") == name] - assert sum(e["EventType"] == terminal for e in entries) == 1, ( - f"Missing {terminal}" - ) - assert len({e["Id"] for e in entries}) == 1, "Operation identity changed" - failures = select(events, "STORED_FAILURE") - assert len(failures) >= 2 and all( - e["message"] == "expected:" + marker and e["error_type"] == "ValueError" - for e in failures - ) - attempts = [ - e["attempt"] for e in select(events, "BODY") if e["operation"] == "retry" - ] - assert attempts == [1, 2], f"Unexpected retry side effects: {attempts}" - assert {"WaitSucceeded", "CallbackSucceeded", "ContextSucceeded"} <= { - e["EventType"] for e in history - } - lifecycle(events) - - -def early_completion(events, grace): - held_loser(events) - winners = select(events, "WINNER_SELECTED") - assert winners, "Completion policy did not select a winner" - winner = winners[0] - returns = [ - e for e in select(events, "WRAPPER_RETURN") if e["request"] == winner["request"] - ] - assert returns, "#741: result computed but wrapper pinned by losing branch" - assert returns[0]["time"] <= winner["time"] + grace, "#741: cleanup exceeded grace" - assert not [ - e for e in select(events, "EFFECT") if e["time"] > winner["time"] + grace - ], "#741: abandoned scope still performs side effects" - lifecycle(events) - - -def deadline(events, request, grace): - local = [e for e in events if e["request"] == request] - check_controls(local) - entry = select(local, "WRAPPER_ENTER")[0] - cutoff = entry["deadline"] + grace - assert not [ - e - for e in local - if e["time"] > cutoff - and e["phase"] in {"BODY", "EFFECT", "CHECKPOINT_CALL", "CHECKPOINT_ACK"} - ], "#741: work continued after invocation deadline + cleanup grace" - assert not select(local, "ESCAPE"), "Emergency release is not SDK deadline handling" # A runtime-supported worker retirement can have no final diagnostic. Recovery # and full-capacity placement must therefore also be asserted by the driver. - - -def warm_resources(events, min_reuses=3): - processes = {} - for event in select(events, "WRAPPER_RETURN"): - processes.setdefault((event["environment"], event["process"]), []).append(event) - reused = [ - sorted(v, key=lambda e: e["time"]) - for v in processes.values() - if len(v) >= min_reuses - ] - if not reused: - raise PlacementError("No observed warm Python process reuse") - for samples in reused: - # First invocation initializes legitimate client/serialization caches. - baseline = samples[1]["resources"] - for sample in samples[2:]: - current = sample["resources"] - assert len(current["threads"]) <= len(baseline["threads"]) + 1 - assert current["fds"] <= baseline["fds"] + 8, "File descriptors accumulate" - assert current["rss"] <= baseline["rss"] + 32 * 1024 * 1024, ( - "RSS grew beyond cache allowance" - ) - lifecycle(events) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 25d46ea19..bb3a215aa 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -3,10 +3,10 @@ """Fault-injection handlers for real LMI. Never install these as SDK examples.""" from contextlib import nullcontext +from dataclasses import dataclass, field import fcntl import json import os -from pathlib import Path import threading import time import uuid @@ -17,10 +17,8 @@ from aws_durable_execution_sdk_python import StepError, durable_execution from aws_durable_execution_sdk_python.config import ( - CompletionConfig, Duration, MapConfig, - ParallelConfig, StepConfig, ) from aws_durable_execution_sdk_python.execution import ( @@ -57,13 +55,7 @@ def environment_id(path="/tmp/python-lmi-environment"): def resources(): - # Current RSS, not ru_maxrss (a lifetime high-water mark). - rss_pages = int(Path("/proc/self/statm").read_text().split()[1]) - return { - "threads": [{"id": t.ident, "name": t.name} for t in threading.enumerate()], - "fds": len(list(Path("/proc/self/fd").iterdir())), - "rss": rss_pages * os.sysconf("SC_PAGE_SIZE"), - } + return {"threads": [{"id": t.ident, "name": t.name} for t in threading.enumerate()]} def observe(trace, kind, **fields): @@ -84,8 +76,6 @@ def __init__(self, payload, context, execution_arn): self.payload = dict(payload) if "runId" in self.payload: self.payload.setdefault("run", self.payload["runId"]) - if "otelView" in self.payload: - self.payload.setdefault("otel_view", self.payload["otelView"]) self.shared = bool(self.payload.get("coverageProfile")) if self.shared: for key in ( @@ -136,8 +126,6 @@ def __init__(self, payload, context, execution_arn): "role", "coverageProfile", "sharedScenario", - "otelView", - "exportMode", "behaviorBudgets", ) if key in self.payload @@ -276,12 +264,25 @@ def gate_release( source="control-poll", ) return released + if ( + self.payload["scenario"] == "deadline" + and time.time() >= self.deadline + 0.2 + ): + observe(self, "cooperative_io_cancelled", source="runtime-deadline") + raise TimeoutError( + "Cooperative fixture I/O crossed the runtime deadline" + ) if not entered: held_at_ms = target_observed_at_ms if on_hold is not None: on_hold(held_at_ms) self.emit("BLOCKED", gate=name) observe(self, "blocked", gateId=name, atMs=held_at_ms) + observe(self, "barrier_arrived", gateId=name, atMs=held_at_ms) + if self.payload["scenario"] == "deadline": + observe(self, "work_in_flight", gateId=name, atMs=held_at_ms) + if self.payload.get("role") == "recovery": + observe(self, "worker_recovered", atMs=held_at_ms) if time.monotonic() >= end: self.emit("ESCAPE", gate=name) observe(self, "escape", gateId=name) @@ -302,6 +303,8 @@ def gate_release( else: self.emit("ALIVE", gate=name) observe(self, "alive", gateId=name) + if self.payload.get("role") == "peer": + observe(self, "peer_progress", gateId=name) if not entered and ready: self.signal(ready) entered = True @@ -379,27 +382,6 @@ def checkpoint(self, **kwargs): source="service", atMs=acknowledged_at, ) - if self.trace.payload["scenario"] == "checkpoint" and any( - u["name"] == "checkpoint-loser" and u["action"] == "SUCCEED" - for u in updates - ): - # Real service response, withheld while a synchronous waiter owns a - # branch. This tests settlement/join ordering without a fake service. - self.trace.emit("ACK_HELD") - observed_operation = next( - item["id"] - for item in updates - if item["name"] == "checkpoint-loser" - and item["action"] == "SUCCEED" - ) - observe( - self.trace, - "ack_held", - callId=call_id, - operationId=observed_operation, - gateId=self.trace.payload["marker"] + "-checkpoint", - ) - self.trace.gate(self.trace.payload["marker"] + "-checkpoint") return response except BaseException as error: self.trace.emit("CHECKPOINT_ERROR", error=type(error).__name__) @@ -416,62 +398,32 @@ def checkpoint(self, **kwargs): observe(self.trace, "checkpoint_exit", callId=call_id, atMs=exited_at) -def nested_progress(context, trace, marker): - """Record public one-lane configurations without changing nested work.""" - trace.emit("PROGRESS_BEGIN") - observe(trace, "progress_begin") - - def children(child): - def mapped(branch): - config = MapConfig(max_concurrency=1) - with configuration(trace, "inner-map", config.max_concurrency): - return branch.map( - [0, 1], - lambda inner, item, _index, _all: inner.run_in_child_context( - lambda leaf: leaf.step( - lambda step: trace.body("leaf", f"{marker}:{item}", step), - name="leaf", - ), - name="grandchild", - ), - name="inner-map", - config=config, - ).get_results() - - config = ParallelConfig(max_concurrency=1) - with configuration(trace, "outer-parallel", config.max_concurrency): - return child.parallel( - [ - mapped, - lambda branch: branch.run_in_child_context( - lambda leaf: leaf.step( - lambda step: trace.body("sibling", marker, step), - name="sibling", - ), - name="sibling-child", - ), - ], - name="outer-parallel", - config=config, - ).get_results() - - result = context.run_in_child_context(children, name="progress-child") - assert result == [[marker + ":0", marker + ":1"], marker] - trace.emit("PROGRESS", value=result) - observe(trace, "progress", value=result) - - def workflow(event, context, trace): - scope = ( - trace.observer.root_scope() if event.get("coverageProfile") else nullcontext() - ) + trace.durable_context = context + scope = trace.observer.root_scope() if trace.shared else nullcontext() with scope: - return run_workflow(event, context, trace) + try: + result = run_workflow(event, context, trace) + except Exception: + observe(trace, "root_completed", outcome="FAILED") + raise + else: + observe(trace, "root_completed", outcome="SUCCEEDED") + return result def run_workflow(event, context, trace): marker, scenario = event["marker"], event["scenario"] - admitted = False + if scenario not in { + "success", + "failure", + "barrier", + "deadline", + "replay", + "warm-owner", + "warm-probe", + }: + raise ValueError(f"Unsupported lifecycle fixture: {scenario}") try: if target := event.get("target_environment"): # Record placement before any fault work. Replays consume the same @@ -486,11 +438,10 @@ def run_workflow(event, context, trace): ) if not accepted: return marker - admitted = True value = context.step(lambda s: trace.body("success", marker, s), name="success") if scenario == "failure": raise ValueError("expected:" + marker) - if scenario in {"barrier", "deadline", "nested-progress"}: + if scenario in {"barrier", "deadline"}: context.step( lambda step: trace.gate( event["gate"], effects=scenario == "deadline", attempt=step.attempt @@ -499,120 +450,18 @@ def run_workflow(event, context, trace): config=NO_RETRY, ) context.step(lambda s: trace.body("after-io", marker, s), name="after-io") - if scenario == "nested-progress": - nested_progress(context, trace, marker) - elif scenario == "suspend-cleanup": - context.wait(Duration.from_seconds(5), name="pause") - elif scenario in { - "parallel", - "map", - "nested", - "return-inflight", - "failure-inflight", - "late-operation", - }: - - def losing(child): - def io(step): - trace.gate( - marker + "-loser", - effects=True, - attempt=step.attempt, - ready=marker + "-loser-started", - ) - return marker - - try: - return child.step(io, name="losing-io", config=NO_RETRY) - finally: - if scenario == "late-operation": - # The preceding step's final checkpoint can itself be - # rejected after the parent finishes. Still test the next - # SDK boundary, without swallowing unrelated failures. - trace.emit("LATE_ATTEMPT") - scope_id = ( - trace.observer.closed_ancestor() - if event.get("coverageProfile") - else None - ) - observe(trace, "late_attempt", scopeId=scope_id) - try: - child.step( - lambda step: trace.body("late-work", marker, step), - name="late-work", - config=NO_RETRY, - ) - except OrphanedChildException as error: - trace.emit("LATE_REJECTED", error=type(error).__name__) - observe( - trace, - "late_rejected", - scopeId=scope_id, - error={"type": type(error).__name__}, - ) - else: - trace.emit("LATE_ACCEPTED") - observe(trace, "late_accepted", scopeId=scope_id) - - def winning(child): - def io(_): - trace.gate(marker + "-loser-started") - trace.emit("WINNER_READY") - observe(trace, "winner_ready") - return marker - - return child.step(io, name="winner", config=NO_RETRY) - - def race(child): - if scenario == "map": - return child.map( - [0, 1], - lambda c, item, _index, _all: ( - losing(c) if item == 0 else winning(c) - ), - name="race-map", - config=MapConfig( - max_concurrency=2, - completion_config=CompletionConfig(min_successful=1), - ), - ) - return child.parallel( - [losing, winning], - name="race-parallel", - config=ParallelConfig( - max_concurrency=2, - completion_config=CompletionConfig.first_successful(), - ), - ) - - if scenario == "nested": - context.map( - [0], - lambda c, _item, _index, _all: c.run_in_child_context( - race, name="nested-child" - ), - name="outer-map", - config=MapConfig(max_concurrency=3), - ) - else: - race(context) - trace.emit("WINNER_SELECTED") - observe(trace, "winner_selected") - if scenario == "failure-inflight": - trace.emit("ROOT_FAILURE_READY") - raise ValueError("expected:" + marker) - elif scenario == "checkpoint": - context.parallel( - [ - lambda c: c.step(lambda _: marker, name="checkpoint-loser"), - lambda c: c.step(lambda _: marker, name="checkpoint-peer"), - ], - name="checkpoint-pool", - config=ParallelConfig(max_concurrency=2), + elif scenario == "warm-owner": + context.step( + lambda _: start_warm_work(trace), + name="start-late-work", + config=NO_RETRY, + ) + elif scenario == "warm-probe": + context.step( + lambda _: release_warm_work(trace), + name="release-late-work", + config=NO_RETRY, ) - if event.get("coverageProfile"): - scope_id = trace.observer.completed_context("checkpoint-pool") - trace.observe("branch_join", scopeId=scope_id) elif scenario == "replay": def fail(step): @@ -672,21 +521,120 @@ def retry(step): trace.emit("USER_RESULT", value=value) return value finally: - if scenario == "suspend-cleanup" and admitted: - # Test-only root-finally diagnostics. This gate changes only exit - # timing; it introduces no durable operations during suspension. - trace.emit("CLEANUP_ENTER") - observe(trace, "cleanup_enter") - try: - trace.gate(marker + "-cleanup") - finally: - trace.emit("CLEANUP_EXIT") - observe(trace, "cleanup_exit") exited_at = time.time() * 1000 trace.emit("USER_EXIT") observe(trace, "user_exit", atMs=exited_at) +def admission_probes(trace): + trace.observe("task_start", taskId="admission-probes", source="fixture") + closed = False + try: + for probe, operation in ( + ( + "checkpoint", + lambda: trace.durable_context.state.create_checkpoint(is_sync=False), + ), + ( + "operation", + lambda: trace.durable_context.step( + lambda _: trace.observe("probe_body_executed"), + name="operation-admission-probe", + config=NO_RETRY, + ), + ), + ): + trace.observe("probe_enter", probe=probe) + error_record = None + try: + operation() + except OrphanedChildException as error: + outcome = "REJECTED" + error_record = {"type": type(error).__name__, "message": str(error)} + except BaseException as error: + outcome = "ERROR" + error_record = {"type": type(error).__name__, "message": str(error)} + else: + outcome = "ACCEPTED" + observed_at = time.time_ns() / 1_000_000 + if outcome == "REJECTED" and not closed: + trace.observe( + "admission_closed", atMs=observed_at, source="sdk-rejection" + ) + closed = True + trace.observe( + "admission_probe", + probe=probe, + outcome=outcome, + error=error_record, + atMs=observed_at, + ) + finally: + trace.observe("task_exit", taskId="admission-probes", source="fixture") + + +def start_admission_probes(trace): + # A broken SDK can leave a late operation waiting forever. Keep that real + # task visible to the retirement audit instead of declaring it settled. + thread = threading.Thread( + target=admission_probes, args=(trace,), daemon=True, name="lmi-admission-probes" + ) + thread.start() + thread.join(timeout=0.5) + + +@dataclass +class WarmWork: + trace: object + ready: threading.Event = field(default_factory=threading.Event) + observed_during: str | None = None + + +WARM_WORK = {} +WARM_LOCK = threading.Lock() + + +def start_warm_work(trace): + key = (trace.payload["runId"], trace.payload["caseId"]) + work = WarmWork(trace) + with WARM_LOCK: + if key in WARM_WORK: + raise ValueError("A warm case cannot replace its original work") + WARM_WORK[key] = work + + def late(): + trace.observe("task_start", taskId="late-callback", source="fixture") + try: + if not work.ready.wait(timeout=120): + trace.observe( + "observation_error", + message="Original late work never observed warm reuse", + ) + return + trace.observe("late_activity", observedDuringRequestId=work.observed_during) + admission_probes(trace) + finally: + trace.observe("task_exit", taskId="late-callback", source="fixture") + with WARM_LOCK: + WARM_WORK.pop(key, None) + + threading.Thread(target=late, daemon=True, name="lmi-late-callback").start() + return None + + +def release_warm_work(trace): + key = (trace.payload["runId"], trace.payload["caseId"]) + with WARM_LOCK: + work = WARM_WORK.get(key) + if work is None: + raise ValueError("Warm request did not reuse the original process") + work.observed_during = trace.identity["request"] + work.ready.set() + # Keep the warm invocation alive long enough to observe the old callback. + time.sleep(0.5) + return None + + def handler(event, context): invocation = ( event @@ -701,8 +649,6 @@ def handler(event, context): payload = dict(payload) if "runId" in payload: payload.setdefault("run", payload["runId"]) - if "otelView" in payload: - payload.setdefault("otel_view", payload["otelView"]) if payload["run"] != os.environ["LMI_RUN_ID"]: raise ValueError("Input belongs to a different deployment") trace = Trace(payload, context, invocation.durable_execution_arn) @@ -750,12 +696,7 @@ def invoke(plugins=None): source="runtime", ) try: - if payload.get("otel_view"): - from lmi_tests.telemetry import invoke as invoke_with_telemetry - - result = invoke_with_telemetry(trace, invoke) - else: - result = invoke() + result = invoke() trace.emit( "WRAPPER_RETURN", status=result["Status"], @@ -771,6 +712,24 @@ def invoke(plugins=None): "resources": trace.sdk_resources, } ) + if result["Status"] == "PENDING": + observe( + trace, + "pending", + atMs=trace.sdk_completed_at * 1000, + source="sdk-wrapper", + ) + if ( + payload.get("sharedScenario") + in { + "normal-return", + "normal-failure", + "deadline-during-step", + "peer-recovery", + } + and payload.get("role") == "victim" + ): + start_admission_probes(trace) return result except BaseException as error: if payload.get("coverageProfile") and trace.sdk_observation is None: diff --git a/lmi_tests/python-lifecycle-schema.md b/lmi_tests/python-lifecycle-schema.md deleted file mode 100644 index 10b8b6166..000000000 --- a/lmi_tests/python-lifecycle-schema.md +++ /dev/null @@ -1,155 +0,0 @@ -# Python lifecycle regression observations - -This SDK-owned version 1 contract reuses the common evidence envelope and -retirement helpers with local `python-lmi-*` requirement IDs. It is validated by -`lmi_tests.python_catalog`, not registered in the shared CLI or its packaged schema. -It applies when `coverageProfile` is `python-lifecycle` and `runtime` is `python`, -and preserves the 14 SDK-owned base fixture variants on each native-concurrency profile -1 and 2. It is not a portable promise to terminate arbitrary non-cooperative I/O. -The standard LMI profile remains unchanged. - -The runner validates the common envelope, original launch ownership, manifest -identity, declared budgets, and final retirement independently of these scenario -assertions. `validate_lifecycle_regressions(evidence, phase=...)` validates each -base cell. `validate_lifecycle_events` also supplies the lifetime checks for OTel -cases, and `validate_replay_evidence(evidence, execution_arn)` validates one replay -execution within a warm sequence. OTel cells additionally use `lmi_tests.python_otel` and the shared canonical -OTel model; passing lifecycle checks alone cannot accept telemetry. - -## Common observations - -Each retained attempt adds its immutable fixture input `marker`; warm sequences -also record `fixture` (`success`, `failure`, or `replay`). Native-overlap evidence -records both the OS `pid` and the stable `processId`, plus the common environment, -worker, request and execution identities. Source-specific diagnostics belong in -the retained raw evidence rather than durable operation names. - -Every event uses the common `requestId`, `ownerRequestId`, `executionArn`, `kind` -and `atMs`, plus a nonnegative, unique per-request `sequence`. The canonical -sequence follows actual semantic boundary times and resolves equal timestamps -using observed causal order. It is **not necessarily the source's emission -index**. The Python fixture records actual SDK completion time and later emits a -retrospective wrapper diagnostic after OTel exit. An adapter must normalize that -record to the actual SDK boundary, retaining the original `sourceSequence`, -`observedAtMs` and source timestamps in raw evidence. Do not invent ordering from -missing observations or use a later resource snapshot to claim an earlier -boundary was quiescent. OTel envelopes use UTC Unix epoch milliseconds. - -Each request needs `wrapper_enter`, `user_exit`, and exactly one `wrapper_return` -or `wrapper_raise`, with the actual observed `outcome`. User exit must precede -wrapper exit. The native platform outcome and SDK wrapper outcome remain -independent after a proven `platform_timeout`: a process can later return or -raise (normalized wrapper outcome `FAILED`) after its native request timed out. -Retain that actual wrapper outcome and its error; do not relabel it `TIMED_OUT`. -Non-timeout requests still require matching outcomes. Task/request settlement, -clock, collection, and stale-activity assertions apply to both paths. Wrapper entry and exit carry `resources` snapshots: - -```json -{"threads": [{"id": "thread-id", "name": "MainThread"}], "fds": 12, "rssBytes": 1048576} -``` - -Thread identities must be unique. New `dex-`/`ThreadPoolExecutor` workers or -threads explicitly tagged with that invocation's `ownerRequestId` cannot survive -wrapper exit. This is a Python-profile diagnostic, not a universal thread naming -contract. Finite FD and RSS counters support the warm baseline checks. - -Tracked `task_start`/`task_exit` and `io_start`/`io_exit` pairs use stable `taskId`. -`checkpoint_call`/`checkpoint_exit` and `poll_call`/`poll_exit` pairs use `callId`. -Every observed start must settle before wrapper exit. `body`, `effect`, -`checkpoint_call`, `checkpoint_ack`, `checkpoint_exit`, `poll_call`, and `poll_exit` -are forbidden after that boundary, including acknowledgment/settlement of an -earlier request. `body` carries `operation`, `operationId`, `attempt` and `value`; -its operation ID/name must match the collected service history. Names may repeat -under distinct parent contexts; IDs remain authoritative. - -An actual timed-out invocation may lack a wrapper exit. It must then have an -observed `platform_timeout` with `source: platform`, plus actual root -`task_exit` (`taskId: root`) or `worker_retired` with `source: platform` under the -explicit `replace-worker` recovery policy. Deadline/recovery cases require an -explicit `reuse-worker` or `replace-worker` policy, bound to the manifest. Absence of a final log is not positive -retirement evidence. Recovery and final owned-resource retirement still apply. -`escape` and `control_error` records cannot establish successful SDK cleanup. - -All gates belong to `faultsCreated`. `blocked`, `effect`, `alive`, and `io_exit` -identify the `gateId`; `fault_released` records the same gate, actual time and -`mode: normal`. Emergency or unclassified releases fail the assertion. The -validator also examines final `retirement.faults`, so later release evidence -cannot erase an earlier failure. - -A timeout fixture also records the invocation's actual `deadlineAtMs`. An owned -`runtime_deadline` event captures `source: runtime`, `atMs`, and the runtime -context's positive `remainingTimeMs`. The remaining time cannot exceed the -deployed `profile.timeoutMs`; `deadlineAtMs` must equal that observation's time -plus remaining time, allowing at most one millisecond for timestamp rounding. -The platform timeout must cross this actual deadline. Cleanup/recovery/retry -budgets start there. Do not derive it from wrapper entry plus the configured full -timeout: native dispatch and setup may already have consumed invocation time. -An input-injected deadline or missing runtime observation cannot establish this -precondition. The configured timeout remains deployment identity. - -## Manifest-declared budgets - -`regression.budgets` must exactly match the case's manifest `behavior_budgets`. -The adapter cannot increase these limits after seeing observations. All base -cases require positive `observationGraceMs`, covered by collection after wrapper -exit. Additional fields are: - -| Cases | Additional declared budgets | -| --- | --- | -| `early-completion-parallel/map/nested` | `cleanupMs` | -| `deadline-recovery` | `cleanupMs`, `recoveryMs` | -| `timeout-retry` | `cleanupMs`, `retryMs` | -| `nested-progress` | `progressMs` | -| `warm-lifecycle` | Nonnegative `maxThreadGrowth`, `maxFdGrowth`, `maxRssGrowthBytes` | - -Budgets are fixture acceptance settings, not product SLAs. Runner provisioning, -placement, execution, collection and retirement budgets remain independent. - -## Required scenario evidence - -| ID and selector | Observations and assertions | -| --- | --- | -| 1, `native-concurrency` | c1 selects `victim`; c2 selects `victim` and `peer`. All selected requests block in one environment, with distinct execution/request/process/PID identities. `alive` or `effect` positively covers the overlap pivot; missing exits alone do not prove liveness. | -| 4, `deadline-recovery` | Original `victim` has controlled work blocked before its actual platform deadline. No original BODY/EFFECT/checkpoint activity survives deadline plus `cleanupMs`. c2 first establishes a healthy `peer`. A full recovered cohort (`recovery`, plus `recovery-peer` for c2) overlaps in the original environment within `recoveryMs`, before fault release. `reuse-worker` includes the original process/worker; `replace-worker` identifies it through `replacesWorkerId`. Recovery requests succeed. | -| 5, `replay-callback-retry` | At least one real PENDING request precedes a successful resume of the same execution. `success` and `failure` bodies each occur exactly once with the original marker and stable service operation ID. `stored_failure` is observed on at least two requests with normalized `{type: ValueError, message: expected:}`. `retry` body attempts are exactly integer `[1, 2]` (booleans are invalid), each with the original marker/ID. Actual history contains successful WAIT, CALLBACK, and CONTEXT operations. | -| 6, `warm-lifecycle` | `regression.warmSequence` lists nine distinct executions in `[success, failure, replay] * 3` order, each `{executionArn, fixture}`. Executions run sequentially; fixture routing and terminal outcomes agree. Replay executions satisfy the full replay validator. At least three wrapper samples reuse one environment/process. Compare samples after the second against that post-cache baseline using the declared thread/FD/RSS allowances. | -| 9, `early-completion-parallel/map/nested` | `regression.gateId` selects the held loser. An effect precedes `winner_ready`, with no prior loser exit; `winner_selected` follows. Wrapper return and cessation of activity occur within `cleanupMs`, before any driver release. Actual service history contains `race-parallel`, `race-map`, or nested `outer-map`/`nested-child`/`race-parallel` scopes respectively. | -| 10, `checkpoint-settlement` | A real `checkpoint_call` receives `checkpoint_ack` (`source: service`, `operationIds`) for the actual `checkpoint-loser` operation. `ack_held` carries the matching `callId`, `operationId`, and gate. After positive `blocked`, normal release precedes `checkpoint_exit`, then `branch_join` of the actual `checkpoint-pool` context, then wrapper exit. An unrelated acknowledgment cannot establish this precondition. | -| 11, `timeout-retry` | Actual selected timeout is followed within `retryMs` by selected `retry` on the same execution ARN. `service_retry` is observed from the platform and binds `previousRequestId` to the original request. The original successful step body occurs exactly once across all retained requests. Interrupted I/O effects are not given an exactly-once requirement. | -| 12, `late-operation` | `scope_closed` and `late_attempt` identify the same actual service child-context ID. Both observed closure and `winner_selected` precede the late attempt; their relative order is not prescribed. In particular, Python's public operation-end hook runs before the SDK call returns and the fixture emits `winner_selected`. The losing I/O settles before the late attempt. `late_rejected` binds the same scope with normalized `OrphanedChildException`. No `late_accepted`, late body, late checkpoint dispatch or `late-work` service operation is allowed. The invocation may still be open: child closure is a separate boundary. | -| 13, `return-inflight` / `failure-inflight` | Root `user_exit` occurs after winner selection while losing I/O is still held. Normal release/IO settlement precedes the actual wrapper exit. c2 positively observes peer progress during this interval. Preserve success or the original normalized `ValueError`/marker failure after settlement. | -| 14, `pending-cleanup` | Root `cleanup_enter` precedes the held gate; normal release and IO exit precede `cleanup_exit`, `user_exit`, then PENDING wrapper return. A later request resumes the same execution, completes a real WAIT, and does not repeat the completed success body. c2 peer progress is observed while cleanup is held. | -| 15, `nested-progress` | All native workers establish live overlap, then `progress_begin`/`progress` bound the nested operation budget. Each request returns its own `[[marker:0, marker:1], marker]` result. Actual `outer-parallel`, `inner-map`, and `progress-child` operations succeed. `operation_config` records the public fixture's integer `maxConcurrency: 1` for the two pool operations, bound by operation ID and `source: public-fixture`; this is not invented service-history metadata. | - -The six OTel-only selectors (`runtime-carrier-sampled`, -`runtime-carrier-unsampled`, `otel-warm-lifecycle`, `otel-concurrent`, -`export-error-recovery`, `held-export`) retain the common settled-lifecycle checks -and delegate their remaining assertions to the OTel extension. The SDK adapter -must implement these fixtures through real public APIs and preserve raw source -evidence. Synthetic positive controls validate the harness; they do not count as -real LMI SDK acceptance or authorize replacing Python-specific regression tests. - -For in-flight return/failure and root-cleanup peer measurements, -`regression.gateId` selects the victim's actual held-work gate. A separate -winner-admission handshake may produce additional `blocked` events, but cannot -supply that gate's liveness or prevent its early exit from being detected. Peer -and other native cohort roles still require one unambiguous blocked gate. - -A deadline fixture that normally releases its held work after measurement can -retain `regression.deadlineMeasurement`: `selectedRequests`, `deadlineAtMs`, -`cleanupCutoffAtMs`, `observationCutoffAtMs`, `observedAtMs`, and -`sourceEvidenceUri`. The selected requests and cutoffs must match the actual -runtime deadline and immutable case budgets. Its source URI must be retained in -`collection.evidenceUris`, and capture must cover the full cleanup-plus-grace -window before the fault's `releaseRequestedAtMs`. - -Only a valid snapshot permits the stale-activity window to end when the -controller begins normal release. Activity between capture and release is still -checked. New user bodies and SDK dispatch after the cutoff remain failures even -after release; only paired completion of already dispatched I/O is cleanup. -All events remain in the main ledger, including later normal cleanup, and task, -request, wrapper and retirement checks still inspect that complete ledger. -Missing snapshots retain the legacy full-window check; inconsistent snapshots -fail rather than shorten the observation window. This separates deliberately -released cleanup from the measured deadline behavior without removing actual -pre-release late effects. diff --git a/lmi_tests/python-otel.md b/lmi_tests/python-otel.md deleted file mode 100644 index 303370f0a..000000000 --- a/lmi_tests/python-otel.md +++ /dev/null @@ -1,67 +0,0 @@ -# Python-owned OTel lifecycle checks - -The SDK-owned `python-lifecycle` suite defines 48 observability cells: twelve scenario selectors, both -OTel views, and deployed native concurrency 1 and 2. It supplements the 28 base -lifecycle cells. Each cell declares one `otelView` and collects exactly that view -under `telemetry.views`. The original generic LMI profile still collects both -views in a cohort. Python requirements use `python-lmi-*` IDs and are registered by -`lmi_tests.python_catalog.suite_definition()`. Local positive controls do not -establish real-LMI acceptance. - -`validate_lifecycle_telemetry` checks all plugin-enabled lifecycle scenarios, -including deadline, timeout retry, pending cleanup, late operation and root -return/failure with work in flight. Call it together with the SDK lifecycle -validator. The matrix fixes the required scenarios, views, configurations and -`behavior_budgets`; the adapter must not select weaker assertions after observing -results. `regression.budgets.exportMs` is the configured exporter-call bound, -alongside `observationGraceMs` and applicable lifecycle budgets. - -Use the existing owned-event fields (`requestId`, `ownerRequestId`, -`executionArn`, `atMs`, `sequence`) for these observations: - -| Event | Additional observed fields | -| --- | --- | -| `otel_enter` | `providerId`, `exporterId`, `contextBefore: {traceId, spanId}`; both context IDs may be null. | -| `otel_factory` | The actual `pluginInstanceId` and factory `pluginExecutionArn`. Python creates a distinct plugin instance per native request. | -| `otel_export_started` | Actual exporter entry with `traceId` and `spanId`. | -| `otel_export` | Matching exporter completion, with `outcome: success | failure | held`. | -| `otel_exit` | Post-call `contextAfter: {traceId, spanId}` and observed integer `exporterShutdowns`, which must be zero. | -| `export_held` / `export_released` | The same case-owned `gateId`; held export also binds `traceId` and `spanId`, and normal release records `mode: normal`. | - -The real boundary order is `otel_enter`, factory creation inside the SDK call, -export completion, actual SDK `wrapper_return`/`wrapper_raise`, then the -`otel_exit` diagnostic. The latter observes restored context after the SDK call; -it is not the SDK completion timestamp. An adapter may record the saved SDK -boundary after emitting a diagnostic, but must retain that original boundary -time. Normalize semantic event order separately from raw emission order and -retain raw emission sequence/timestamps in the referenced evidence. Held export -must prevent actual SDK completion until normal release, independently of when -post-call diagnostics arrive. Emergency release cannot establish this result. - -Every `telemetry.views[otelView].spans` record retains its independent export -request, a boolean `sampled` from the actual SpanContext, and the canonical -`span_to_dict` payload. Failed exporter attempts retain their actual canonical -spans and matching failure events; they do not claim backend delivery. Carrier -records use the genuine Python `process-environment` source. The sampled and -unsampled runtime cases additionally retain `callerHeader` from the actual Invoke -request, allowing the runtime Root and sampling decision to be compared with the -caller. Missing, empty, malformed and injected carriers are local negative -controls, never substitutes for genuine runtime coverage. - -The validators require one logical Invocation span per sampled request and one -terminal Workflow with its actual success/failure status in terminal fixtures. -Logical identities are counted across replay; identical transport re-exports are -retained without promising exactly-once export. A Workflow may start at the -service execution boundary before native admission and export from a terminal -invocation, retaining its original runtime parent. Its recorded service history -must establish that earlier boundary. Export failure must be followed by a -successful invocation in the same worker/process with the same provider. Held -peers must remain active and make progress after another request ends. - -`regression.warmSequence` identifies six distinct executions and their actual -`fixture` values in the order `[success, failure, replay]` twice. Their invocation -records retain `fixture` and the checkpointed `marker`. Replays must satisfy the -same stored-result, stored-failure, callback/wait and retry proof used by the Python -lifecycle validator. Warm process/provider identity is observed rather than -assumed from consecutive requests. These object-allocation and fixture details -belong to this explicit Python profile, not the language-neutral generic suite. diff --git a/lmi_tests/python_catalog.py b/lmi_tests/python_catalog.py deleted file mode 100644 index b76edaf7f..000000000 --- a/lmi_tests/python_catalog.py +++ /dev/null @@ -1,612 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Python-owned LMI requirements and coverage using the shared runner machinery.""" - -from __future__ import annotations - -from collections.abc import Mapping -from dataclasses import dataclass -import json -from pathlib import Path -from typing import Any - -from aws_durable_execution_conformance_tests.lmi.app import SuiteDefinition -from aws_durable_execution_conformance_tests.lmi.matrix import CaseRequirement - - -@dataclass(frozen=True) -class ScenarioDefinition: - selector: str - roles: tuple[str, ...] - slots: int - exclusive: bool - budget_names: tuple[str, ...] - source_tests: tuple[str, ...] - roles_at_two: tuple[str, ...] | None = None - slots_at_two: int | None = None - - -@dataclass(frozen=True) -class Requirement: - number: int - description: str - contract: str - baseline: tuple[ScenarioDefinition, ...] - instrumented: tuple[ScenarioDefinition, ...] - - -# These are Python SDK regression contracts, not shared conformance requirements. -_REQUIREMENTS = ( - Requirement( - number=1, - description="Configured native worker capacity and execution isolation", - contract="Verify genuine native concurrency one and two deployments independently. Establish one original request per configured worker under a bounded barrier; at concurrency two require distinct worker processes overlapping in the same environment. Preserve the selected original requests, results and service histories. A single request to a concurrency-two deployment cannot replace the concurrency-one baseline.", - baseline=( - ScenarioDefinition( - selector="native-concurrency", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_actual_environment_process_concurrency", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ), - instrumented=(), - ), - Requirement( - number=10, - description="Synchronous checkpoint settlement at the branch boundary", - contract="Hold an actual SDK checkpoint request in flight while a child branch completes. The durable wrapper must preserve the supported checkpoint and child-scope settlement boundary. Require independently observed request entry, held interval, service response and wrapper exit; a mocked checkpoint flag or a callback without a real dispatch is insufficient.", - baseline=( - ScenarioDefinition( - selector="checkpoint-settlement", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_real_synchronous_checkpoint_settles_before_branch_join", - ), - ), - ), - instrumented=(), - ), - Requirement( - number=11, - description="Platform timeout retries preserve completed work", - contract="Cross the configured platform invocation timeout and observe a service-started retry with a different request ID for the same durable execution. Completed successful side effects remain checkpointed and are not repeated. Record interrupted work per attempt without imposing exactly-once execution on at-least-once steps; an independently invoked replacement execution cannot satisfy retry.", - baseline=( - ScenarioDefinition( - selector="timeout-retry", - roles=("victim", "retry"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs", "retryMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_service_timeout_retry_does_not_repeat_completed_step", - ), - ), - ), - instrumented=( - ScenarioDefinition( - selector="timeout-retry", - roles=("victim", "retry"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs", "retryMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_service_timeout_retry_does_not_repeat_completed_step", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - ), - ), - ), - Requirement( - number=12, - description="Abandoned child rejects a late durable operation", - contract="After a parent or sibling result abandons a child, allow that residual child to attempt another public durable operation. Prove the attempt was rejected before its user body and checkpoint dispatch; keep the operation absent from service history. Retain the original invocation and scope identities and distinguish fault release from the assertion boundary.", - baseline=( - ScenarioDefinition( - selector="late-operation", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_abandoned_child_rejects_late_durable_operation", - ), - ), - ), - instrumented=( - ScenarioDefinition( - selector="late-operation", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_abandoned_child_rejects_late_durable_operation", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - ), - ), - ), - Requirement( - number=13, - description="Root return and failure settle in-flight work", - contract="Exercise successful root return and root failure while an admitted child operation remains in flight. Observe root completion, child cleanup, request settlement and wrapper exit independently, preserving the original returned result or failure. At native concurrency two, keep a selected healthy peer progressing in the same environment without inheriting the victim scope.", - baseline=( - ScenarioDefinition( - selector="return-inflight", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ScenarioDefinition( - selector="failure-inflight", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ), - instrumented=( - ScenarioDefinition( - selector="return-inflight", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ScenarioDefinition( - selector="failure-inflight", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_root_return_or_failure_settles_inflight_work", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ), - ), - Requirement( - number=14, - description="Pending waits for root cleanup before replay", - contract="Suspend through the public durable API while controlled work remains in the root cleanup path. Observe root-finally entry and exit, actual PENDING wrapper completion, service resume and stable checkpoint history. The wrapper must honor the supported scope-cleanup boundary and replay must not repeat the completed operation body. Native concurrency two includes an established healthy peer.", - baseline=( - ScenarioDefinition( - selector="pending-cleanup", - roles=("victim", "resume"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_pending_waits_for_root_finally_and_then_replays", - ), - roles_at_two=("victim", "resume", "peer"), - slots_at_two=2, - ), - ), - instrumented=( - ScenarioDefinition( - selector="pending-cleanup", - roles=("victim", "resume"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_pending_waits_for_root_finally_and_then_replays", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - roles_at_two=("victim", "resume", "peer"), - slots_at_two=2, - ), - ), - ), - Requirement( - number=15, - description="Nested durable work progresses with bounded branch capacity", - contract="Use the real public child, map and parallel APIs with a single sibling slot to establish nested progress. Launch one root per configured native worker and positively establish the required environment and process placement. Nested work must progress within its declared test budget without starving behind its owning branch, preserving result attribution for every original root.", - baseline=( - ScenarioDefinition( - selector="nested-progress", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "progressMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_nested_single_lane_pools_progress_for_all_runtime_workers", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ), - instrumented=(), - ), - Requirement( - number=4, - description="Deadline admission and original environment capacity recovery", - contract="Observe a real platform timeout of the original victim with controlled work in flight. At native concurrency two, establish a healthy peer in the same environment and verify its progress and successful completion. Before fault release, demand every configured worker slot in the original environment and prove full capacity recovers within the declared deadline-relative cleanup budget. Preserve victim admission and stale-side-effect assertions independently of recovery and final retirement.", - baseline=( - ScenarioDefinition( - selector="deadline-recovery", - roles=("victim", "recovery"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs", "recoveryMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_invocation_deadline_isolation_and_capacity_recovery", - ), - roles_at_two=("victim", "peer", "recovery", "recovery-peer"), - slots_at_two=2, - ), - ), - instrumented=( - ScenarioDefinition( - selector="deadline-recovery", - roles=("victim", "recovery"), - slots=1, - exclusive=True, - budget_names=( - "observationGraceMs", - "cleanupMs", - "recoveryMs", - "exportMs", - ), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_invocation_deadline_isolation_and_capacity_recovery", - "lmi_tests/tests/e2e/test_observability.py::test_plugin_enabled_lifecycle_boundaries", - ), - roles_at_two=("victim", "peer", "recovery", "recovery-peer"), - slots_at_two=2, - ), - ), - ), - Requirement( - number=5, - description="Callback suspension and retry preserve checkpointed work", - contract="Exercise real durable callback suspension, operation retry and stored success and failure results. Resume the same durable execution through the service callback path, retaining every invocation and checkpoint history. Completed user bodies and side effects must not repeat on replay, and retry effects follow the configured per-attempt contract.", - baseline=( - ScenarioDefinition( - selector="replay-callback-retry", - roles=("victim", "resume"), - slots=1, - exclusive=False, - budget_names=("observationGraceMs",), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_checkpoint_replay_callback_and_retry", - ), - ), - ), - instrumented=(), - ), - Requirement( - number=6, - description="Warm success, failure and suspension resource cleanup", - contract="Exercise a repeated sequence of successful, failed and suspended/resumed public handlers. Identify actual process reuse and compare invocation-owned state and process resource baselines over the sequence. Verify branch/thread ownership and cleanup, preserving failure and replay outcomes; an unrelated process cannot replace the observed warm worker.", - baseline=( - ScenarioDefinition( - selector="warm-lifecycle", - roles=( - "victim", - "warm", - "warm-2", - "warm-2-resume", - "warm-3", - "warm-4", - "warm-5", - "warm-5-resume", - "warm-6", - "warm-7", - "warm-8", - "warm-8-resume", - ), - slots=1, - exclusive=True, - budget_names=( - "observationGraceMs", - "maxThreadGrowth", - "maxFdGrowth", - "maxRssGrowthBytes", - ), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_warm_success_failure_and_suspension_cleanup", - ), - ), - ), - instrumented=(), - ), - Requirement( - number=7, - description="Runtime carriers, warm telemetry and native worker attribution", - contract="In both OTel views and both native configurations, separately verify genuine sampled and unsampled runtime carriers, identified warm success/failure/replay, and one original request per configured worker. Preserve caller Root and sampling correlation and execution attribution. At concurrency two, ending one request must not end a held healthy peer; controlled malformed or missing-carrier unit fixtures cannot substitute for native runtime evidence.", - baseline=(), - instrumented=( - ScenarioDefinition( - selector="runtime-carrier-sampled", - roles=("victim",), - slots=1, - exclusive=False, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_native_runtime_carrier_and_sampling", - ), - ), - ScenarioDefinition( - selector="runtime-carrier-unsampled", - roles=("victim",), - slots=1, - exclusive=False, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_native_runtime_carrier_and_sampling", - ), - ), - ScenarioDefinition( - selector="otel-warm-lifecycle", - roles=( - "victim", - "warm-1", - "warm-2", - "warm-2-resume", - "warm-3", - "warm-4", - "warm", - "warm-resume", - ), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_warm_process_cleanup_and_replay", - ), - ), - ScenarioDefinition( - selector="otel-concurrent", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_concurrent_worker_attribution_and_peer_end", - ), - roles_at_two=("victim", "peer"), - slots_at_two=2, - ), - ), - ), - Requirement( - number=8, - description="Exporter failure recovery and held-export settlement", - contract="In both OTel views and both native configurations, separately observe an exporter failure followed by a healthy invocation and an intentionally held export. Record provider/exporter ownership and all export outcomes. Successful wrapper/OTel cleanup cannot precede the held export settlement; driver release does not turn an earlier failure or incomplete collection into a pass.", - baseline=(), - instrumented=( - ScenarioDefinition( - selector="export-error-recovery", - roles=("victim", "warm"), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_export_failure_then_healthy_invocation", - ), - ), - ScenarioDefinition( - selector="held-export", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "exportMs"), - source_tests=( - "lmi_tests/tests/e2e/test_observability.py::test_held_export_must_settle_before_successful_cleanup", - ), - ), - ), - ), - Requirement( - number=9, - description="Early completion closes losing branch scopes", - contract="For parallel, map and nested public operations configured to finish after a winning branch, observe the winner and controlled losing user work separately. The supported wrapper and branch cleanup boundaries must satisfy the declared cleanup budget before the driver releases the fault. Record actual task and request dispatch observations; final success alone cannot prove cleanup.", - baseline=( - ScenarioDefinition( - selector="early-completion-parallel", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", - ), - ), - ScenarioDefinition( - selector="early-completion-map", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", - ), - ), - ScenarioDefinition( - selector="early-completion-nested", - roles=("victim",), - slots=1, - exclusive=True, - budget_names=("observationGraceMs", "cleanupMs"), - source_tests=( - "lmi_tests/tests/e2e/test_lifecycle.py::test_early_completion_reclaims_losing_branches", - ), - ), - ), - instrumented=(), - ), -) - - -def requirements(capability: str = "lifecycle-otel") -> dict[str, dict[str, Any]]: - if capability not in ("lifecycle", "lifecycle-otel"): - raise ValueError("Unknown Python lifecycle capability") - result = {} - for item in _REQUIREMENTS: - definitions = item.baseline + ( - item.instrumented if capability == "lifecycle-otel" else () - ) - if not definitions: - continue - identity = f"python-lmi-{item.number}" - result[identity] = { - "id": identity, - "description": item.description, - "contract": item.contract, - "mandatory": True, - "capability": "lifecycle" if item.baseline else "otel", - "runtimes": ["python"], - "requiredScenarios": list( - dict.fromkeys(case.selector for case in definitions) - ), - } - return result - - -def required_cases(capability: str = "lifecycle-otel") -> tuple[CaseRequirement, ...]: - requirements(capability) - cases = [] - for item in _REQUIREMENTS: - groups: tuple[ - tuple[tuple[ScenarioDefinition, ...], tuple[str | None, ...]], ... - ] = ((item.baseline, (None,)),) - if capability == "lifecycle-otel": - groups += ((item.instrumented, ("invocation", "execution")),) - for definitions, views in groups: - for case in definitions: - for native in (1, 2): - roles = ( - case.roles_at_two - if native == 2 and case.roles_at_two is not None - else case.roles - ) - slots = ( - case.slots_at_two - if native == 2 and case.slots_at_two is not None - else case.slots - ) - for view in views: - cases.append( - CaseRequirement( - requirement_id=f"python-lmi-{item.number}", - scenario=case.selector, - native_concurrency=native, - otel_view=view, - roles=roles, - exclusive=case.exclusive, - minimum_native_concurrency=slots, - source_tests=case.source_tests, - budget_names=case.budget_names, - ) - ) - return tuple(cases) - - -def validate(evidence: Mapping[str, Any], *, phase: str = "final") -> list[str]: - from aws_durable_execution_conformance_tests.lmi.evidence import ( - validate_common_evidence, - ) - from lmi_tests.python_lifecycle import validate_lifecycle_regressions - - if phase not in ("observations", "final"): - return ["INCOMPLETE: unknown Python lifecycle validation phase"] - if ( - not isinstance(evidence, Mapping) - or evidence.get("coverageProfile") != "python-lifecycle" - ): - return [ - "INCOMPLETE: Python lifecycle validation requires its SDK-owned coverage profile" - ] - if evidence.get("runtime") != "python": - return ["INCOMPLETE: Python lifecycle contracts apply only to the Python SDK"] - profile = evidence.get("profile") - if not isinstance(profile, Mapping) or profile.get("recoveryPolicy") not in ( - "reuse-worker", - "replace-worker", - ): - return [ - "INCOMPLETE: Python lifecycle evidence requires an explicit supported recoveryPolicy" - ] - errors = validate_common_evidence( - evidence, phase=phase, requirement_ids=tuple(requirements()) - ) - errors.extend(validate_lifecycle_regressions(evidence, phase=phase)) - if evidence.get("otelView") is not None: - from lmi_tests.python_otel import validate_lifecycle_telemetry - - errors.extend(validate_lifecycle_telemetry(evidence, phase=phase)) - return errors - - -def suite_definition(capability: str = "lifecycle-otel") -> SuiteDefinition: - return SuiteDefinition( - coverage_profile="python-lifecycle", - suite_name="python-lmi", - requirements=requirements(capability), - cases=required_cases(capability), - validator=validate, - evidence_policy_key="regression", - nonnegative_budgets=frozenset( - {"maxThreadGrowth", "maxFdGrowth", "maxRssGrowthBytes"} - ), - ) - - -def load_manifest(path: Path) -> dict[str, Any]: - from aws_durable_execution_conformance_tests.lmi.app import ( - load_manifest as load_shared_manifest, - ) - - manifest = json.loads(path.read_text()) - if not isinstance(manifest, Mapping) or manifest.get("runtime") != "python": - raise ValueError("Python lifecycle manifests require runtime python") - if manifest.get("coverageProfile") != "python-lifecycle": - raise ValueError( - "Python lifecycle manifests require the SDK-owned coverage profile" - ) - profiles = manifest.get("profiles") - if isinstance(profiles, list): - for profile in profiles: - if not isinstance(profile, Mapping) or profile.get( - "recovery_policy" - ) not in ("reuse-worker", "replace-worker"): - raise ValueError( - "Python lifecycle profiles require an explicit recovery_policy" - ) - capability = manifest.get("capabilityProfile") - if not isinstance(capability, str): - raise ValueError( - "Python lifecycle manifests require an explicit capabilityProfile" - ) - return load_shared_manifest(path, suite=suite_definition(capability)) diff --git a/lmi_tests/python_lifecycle.py b/lmi_tests/python_lifecycle.py deleted file mode 100644 index a002db787..000000000 --- a/lmi_tests/python_lifecycle.py +++ /dev/null @@ -1,1224 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""Python SDK LMI lifecycle, replay, deadline and settled-scope regressions.""" - -from __future__ import annotations - -import math -from collections.abc import Mapping -from itertools import pairwise -from typing import Any - -SCENARIOS = { - "native-concurrency": 1, - "deadline-recovery": 4, - "replay-callback-retry": 5, - "warm-lifecycle": 6, - "early-completion-parallel": 9, - "early-completion-map": 9, - "early-completion-nested": 9, - "checkpoint-settlement": 10, - "timeout-retry": 11, - "late-operation": 12, - "return-inflight": 13, - "failure-inflight": 13, - "pending-cleanup": 14, - "nested-progress": 15, -} -OTEL_SCENARIOS = { - "runtime-carrier-sampled": 7, - "runtime-carrier-unsampled": 7, - "otel-warm-lifecycle": 7, - "otel-concurrent": 7, - "export-error-recovery": 8, - "held-export": 8, -} -ACTIVITY = { - "body", - "effect", - "checkpoint_call", - "checkpoint_ack", - "checkpoint_exit", - "poll_call", - "poll_exit", -} -ENDINGS = {"wrapper_return", "wrapper_raise"} - - -class _Observations: - def __init__(self, evidence: Mapping[str, Any]) -> None: - self.evidence = evidence - self.errors: list[str] = [] - self.config = self.mapping(evidence.get("regression"), "regression") - self.profile = self.mapping(evidence.get("profile"), "profile") - self.attempts = { - self.string(item.get("requestId"), "attempt.requestId"): item - for item in self.records(evidence.get("attempts"), "attempts") - } - self.events = self.records(evidence.get("events"), "events") - self.histories = self.records(evidence.get("histories"), "histories") - self.selected = self.mapping( - evidence.get("selectedRequests"), "selectedRequests" - ) - self.end = self.number( - evidence.get("observationEndedAtMs"), "observationEndedAtMs" - ) - - def fail(self, message: str, category: str = "ASSERTION") -> None: - self.errors.append(f"{category}: {message}") - - def mapping(self, value: Any, path: str) -> Mapping[str, Any]: - if not isinstance(value, Mapping): - self.fail(f"{path} must be an object", "INCOMPLETE") - return {} - return value - - def records(self, value: Any, path: str) -> list[Mapping[str, Any]]: - if not isinstance(value, list): - self.fail(f"{path} must be an array", "INCOMPLETE") - return [] - return [self.mapping(item, path) for item in value] - - def string(self, value: Any, path: str) -> str: - if not isinstance(value, str) or not value: - self.fail(f"{path} must be a non-empty string", "INCOMPLETE") - return "" - return value - - def number(self, value: Any, path: str) -> float: - if type(value) not in (int, float) or not math.isfinite(value) or value < 0: - self.fail(f"{path} must be a finite non-negative number", "INCOMPLETE") - return 0 - return float(value) - - def budget(self, name: str) -> float: - budgets = self.mapping(self.config.get("budgets"), "regression.budgets") - value = self.number(budgets.get(name), f"regression.budgets.{name}") - if value <= 0: - self.fail(f"regression.{name} must be positive", "INCOMPLETE") - return value - - def role(self, role: str) -> Mapping[str, Any]: - request_id = self.selected.get(role) - if not isinstance(request_id, str) or request_id not in self.attempts: - self.fail(f"selected original {role} request is missing", "INCOMPLETE") - return {} - return self.attempts[request_id] - - def select( - self, - kind: str | set[str], - request: Mapping[str, Any] | None = None, - **fields: Any, - ) -> list[Mapping[str, Any]]: - kinds = {kind} if isinstance(kind, str) else kind - return [ - item - for item in self.events - if item.get("kind") in kinds - and (request is None or item.get("requestId") == request.get("requestId")) - and all(item.get(key) == value for key, value in fields.items()) - ] - - def one( - self, kind: str | set[str], request: Mapping[str, Any], **fields: Any - ) -> Mapping[str, Any]: - matches = self.select(kind, request, **fields) - if len(matches) != 1: - self.fail( - f"request {request.get('requestId')} requires exactly one {kind} observation {fields}", - "INCOMPLETE", - ) - return {} - return matches[0] - - def ordered(self, *events: Mapping[str, Any]) -> bool: - return all( - first.get("requestId") == second.get("requestId") - and first.get("sequence", -1) < second.get("sequence", -1) - and first.get("atMs", -1) <= second.get("atMs", -1) - for first, second in pairwise(events) - ) - - def history(self, request: Mapping[str, Any]) -> Mapping[str, Any]: - matches = [ - item - for item in self.histories - if item.get("executionArn") == request.get("executionArn") - ] - if len(matches) != 1: - self.fail( - "one collected history is required for each selected execution", - "INCOMPLETE", - ) - return {} - return matches[0] - - def operation( - self, request: Mapping[str, Any], name: str, status: str | None = None - ) -> Mapping[str, Any]: - matches = [ - item - for item in self.records( - self.history(request).get("operations"), "history.operations" - ) - if item.get("name") == name - ] - if len(matches) != 1 or ( - status is not None and matches[0].get("status") != status - ): - self.fail( - f"collected history needs one {status or ''} {name} operation", - "INCOMPLETE", - ) - return {} - return matches[0] - - def resources(self, event: Mapping[str, Any]) -> Mapping[str, Any]: - resources = self.mapping(event.get("resources"), "wrapper.resources") - threads = self.records(resources.get("threads"), "wrapper.resources.threads") - ids = [] - for thread in threads: - ids.append(self.string(thread.get("id"), "thread.id")) - self.string(thread.get("name"), "thread.name") - if len(ids) != len(set(ids)): - self.fail("duplicate thread identities in resource snapshot", "INCOMPLETE") - for counter in ("fds", "rssBytes"): - self.number(resources.get(counter), f"wrapper.resources.{counter}") - return resources - - def lifecycle(self) -> None: - """Validate every request, including unselected/resumed attempts.""" - if self.evidence.get("runtime") != "python": - self.fail("Python lifecycle coverage requires runtime python", "INCOMPLETE") - grace = self.budget("observationGraceMs") - if not self.attempts or not self.events: - self.fail("lifecycle attempts and events must not be empty", "INCOMPLETE") - per_request: dict[str, list[Mapping[str, Any]]] = {} - for event in self.events: - request_id = event.get("requestId") - if not isinstance(request_id, str) or request_id not in self.attempts: - self.fail("event has no retained invocation owner", "INCOMPLETE") - continue - if event.get("ownerRequestId") != request_id or event.get( - "executionArn" - ) != self.attempts[request_id].get("executionArn"): - self.fail("lifecycle event belongs to another request or execution") - if type(event.get("sequence")) is not int or event["sequence"] < 0: - self.fail("event.sequence must be a non-negative integer", "INCOMPLETE") - when = self.number(event.get("atMs"), "event.atMs") - if when > self.end or when < self.attempts[request_id].get( - "startedAtMs", 0 - ): - self.fail( - "event lies outside retained observation window", "INCOMPLETE" - ) - per_request.setdefault(request_id, []).append(event) - if event.get("kind") in ("escape", "control_error"): - self.fail( - "emergency escape or failed fault control cannot establish SDK cleanup" - ) - for request_id, request in self.attempts.items(): - self.string(request.get("marker"), "attempt.marker") - events = sorted( - per_request.get(request_id, []), - key=lambda item: item.get("sequence", -1), - ) - if len({item.get("sequence") for item in events}) != len(events) or any( - a.get("atMs", 0) > b.get("atMs", 0) for a, b in pairwise(events) - ): - self.fail( - "lifecycle sequence is duplicated or contradicts the normalized clock", - "INCOMPLETE", - ) - entry = self.one("wrapper_enter", request) - baseline = self.resources(entry) - if request.get("outcome") == "TIMED_OUT": - self.one("platform_timeout", request, source="platform") - exits = self.select(ENDINGS, request) - if not exits: - if request.get("outcome") != "TIMED_OUT": - self.fail( - "non-timeout invocation has no actual wrapper exit", - "INCOMPLETE", - ) - retirement = self.select("worker_retired", request, source="platform") - task_exit = self.select("task_exit", request, taskId="root") - if not retirement and not task_exit: - self.fail( - "timed-out request has neither actual root exit nor platform worker retirement", - "INCOMPLETE", - ) - if ( - retirement - and self.profile.get("recoveryPolicy") != "replace-worker" - ): - self.fail( - "worker retirement requires the declared runtime replacement policy" - ) - continue - end = self.one(ENDINGS, request) - wrapper_outcome = end.get("outcome") - if wrapper_outcome not in ("SUCCEEDED", "FAILED", "PENDING", "TIMED_OUT"): - self.fail( - "actual wrapper outcome is missing or unrecognized", "INCOMPLETE" - ) - elif request.get( - "outcome" - ) != "TIMED_OUT" and wrapper_outcome != request.get("outcome"): - self.fail("actual wrapper outcome differs from its original request") - # A platform timeout does not stop the process. A later SDK return - # or exception is a separate boundary, still subject to settlement - # and stale-activity checks below. - user_exit = self.one("user_exit", request) - if not self.ordered(entry, user_exit, end): - self.fail("wrapper exited before actual user-scope exit") - if self.end < end.get("atMs", 0) + grace: - self.fail( - "collection ended before the declared post-wrapper observation window", - "INCOMPLETE", - ) - resources = self.resources(end) - baseline_ids = {thread.get("id") for thread in baseline.get("threads", [])} - for thread in resources.get("threads", []): - if thread.get("id") not in baseline_ids and ( - str(thread.get("name", "")).startswith( - ("dex-", "ThreadPoolExecutor") - ) - or thread.get("ownerRequestId") == request_id - ): - self.fail( - "invocation-owned worker survived the actual wrapper exit" - ) - if any( - item.get("kind") in ACTIVITY - and item.get("sequence", -1) > end.get("sequence", -1) - for item in events - ): - self.fail( - "user activity or request settlement continued after wrapper exit" - ) - for start_kind, end_kind, identity in ( - ("task_start", "task_exit", "taskId"), - ("io_start", "io_exit", "taskId"), - ("checkpoint_call", "checkpoint_exit", "callId"), - ("poll_call", "poll_exit", "callId"), - ): - for start in self.select(start_kind, request): - token = self.string(start.get(identity), f"{start_kind}.{identity}") - settled = self.one(end_kind, request, **{identity: token}) - if not self.ordered(start, settled, end): - self.fail(f"{start_kind} remained unsettled at wrapper exit") - for effect in self.select("body", request): - name = self.string(effect.get("operation"), "body.operation") - # Static names may repeat in distinct child contexts. Bind to - # service identity rather than requiring globally unique names. - operations = self.records( - self.history(request).get("operations"), "history.operations" - ) - matches = [ - item - for item in operations - if item.get("id") == effect.get("operationId") - and item.get("name") == name - ] - if len(matches) != 1: - self.fail("body observation changed its service operation identity") - - def overlap( - self, - roles: list[str], - *, - environment: str | None = None, - gate_ids: Mapping[str, str] | None = None, - ) -> list[Mapping[str, Any]]: - requests = [self.role(role) for role in roles] - if not requests or any(not request for request in requests): - return requests - n = len(requests) - for field in ("requestId", "executionArn", "processId", "pid"): - if len({request.get(field) for request in requests}) != n: - self.fail( - f"native cohort lacks distinct {field} identities", "PLACEMENT" - ) - if any( - type(request.get("pid")) is not int or request["pid"] <= 0 - for request in requests - ): - self.fail("actual runtime PID evidence is required", "INCOMPLETE") - environments = {request.get("environmentId") for request in requests} - if len(environments) != 1 or ( - environment is not None and environments != {environment} - ): - self.fail("native cohort reached an unrelated environment", "PLACEMENT") - # A fixture may also hold a handshake gate. Only its explicitly - # selected work gate establishes this role's cohort participation. - # Roles without a declared selector still require an unambiguous gate. - blocked = [ - self.one( - "blocked", - request, - **({"gateId": gate_ids[role]} if gate_ids and role in gate_ids else {}), - ) - for role, request in zip(roles, requests, strict=True) - ] - pivot = max(item.get("atMs", 0) for item in blocked) - for request, start in zip(requests, blocked, strict=True): - alive = self.select( - {"alive", "effect"}, request, gateId=start.get("gateId") - ) - if ( - not any(item.get("atMs", -1) >= pivot for item in alive) - or request.get("endedAtMs", 0) <= pivot - ): - self.fail( - "native overlap lacks positive continued liveness", "PLACEMENT" - ) - if any( - item.get("atMs", 0) <= pivot - for item in self.select("io_exit", request, gateId=start.get("gateId")) - ): - self.fail( - "native capacity stopped before overlap was established", - "PLACEMENT", - ) - return requests - - def gate(self, request: Mapping[str, Any]) -> str: - gate_id = self.string(self.config.get("gateId"), "regression.gateId") - faults = self.records(self.evidence.get("faultsCreated"), "faultsCreated") - if not any( - item.get("id") == gate_id - and item.get("ownerCaseId") == self.evidence.get("caseId") - for item in faults - ): - self.fail("fault gate lacks retained case ownership", "INCOMPLETE") - self.one("blocked", request, gateId=gate_id) - return gate_id - - def released(self, gate_id: str) -> float | None: - releases = self.select("fault_released", gateId=gate_id) - retirement = self.mapping(self.evidence.get("retirement", {}), "retirement") - releases += [ - item - for item in self.records(retirement.get("faults", []), "retirement.faults") - if item.get("id") == gate_id - ] - times = [] - for release in releases: - if release.get("mode") != "normal": - self.fail( - "emergency or unclassified fault release cannot establish the lifecycle assertion" - ) - times.append( - self.number( - release.get("atMs", release.get("releasedAtMs")), - "fault release time", - ) - ) - return min(times) if times else None - - def held_loser(self, request: Mapping[str, Any]) -> tuple[str, Mapping[str, Any]]: - gate_id = self.gate(request) - ready = self.one("winner_ready", request) - selected = self.one("winner_selected", request) - if not self.ordered(ready, selected): - self.fail("parent winner selection preceded readiness") - if not any( - self.ordered(item, ready) - for item in self.select("effect", request, gateId=gate_id) - ) or any( - self.ordered(item, ready) - for item in self.select("io_exit", request, gateId=gate_id) - ): - self.fail( - "losing branch was not actually held when its winner became ready", - "INCOMPLETE", - ) - return gate_id, selected - - def early(self) -> None: - victim = self.role("victim") - gate_id, winner = self.held_loser(victim) - cutoff = winner.get("atMs", 0) + self.budget("cleanupMs") - end = self.one(ENDINGS, victim) - if end.get("atMs", self.end) > cutoff: - self.fail( - "early completion left the wrapper pinned beyond the cleanup budget" - ) - if self.end < cutoff + self.budget("observationGraceMs"): - self.fail( - "early-completion collection ended before the cleanup assertion window", - "INCOMPLETE", - ) - if any(item.get("atMs", 0) > cutoff for item in self.select(ACTIVITY, victim)): - self.fail( - "abandoned scope continued effects or request activity beyond its cleanup budget" - ) - released = self.released(gate_id) - if released is not None and released <= cutoff: - self.fail( - "driver release supplied the losing-branch cleanup being asserted" - ) - names = { - "early-completion-parallel": ("race-parallel",), - "early-completion-map": ("race-map",), - "early-completion-nested": ("outer-map", "nested-child", "race-parallel"), - } - for name in names[self.evidence["scenario"]]: - self.operation(victim, name, "SUCCEEDED") - - def checkpoint(self) -> None: - victim = self.role("victim") - gate_id = self.gate(victim) - held = self.one("ack_held", victim, gateId=gate_id) - blocked = self.one("blocked", victim, gateId=gate_id) - call_id = self.string(held.get("callId"), "ack_held.callId") - call = self.one("checkpoint_call", victim, callId=call_id) - ack = self.one("checkpoint_ack", victim, callId=call_id, source="service") - exited = self.one("checkpoint_exit", victim, callId=call_id) - end = self.one(ENDINGS, victim) - operation = self.operation(victim, "checkpoint-loser", "SUCCEEDED") - pool = self.operation(victim, "checkpoint-pool", "SUCCEEDED") - joined = self.one("branch_join", victim, scopeId=pool.get("id")) - if held.get("operationId") != operation.get("id") or operation.get( - "id" - ) not in ack.get("operationIds", []): - self.fail( - "held acknowledgment does not belong to the selected real checkpoint" - ) - release = self.released(gate_id) - if not self.ordered(call, ack, held, blocked, exited, joined, end): - self.fail( - "real checkpoint acknowledgment/settlement did not precede branch join and wrapper return" - ) - if release is None or not blocked.get("atMs", 0) < release <= exited.get( - "atMs", 0 - ): - self.fail( - "checkpoint did not stay held until the scenario's normal release" - ) - - def deadline_measurement_end( - self, gate_id: str, deadline: float, cutoff: float, grace: float - ) -> float: - if "deadlineMeasurement" not in self.config: - return self.end - prior_errors = len(self.errors) - measurement = self.mapping( - self.config["deadlineMeasurement"], "deadline measurement" - ) - if measurement.get("selectedRequests") != self.selected: - self.fail( - "deadline measurement changed the selected original requests", - "INCOMPLETE", - ) - for field, expected in ( - ("deadlineAtMs", deadline), - ("cleanupCutoffAtMs", cutoff), - ("observationCutoffAtMs", cutoff + grace), - ): - actual = self.number( - measurement.get(field), f"deadline measurement.{field}" - ) - if not math.isclose(actual, expected, rel_tol=0, abs_tol=1): - self.fail( - "deadline measurement differs from the runtime deadline or declared budgets", - "INCOMPLETE", - ) - observed = self.number( - measurement.get("observedAtMs"), "deadline measurement.observedAtMs" - ) - if not cutoff + grace <= observed <= self.end: - self.fail( - "deadline measurement does not cover its complete original observation window", - "INCOMPLETE", - ) - uri = measurement.get("sourceEvidenceUri") - collection = self.mapping(self.evidence.get("collection"), "collection") - sources = collection.get("evidenceUris") - if ( - not isinstance(uri, str) - or not uri - or not isinstance(sources, list) - or uri not in sources - ): - self.fail( - "deadline measurement has no retained source evidence", "INCOMPLETE" - ) - faults = [ - fault - for fault in self.records( - self.evidence.get("faultsCreated"), "faultsCreated" - ) - if fault.get("id") == gate_id - and fault.get("ownerCaseId") == self.evidence.get("caseId") - ] - requested = self.end - if len(faults) != 1: - self.fail( - "deadline measurement must retain its original fault gate", "INCOMPLETE" - ) - else: - requested = self.number( - faults[0].get("releaseRequestedAtMs"), - "deadline measurement.releaseRequestedAtMs", - ) - acknowledged = self.number( - faults[0].get("releaseAcknowledgedAtMs"), - "deadline measurement.releaseAcknowledgedAtMs", - ) - receipt = self.number( - faults[0].get("releasedAtMs"), "deadline measurement.releasedAtMs" - ) - earliest_release = self.released(gate_id) - if ( - not observed <= requested <= acknowledged <= self.end - or receipt < requested - or (earliest_release is not None and earliest_release < requested) - ): - self.fail( - "deadline measurement contradicts the controller request or release evidence", - "INCOMPLETE", - ) - # Observe the capture-to-release gap too; invalid snapshots never - # shorten validation of the full observation tail. - return requested if len(self.errors) == prior_errors else self.end - - def late_deadline_activity( - self, victim: Mapping[str, Any], cutoff: float, released_after: float - ) -> bool: - for event in self.select(ACTIVITY, victim): - when = event.get("atMs", 0) - if when <= cutoff: - continue - # Release cannot authorize new user bodies or SDK dispatch. Only - # completion of previously dispatched I/O is cleanup activity. - if when <= released_after or event.get("kind") in { - "body", - "effect", - "checkpoint_call", - "poll_call", - }: - return True - start_kind = { - "checkpoint_ack": "checkpoint_call", - "checkpoint_exit": "checkpoint_call", - "poll_exit": "poll_call", - }.get(event["kind"]) - if start_kind is not None: - calls = self.select(start_kind, victim, callId=event.get("callId")) - if len(calls) != 1 or calls[0].get("atMs", when) > when: - return True - return False - - def timeout(self, victim: Mapping[str, Any]) -> float: - timeout = self.one("platform_timeout", victim, source="platform") - blocked = self.one("blocked", victim) - observed = self.one("runtime_deadline", victim, source="runtime") - remaining = self.number( - observed.get("remainingTimeMs"), "runtime_deadline.remainingTimeMs" - ) - configured = self.number(self.profile.get("timeoutMs"), "profile.timeoutMs") - deadline = self.number(victim.get("deadlineAtMs"), "attempt.deadlineAtMs") - observed_at = self.number(observed.get("atMs"), "runtime_deadline.atMs") - if not 0 < remaining <= configured: - self.fail( - "runtime remaining time must be positive and within the configured timeout", - "INCOMPLETE", - ) - # A runtime deadline can precede wrapper-entry + configured timeout: - # native dispatch/setup has already consumed some invocation lifetime. - # One millisecond allows canonical timestamp rounding, not extra cleanup. - if deadline <= observed_at or not math.isclose( - deadline, observed_at + remaining, rel_tol=0, abs_tol=1 - ): - self.fail( - "attempt deadline does not match its observed runtime remaining time", - "INCOMPLETE", - ) - if ( - victim.get("outcome") != "TIMED_OUT" - or timeout.get("atMs", 0) < deadline - or blocked.get("atMs", deadline) >= deadline - ): - self.fail( - "fixture did not cross the selected request's actual platform deadline with work held" - ) - cutoff = deadline + self.budget("cleanupMs") - grace = self.budget("observationGraceMs") - if self.end < cutoff + grace: - self.fail( - "timeout collection ended before the stale-activity observation window", - "INCOMPLETE", - ) - gate_id = self.string(blocked.get("gateId"), "blocked.gateId") - measured_until = self.deadline_measurement_end(gate_id, deadline, cutoff, grace) - if self.late_deadline_activity(victim, cutoff, measured_until): - self.fail( - "original timed-out request continued side effects or checkpoint activity" - ) - released = self.released(gate_id) - if released is not None and released <= cutoff: - self.fail("fault release supplied the timeout cleanup being asserted") - return deadline - - def recovery(self) -> None: - victim = self.role("victim") - policy = self.profile.get("recoveryPolicy") - if policy not in ("reuse-worker", "replace-worker"): - self.fail( - "deadline recovery requires an explicit supported recoveryPolicy", - "INCOMPLETE", - ) - deadline = self.timeout(victim) - native = self.profile.get("nativeConcurrency") - if native == 2: - self.overlap(["victim", "peer"]) - peer = self.role("peer") - if peer.get("outcome") != "SUCCEEDED": - self.fail("healthy peer did not complete successfully") - roles = ["recovery"] + (["recovery-peer"] if native == 2 else []) - requests = self.overlap(roles, environment=victim.get("environmentId")) - bound = deadline + self.budget("recoveryMs") - release = self.released(self.one("blocked", victim).get("gateId", "")) - recovered = max( - (self.one("blocked", request).get("atMs", 0) for request in requests), - default=self.end, - ) - if release is not None and release <= recovered: - self.fail("driver release supplied the capacity recovery being asserted") - for request in requests: - blocked = self.one("blocked", request) - if ( - not deadline <= blocked.get("atMs", -1) <= bound - or request.get("outcome") != "SUCCEEDED" - ): - self.fail( - "full original-environment capacity did not recover within its declared budget" - ) - if self.profile.get("recoveryPolicy") == "reuse-worker" and not any( - request.get("processId") == victim.get("processId") - and request.get("workerId") == victim.get("workerId") - for request in requests - ): - self.fail("full-capacity recovery omitted the original worker", "PLACEMENT") - if self.profile.get("recoveryPolicy") == "replace-worker" and not any( - request.get("replacesWorkerId") == victim.get("workerId") - for request in requests - ): - self.fail( - "replacement capacity did not identify the retired original worker", - "PLACEMENT", - ) - - def replay(self, execution_arn: str) -> None: - requests = [ - item - for item in self.attempts.values() - if item.get("executionArn") == execution_arn - ] - pending = [item for item in requests if item.get("outcome") == "PENDING"] - completed = [item for item in requests if item.get("outcome") == "SUCCEEDED"] - if ( - len(requests) < 2 - or not pending - or not completed - or not any( - first.get("endedAtMs", 0) < last.get("startedAtMs", 0) - for first in pending - for last in completed - ) - ): - self.fail( - "replay needs a real PENDING request followed by a service resume of the same execution" - ) - if not requests: - return - original = min(requests, key=lambda item: item.get("startedAtMs", 0)) - marker = self.string(original.get("marker"), "attempt.marker") - if any(item.get("marker") != marker for item in requests): - self.fail("replay input marker changed between invocations") - ids = {item["requestId"] for item in requests} - for name, terminal in (("success", "SUCCEEDED"), ("failure", "FAILED")): - operation = self.operation(original, name, terminal) - bodies = [ - item - for item in self.select("body", operation=name) - if item.get("requestId") in ids - ] - if ( - len(bodies) != 1 - or bodies[0].get("value") != marker - or bodies[0].get("operationId") != operation.get("id") - ): - self.fail( - f"completed {name} body was repeated, omitted or misattributed" - ) - failures = [ - item - for item in self.select("stored_failure") - if item.get("requestId") in ids - ] - expected_error = {"type": "ValueError", "message": "expected:" + marker} - if len({item.get("requestId") for item in failures}) < 2 or any( - item.get("error") != expected_error for item in failures - ): - self.fail( - "stored failure was not rethrown with the original meaning on replay" - ) - retry = self.operation(original, "retry", "SUCCEEDED") - bodies = sorted( - [ - item - for item in self.select("body", operation="retry") - if item.get("requestId") in ids - ], - key=lambda item: item.get("atMs", 0), - ) - if ( - any(type(item.get("attempt")) is not int for item in bodies) - or [item.get("attempt") for item in bodies] != [1, 2] - or any( - item.get("value") != marker - or item.get("operationId") != retry.get("id") - for item in bodies - ) - ): - self.fail( - "retry fixture did not execute exactly its two configured attempts" - ) - operations = self.records( - self.history(original).get("operations"), "history.operations" - ) - for kind in ("WAIT", "CALLBACK", "CONTEXT"): - if not any( - item.get("type") == kind and item.get("status") == "SUCCEEDED" - for item in operations - ): - self.fail( - f"real replay history lacks a successful {kind} operation", - "INCOMPLETE", - ) - - def service_retry(self) -> None: - victim, retry = self.role("victim"), self.role("retry") - deadline = self.timeout(victim) - timeout = self.one("platform_timeout", victim, source="platform") - if victim.get("executionArn") != retry.get("executionArn") or retry.get( - "startedAtMs", 0 - ) <= timeout.get("atMs", deadline): - self.fail( - "timeout retry must be a later platform request of the same durable execution" - ) - resumed = self.one("service_retry", retry) - if ( - resumed.get("previousRequestId") != victim.get("requestId") - or resumed.get("source") != "platform" - ): - self.fail( - "driver-created invocation is not evidence of a service timeout retry" - ) - if retry.get("startedAtMs", self.end) > deadline + self.budget("retryMs"): - self.fail("service timeout retry exceeded its declared observation budget") - success = self.operation(victim, "success", "SUCCEEDED") - ids = { - item["requestId"] - for item in self.attempts.values() - if item.get("executionArn") == victim.get("executionArn") - } - bodies = [ - item - for item in self.select("body", operation="success") - if item.get("requestId") in ids - ] - if ( - len(bodies) != 1 - or bodies[0].get("requestId") != victim.get("requestId") - or bodies[0].get("operationId") != success.get("id") - or bodies[0].get("value") != victim.get("marker") - ): - self.fail( - "completed success was repeated or lost across the timeout service retry" - ) - - def peer_progress( - self, victim: Mapping[str, Any], boundary: Mapping[str, Any], gate_id: str - ) -> None: - if self.profile.get("nativeConcurrency") != 2: - return - self.overlap(["victim", "peer"], gate_ids={"victim": gate_id}) - peer = self.role("peer") - progress = self.select({"alive", "peer_progress"}, peer) - if not any( - boundary.get("atMs", 0) <= item.get("atMs", -1) < victim.get("endedAtMs", 0) - for item in progress - ): - self.fail( - "healthy peer did not progress while victim cleanup remained held" - ) - - def scope_settlement(self, victim: Mapping[str, Any], gate_id: str) -> None: - blocked = self.one("blocked", victim, gateId=gate_id) - exited = self.one("io_exit", victim, gateId=gate_id) - end = self.one(ENDINGS, victim) - release = self.released(gate_id) - if ( - not self.ordered(blocked, exited, end) - or release is None - or not blocked.get("atMs", 0) < release <= exited.get("atMs", 0) - ): - self.fail( - "wrapper abandoned held invocation-owned work before normal settlement" - ) - - def inflight(self) -> None: - victim = self.role("victim") - gate_id, selected = self.held_loser(victim) - root = self.one("user_exit", victim) - exited = self.one("io_exit", victim, gateId=gate_id) - if not self.ordered(selected, root, exited): - self.fail( - "root completion was not observed with losing work still in flight" - ) - self.peer_progress(victim, root, gate_id) - self.scope_settlement(victim, gate_id) - status = ( - "FAILED" - if self.evidence.get("scenario") == "failure-inflight" - else "SUCCEEDED" - ) - if victim.get("outcome") != status: - self.fail("inflight root outcome changed during scope settlement") - if status == "FAILED" and self.history(victim).get("error") != { - "type": "ValueError", - "message": "expected:" + victim.get("marker", ""), - }: - self.fail("settling inflight work replaced the original root failure") - - def pending_cleanup(self) -> None: - victim, resume = self.role("victim"), self.role("resume") - gate_id = self.gate(victim) - blocked = self.one("blocked", victim, gateId=gate_id) - self.peer_progress(victim, blocked, gate_id) - self.scope_settlement(victim, gate_id) - enter, exited = ( - self.one("cleanup_enter", victim), - self.one("cleanup_exit", victim), - ) - end = self.one(ENDINGS, victim) - io_exit = self.one("io_exit", victim, gateId=gate_id) - user_exit = self.one("user_exit", victim) - if ( - not self.ordered(enter, blocked, io_exit, exited, user_exit, end) - or victim.get("outcome") != "PENDING" - ): - self.fail("PENDING preceded completion of the original root finally block") - if ( - resume.get("executionArn") != victim.get("executionArn") - or resume.get("startedAtMs", 0) <= end.get("atMs", 0) - or resume.get("outcome") != "SUCCEEDED" - ): - self.fail( - "cleanup fixture did not really resume the original suspended execution" - ) - bodies = [ - item - for item in self.select("body", operation="success") - if item.get("executionArn") == victim.get("executionArn") - ] - if len(bodies) != 1 or bodies[0].get("value") != victim.get("marker"): - self.fail( - "completed side effect repeated after finally cleanup and suspension" - ) - if not any( - item.get("type") == "WAIT" and item.get("status") == "SUCCEEDED" - for item in self.records( - self.history(victim).get("operations"), "history.operations" - ) - ): - self.fail("cleanup replay did not use a real durable wait", "INCOMPLETE") - - def late_operation(self) -> None: - victim = self.role("victim") - gate_id, selected = self.held_loser(victim) - self.scope_settlement(victim, gate_id) - attempted, rejected = ( - self.one("late_attempt", victim), - self.one("late_rejected", victim), - ) - closed = self.one("scope_closed", victim, scopeId=attempted.get("scopeId")) - operations = self.records( - self.history(victim).get("operations"), "history.operations" - ) - if not any( - item.get("id") == attempted.get("scopeId") and item.get("type") == "CONTEXT" - for item in operations - ): - self.fail( - "late attempt is not bound to a real completed child scope", - "INCOMPLETE", - ) - if not self.ordered(self.one("io_exit", victim, gateId=gate_id), attempted): - self.fail( - "late operation fixture did not follow settlement of the held losing step" - ) - if ( - not self.ordered(closed, attempted, rejected) - or not self.ordered(selected, attempted) - or rejected.get("scopeId") != attempted.get("scopeId") - or rejected.get("error") != {"type": "OrphanedChildException"} - ): - self.fail( - "completed child scope did not reject the attempted late durable operation" - ) - if self.select("late_accepted", victim) or self.select( - "body", victim, operation="late-work" - ): - self.fail("abandoned child executed or admitted late user work") - if any( - item.get("name") == "late-work" - for item in self.records( - self.history(victim).get("operations"), "history.operations" - ) - ): - self.fail("late child operation reached the durable service") - for request in self.records(self.evidence.get("sdkRequests"), "sdkRequests"): - if ( - request.get("requestId") == victim.get("requestId") - and request.get("operation") == "late-work" - and request.get("outcome") == "DISPATCHED" - ): - self.fail("late child operation dispatched a checkpoint") - - def nested(self) -> None: - roles = ["victim"] + ( - ["peer"] if self.profile.get("nativeConcurrency") == 2 else [] - ) - requests = self.overlap(roles) - budget = self.budget("progressMs") - for request in requests: - begin, end = ( - self.one("progress_begin", request), - self.one("progress", request), - ) - if ( - not self.ordered(begin, end) - or end.get("atMs", self.end) - begin.get("atMs", 0) > budget - ): - self.fail( - "nested public operations starved beyond their progress budget" - ) - marker = request.get("marker", "") - if end.get("value") != [[marker + ":0", marker + ":1"], marker]: - self.fail("nested results crossed invocation boundaries") - for name in ("outer-parallel", "inner-map"): - operation = self.operation(request, name, "SUCCEEDED") - configuration = self.one( - "operation_config", - request, - operationId=operation.get("id"), - source="public-fixture", - ) - if ( - type(configuration.get("maxConcurrency")) is not int - or configuration.get("maxConcurrency") != 1 - ): - self.fail( - "nested fixture did not exercise configured single-lane branch pools", - "INCOMPLETE", - ) - self.operation(request, "progress-child", "SUCCEEDED") - - def warm(self) -> None: - sequence = self.records( - self.config.get("warmSequence"), "regression.warmSequence" - ) - if [item.get("fixture") for item in sequence] != [ - "success", - "failure", - "replay", - ] * 3: - self.fail( - "warm lifecycle requires the nine success/failure/replay fixture executions", - "INCOMPLETE", - ) - arns = [item.get("executionArn") for item in sequence] - if len(set(arns)) != len(arns): - self.fail( - "warm sequence substituted an execution for a later fixture", - "INCOMPLETE", - ) - previous_end = -1.0 - samples: dict[tuple[Any, Any], list[Mapping[str, Any]]] = {} - for item in sequence: - requests = [ - request - for request in self.attempts.values() - if request.get("executionArn") == item.get("executionArn") - ] - if not requests: - self.fail("warm fixture has no retained requests", "INCOMPLETE") - continue - if ( - any( - request.get("fixture") != item.get("fixture") - for request in requests - ) - or min(request.get("startedAtMs", 0) for request in requests) - <= previous_end - ): - self.fail( - "warm fixture routing changed or executions were not sequential" - ) - previous_end = max(request.get("endedAtMs", 0) for request in requests) - history = self.history(requests[0]) - expected = "FAILED" if item.get("fixture") == "failure" else "SUCCEEDED" - if history.get("status") != expected: - self.fail("warm fixture execution has the wrong terminal result") - if item.get("fixture") == "replay": - self.replay(item["executionArn"]) - for request in requests: - for end in self.select(ENDINGS, request): - samples.setdefault( - (request.get("environmentId"), request.get("processId")), [] - ).append(end) - reused = [ - sorted(values, key=lambda item: item.get("atMs", 0)) - for values in samples.values() - if len(values) >= 3 - ] - if not reused: - self.fail( - "warm resources lack at least three observed uses of one original process", - "PLACEMENT", - ) - growth = self.mapping(self.config.get("budgets"), "regression.budgets") - allowances = { - counter: self.number(growth.get(key), f"regression.budgets.{key}") - for counter, key in ( - ("threads", "maxThreadGrowth"), - ("fds", "maxFdGrowth"), - ("rssBytes", "maxRssGrowthBytes"), - ) - } - for values in reused: - baseline = self.resources(values[1]) - for sample in values[2:]: - current = self.resources(sample) - for counter, allowance in allowances.items(): - before = ( - len(baseline.get("threads", [])) - if counter == "threads" - else baseline.get(counter, 0) - ) - after = ( - len(current.get("threads", [])) - if counter == "threads" - else current.get(counter, 0) - ) - if after > before + allowance: - self.fail( - f"warm process {counter} grew beyond its declared post-cache allowance" - ) - - -def _run( - evidence: Mapping[str, Any], check: str, execution_arn: str | None = None -) -> list[str]: - if not isinstance(evidence, Mapping): - return ["INCOMPLETE: lifecycle evidence must be an object"] - observations: _Observations | None = None - try: - observations = _Observations(evidence) - if check == "events": - observations.lifecycle() - elif check == "replay": - observations.replay(execution_arn or "") - else: - observations.lifecycle() - scenario = evidence.get("scenario") - if evidence.get("coverageProfile") != "python-lifecycle": - observations.fail( - "Python regression semantics require the explicit coverageProfile", - "INCOMPLETE", - ) - elif scenario in OTEL_SCENARIOS: - if ( - evidence.get("requirementId") - != f"python-lmi-{OTEL_SCENARIOS[scenario]}" - ): - observations.fail( - "OTel scenario does not match its lifecycle requirement", - "INCOMPLETE", - ) - # The OTel extension validates carriers/providers/exports. Keep - # invocation-owned lifecycle checks for these six cases too. - elif ( - scenario not in SCENARIOS - or evidence.get("requirementId") != f"python-lmi-{SCENARIOS[scenario]}" - ): - observations.fail( - "scenario does not match a Python lifecycle requirement", - "INCOMPLETE", - ) - elif not observations.errors: - native = observations.profile.get("nativeConcurrency") - if native not in (1, 2): - observations.fail( - "Python regression matrix requires native concurrency 1 or 2", - "INCOMPLETE", - ) - elif scenario == "native-concurrency": - observations.overlap(["victim"] + (["peer"] if native == 2 else [])) - elif scenario.startswith("early-completion-"): - observations.early() - else: - checks = { - "deadline-recovery": observations.recovery, - "warm-lifecycle": observations.warm, - "checkpoint-settlement": observations.checkpoint, - "timeout-retry": observations.service_retry, - "late-operation": observations.late_operation, - "return-inflight": observations.inflight, - "failure-inflight": observations.inflight, - "pending-cleanup": observations.pending_cleanup, - "nested-progress": observations.nested, - } - if scenario == "replay-callback-retry": - observations.replay( - observations.role("victim").get("executionArn", "") - ) - else: - checks[scenario]() - except (KeyError, TypeError, ValueError, OverflowError) as exc: - error = f"INCOMPLETE: malformed lifecycle regression evidence: {exc}" - return [*(observations.errors if observations else []), error] - return observations.errors - - -def validate_lifecycle_events(evidence: Mapping[str, Any]) -> list[str]: - """Check settled wrapper/task/request/resource lifetimes, including OTel cases.""" - return _run(evidence, "events") - - -def validate_replay_evidence( - evidence: Mapping[str, Any], execution_arn: str -) -> list[str]: - """Check real wait/callback/child replay and configured retries for one execution.""" - return _run(evidence, "replay", execution_arn) - - -def validate_lifecycle_regressions( - evidence: Mapping[str, Any], *, phase: str = "final" -) -> list[str]: - """Validate one coverage cell; core common/retirement validation also applies. - - Both phases preserve behavioral observations. Final validation also inspects - any subsequently recorded release, so emergency release never repairs an - earlier failed assertion. The caller must retain observation-phase errors. - """ - if phase not in ("observations", "final"): - return ["INCOMPLETE: unknown lifecycle regression validation phase"] - return _run(evidence, "scenario") diff --git a/lmi_tests/python_otel.py b/lmi_tests/python_otel.py deleted file mode 100644 index 01da69dc3..000000000 --- a/lmi_tests/python_otel.py +++ /dev/null @@ -1,740 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""Mandatory OTel observations for the Python LMI regression matrix. - -The SDK adapter records actual public-plugin boundaries, exporter attempts and -runtime carriers. Canonical spans still use the ordinary backend normalization -and assertion engine. This profile describes Python's supported fixtures; it -does not require other runtimes to allocate identical plugin objects. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from typing import Any - -from aws_durable_execution_conformance_tests_otel.chained_invoke import ( - execution_trace_context, -) -from aws_durable_execution_conformance_tests_otel.lmi import ( - _identifier, - _number, - _records, - _service_execution_starts, - _span, - _view_errors, -) -from aws_durable_execution_conformance_tests_otel.model import ( - Span, - TelemetryQuery, - Trace, -) -from aws_durable_execution_conformance_tests_otel.validators import validate_trace - -SCENARIOS = ( - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - "otel-warm-lifecycle", - "otel-concurrent", - "export-error-recovery", - "held-export", - "deadline-recovery", - "timeout-retry", - "pending-cleanup", - "late-operation", - "return-inflight", - "failure-inflight", -) -_TERMINAL_SCENARIOS = frozenset(SCENARIOS[:6]) - - -class _Telemetry: - def __init__(self, evidence: Mapping[str, Any]) -> None: - self.evidence = evidence - self.errors: list[str] = [] - self.scenario = evidence.get("scenario") - self.view_name = str(evidence.get("otelView", "")) - self.attempts: dict[str, Mapping[str, Any]] = {} - self.events: list[Mapping[str, Any]] = [] - self.enters: dict[str, Mapping[str, Any]] = {} - self.exits: dict[str, Mapping[str, Any]] = {} - self.boundaries: dict[str, Mapping[str, Any]] = {} - self.carriers: dict[str, Mapping[str, Any]] = {} - self.view: Mapping[str, Any] = {} - - def fail(self, detail: str, kind: str = "ASSERTION") -> None: - self.errors.append(f"{kind}: {detail}") - - def records(self, request_id: str, kind: str) -> list[Mapping[str, Any]]: - return [ - event - for event in self.events - if event.get("requestId") == request_id and event.get("kind") == kind - ] - - def one(self, request_id: str, kind: str) -> Mapping[str, Any]: - records = self.records(request_id, kind) - if len(records) != 1: - self.fail(f"{request_id} needs exactly one observed {kind}", "INCOMPLETE") - return {} - return records[0] - - def selected(self, role: str) -> str: - request_id = self.evidence.get("selectedRequests", {}).get(role) - if not isinstance(request_id, str) or request_id not in self.attempts: - raise ValueError(f"missing original selected {role} request") - return request_id - - def common(self) -> bool: - if ( - self.evidence.get("coverageProfile") != "python-lifecycle" - or self.evidence.get("runtime") != "python" - ): - self.fail( - "Python OTel regressions require coverageProfile=python-lifecycle and runtime=python", - "INCOMPLETE", - ) - return False - if self.scenario not in SCENARIOS or self.view_name not in ( - "invocation", - "execution", - ): - self.fail("unknown OTel regression scenario or view", "INCOMPLETE") - return False - if not self.evidence.get("artifact", {}).get("plugin"): - self.fail("exact plugin artifact identity is required", "INCOMPLETE") - for attempt in _records(self.evidence.get("attempts"), "attempts"): - request_id = attempt.get("requestId") - if ( - not isinstance(request_id, str) - or not request_id - or request_id in self.attempts - ): - raise ValueError("attempts need unique request identities") - self.attempts[request_id] = attempt - if not self.attempts: - raise ValueError("no observed native requests") - self.events = _records(self.evidence.get("events"), "events") - sequences: set[tuple[str, int]] = set() - for event in self.events: - request_id = event.get("requestId") - if ( - request_id not in self.attempts - or event.get("ownerRequestId") != request_id - ): - self.fail("event does not retain original request ownership") - continue - if event.get("executionArn") != self.attempts[request_id]["executionArn"]: - self.fail("event does not retain original execution ownership") - sequence = event.get("sequence") - if ( - type(sequence) is not int - or sequence < 1 - or (request_id, sequence) in sequences - ): - self.fail( - "events require unique positive per-request sequence numbers", - "INCOMPLETE", - ) - else: - sequences.add((request_id, sequence)) - if ( - not _number(self.attempts[request_id]["startedAtMs"]) - <= _number(event.get("atMs")) - <= _number(self.evidence.get("observationEndedAtMs")) - ): - self.fail( - "event lies outside its original observation window", "INCOMPLETE" - ) - telemetry = self.evidence.get("telemetry", {}) - views = telemetry.get("views", {}) - if ( - not isinstance(views, Mapping) - or set(views) != {self.view_name} - or not isinstance(views[self.view_name], Mapping) - ): - raise ValueError( - "the matrix cell must collect exactly its declared otelView" - ) - self.view = views[self.view_name] - for carrier in _records(self.view.get("carriers"), "carriers"): - request_id = carrier.get("requestId") - if request_id not in self.attempts or request_id in self.carriers: - raise ValueError("carriers need one record per observed request") - self.carriers[request_id] = carrier - if set(self.carriers) != set(self.attempts): - raise ValueError("carriers must include every original and resumed request") - return not self.errors - - def lifecycle(self) -> None: - budget = _number( - self.evidence.get("regression", {}).get("budgets", {}).get("exportMs") - ) - if budget <= 0: - self.fail( - "regression.budgets.exportMs must be declared before measurement", - "INCOMPLETE", - ) - plugin_instances: set[str] = set() - providers: dict[tuple[str, str], tuple[str, str]] = {} - for request_id, attempt in self.attempts.items(): - enter = self.one(request_id, "otel_enter") - end = self.one(request_id, "otel_exit") - factory = self.one(request_id, "otel_factory") - self.enters[request_id], self.exits[request_id] = enter, end - boundary = self.records(request_id, "wrapper_return") + self.records( - request_id, "wrapper_raise" - ) - if len(boundary) != 1: - self.fail( - f"{request_id} is missing the actual wrapper completion boundary", - "INCOMPLETE", - ) - else: - self.boundaries[request_id] = boundary[0] - if ( - not _number(enter.get("atMs")) - <= _number(factory.get("atMs")) - <= _number(boundary[0].get("atMs")) - <= _number(end.get("atMs")) - ): - self.fail( - f"{request_id} plugin diagnostics contradict the actual SDK wrapper boundary" - ) - before, after = enter.get("contextBefore"), end.get("contextAfter") - for snapshot in (before, after): - if not isinstance(snapshot, Mapping) or set(snapshot) != { - "traceId", - "spanId", - }: - self.fail( - "context restoration needs actual before/after trace/span snapshots", - "INCOMPLETE", - ) - elif snapshot["traceId"] is None and snapshot["spanId"] is None: - pass - else: - _identifier(snapshot["traceId"], 32) - _identifier(snapshot["spanId"], 16) - if before != after: - self.fail(f"{request_id} did not restore its prior OTel context") - if ( - type(end.get("exporterShutdowns")) is not int - or end["exporterShutdowns"] != 0 - ): - self.fail( - f"{request_id} shut down or failed to observe the shared exporter" - ) - instance = factory.get("pluginInstanceId") - if ( - not isinstance(instance, str) - or not instance - or instance in plugin_instances - ): - self.fail( - "Python plugin instances must be distinct per native invocation" - ) - else: - plugin_instances.add(instance) - if factory.get("pluginExecutionArn") != attempt["executionArn"]: - self.fail( - "plugin factory did not receive the original durable execution identity" - ) - provider, exporter = enter.get("providerId"), enter.get("exporterId") - if ( - not isinstance(provider, str) - or not provider - or not isinstance(exporter, str) - or not exporter - ): - self.fail( - "provider/exporter identities must be observed at entry", - "INCOMPLETE", - ) - continue - process = (attempt["environmentId"], attempt["processId"]) - if providers.setdefault(process, (provider, exporter)) != ( - provider, - exporter, - ): - self.fail("warm process replaced its shared provider/exporter") - - def spans(self) -> None: - expected_sampled = self.scenario != "runtime-carrier-unsampled" - if self.scenario in { - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - "otel-concurrent", - "held-export", - } and any( - attempt.get("outcome") != "SUCCEEDED" for attempt in self.attempts.values() - ): - self.fail( - f"{self.scenario} requires the original successful handler outcome" - ) - for request_id, carrier in self.carriers.items(): - if carrier.get("expectedSampled") is not expected_sampled: - self.fail( - f"{self.scenario} requires explicit Sampled={int(expected_sampled)} for every request" - ) - if self.scenario in ( - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - ): - actual = execution_trace_context( - {"TraceHeader": {"XAmznTraceId": carrier.get("header")}} - ) - caller = execution_trace_context( - {"TraceHeader": {"XAmznTraceId": carrier.get("callerHeader")}} - ) - if ( - actual.trace_id != caller.trace_id - or actual.sampled != caller.sampled - ): - self.fail( - f"{request_id} runtime carrier did not preserve the caller Root and explicit sampling" - ) - records = _records(self.view.get("spans"), "spans") - first_observation: dict[tuple[str, str], tuple[Any, Span]] = {} - occurrences: list[tuple[Any, Span]] = [] - observed: list[Mapping[str, Any]] = [] - for record in records: - span = _span(record.get("span")) - key = (span.trace_id, span.span_id) - occurrence = (record.get("requestId"), span) - if first_observation.setdefault(key, occurrence) != occurrence: - self.fail( - "duplicate span identity has inconsistent export observations" - ) - # Collapse identical transport receipts only. Preserve contradictory - # data and every native owner for attribution checks and diagnostics. - if occurrence not in occurrences: - occurrences.append(occurrence) - observed.append(record) - if record.get("sampled") is not True: - self.fail("recording exporter span has no sampled SpanContext") - # Preserve raw duplicates, but assert logical span identity/counts rather - # than promising exactly-once transport delivery. - assertions: dict[str, dict[str, Any]] = {} - histories = { - item["executionArn"]: item - for item in _records(self.evidence.get("histories"), "histories") - } - for arn in {attempt["executionArn"] for attempt in self.attempts.values()}: - requests = [ - request_id - for request_id, attempt in self.attempts.items() - if attempt["executionArn"] == arn - ] - expected: dict[str, Any] = { - "minimum_spans": 1, - "require_execution_correlation": True, - "allowed_execution_arns": [arn], - "span_assertions": [], - } - if expected_sampled: - expected["span_assertions"].append( - { - "select": {"name": "Invocation"}, - "count": len(requests), - "expect": {}, - } - ) - for request_id in requests: - invocations = [ - record - for record in observed - if record.get("requestId") == request_id - and record["span"]["name"] == "Invocation" - ] - if len(invocations) != int(expected_sampled): - self.fail( - f"{request_id} must emit exactly {int(expected_sampled)} logical Invocation span" - ) - if expected_sampled and self.scenario in _TERMINAL_SCENARIOS: - terminal = max( - requests, - key=lambda request_id: self.attempts[request_id]["endedAtMs"], - ) - status = histories.get(arn, {}).get("status") - if status not in ("SUCCEEDED", "FAILED"): - self.fail( - f"{arn} lacks a successful or failed terminal service history", - "INCOMPLETE", - ) - expected["span_assertions"].append( - { - "select": {"name": "Workflow"}, - "expect": { - "status": "OK" if status == "SUCCEEDED" else "ERROR" - }, - } - ) - workflows = [ - record - for record in observed - if record["span"]["name"] == "Workflow" - and self.attempts.get(record.get("requestId", ""), {}).get( - "executionArn" - ) - == arn - ] - if len(workflows) != 1 or workflows[0].get("requestId") != terminal: - self.fail( - f"{arn} must emit its Workflow once at the terminal invocation, not during replay" - ) - assertions[arn] = expected - selected = [ - _span(record["span"]) - for record in observed - if record.get("requestId") in requests - ] - if selected and expected_sampled: - query = TelemetryQuery( - arn, - selected[0].service_name or "", - min(span.start_time for span in selected), - max(span.end_time for span in selected), - ) - self.errors.extend( - f"ASSERTION: {error}" - for error in validate_trace( - Trace(selected[0].trace_id, tuple(selected)), expected, query - ) - ) - strict_view = {**self.view, "spans": observed, "assertions": assertions} - self.errors.extend( - _view_errors( - self.view_name, - strict_view, - self.attempts, - "python", - _number(self.evidence.get("observationEndedAtMs")), - execution_starts=_service_execution_starts( - self.evidence.get("histories", []) - ), - operation_histories=_records( - self.evidence.get("histories", []), "histories" - ), - ) - ) - outcomes = { - item["requestId"]: item.get("status") - for item in _records(self.view.get("exportOutcomes"), "exportOutcomes") - } - victim = self.evidence.get("selectedRequests", {}).get("victim") - for request_id in self.attempts: - expected_outcome = ( - "unsampled" - if not expected_sampled - else "export-error" - if self.scenario == "export-error-recovery" and request_id == victim - else "delivered" - ) - if outcomes.get(request_id) != expected_outcome: - self.fail( - f"{self.scenario}/{request_id} requires export outcome {expected_outcome}" - ) - exports = self.records(request_id, "otel_export") - starts = self.records(request_id, "otel_export_started") - if {(event.get("traceId"), event.get("spanId")) for event in starts} != { - (event.get("traceId"), event.get("spanId")) for event in exports - }: - self.fail( - f"{request_id} export start/completion observations disagree", - "INCOMPLETE", - ) - emitted = { - (record["span"]["trace_id"], record["span"]["span_id"]) - for record in records - if record.get("requestId") == request_id - } - if { - (event.get("traceId"), event.get("spanId")) for event in exports - } != emitted: - self.fail( - f"{request_id} export observations omit or add canonical spans", - "INCOMPLETE", - ) - expected_event = ( - "failure" - if expected_outcome == "export-error" - else "held" - if self.scenario == "held-export" - else "success" - ) - for event in exports: - begun = [ - start - for start in starts - if (start.get("traceId"), start.get("spanId")) - == (event.get("traceId"), event.get("spanId")) - ] - if len(begun) != 1: - self.fail( - f"{request_id} needs a unique observed exporter start", - "INCOMPLETE", - ) - elif ( - not 0 - <= _number(event.get("atMs")) - _number(begun[0].get("atMs")) - <= _number(self.evidence["regression"]["budgets"]["exportMs"]) - ): - self.fail( - f"{request_id} exporter exceeded its immutable exportMs budget" - ) - if event.get("outcome") != expected_event: - self.fail( - f"{request_id} exporter outcome differs from the selected fixture" - ) - if ( - not _number(self.enters[request_id].get("atMs")) - <= _number(event.get("atMs")) - <= _number(self.boundaries[request_id].get("atMs")) - ): - self.fail( - f"{request_id} exporter activity escaped its OTel lifetime" - ) - - def warm(self) -> None: - sequence = _records( - self.evidence.get("regression", {}).get("warmSequence"), - "regression.warmSequence", - ) - if [item.get("fixture") for item in sequence] != [ - "success", - "failure", - "replay", - ] * 2: - self.fail( - "OTel warm coverage requires success/failure/replay repeated twice", - "INCOMPLETE", - ) - return - arns = [item.get("executionArn") for item in sequence] - if len(set(arns)) != 6 or set(arns) != { - attempt["executionArn"] for attempt in self.attempts.values() - }: - self.fail( - "warm sequence must identify all six distinct actual executions", - "INCOMPLETE", - ) - return - previous_end = -1.0 - reused: set[tuple[str, str]] = set() - saw_reuse = False - for item in sequence: - attempts = sorted( - ( - attempt - for attempt in self.attempts.values() - if attempt["executionArn"] == item["executionArn"] - ), - key=lambda attempt: attempt["startedAtMs"], - ) - if not attempts or any( - attempt.get("fixture") != item["fixture"] for attempt in attempts - ): - self.fail( - "warm fixtures must match observed invocation routing", "INCOMPLETE" - ) - continue - if _number(attempts[0]["startedAtMs"]) <= previous_end: - self.fail("warm executions were not sequential") - previous_end = _number(attempts[-1]["endedAtMs"]) - key = (attempts[0]["environmentId"], attempts[0]["processId"]) - saw_reuse |= key in reused - reused.add(key) - expected = "FAILED" if item["fixture"] == "failure" else "SUCCEEDED" - history: Mapping[str, Any] = next( - ( - history - for history in self.evidence["histories"] - if history["executionArn"] == item["executionArn"] - ), - {}, - ) - if attempts[-1]["outcome"] != expected or history.get("status") != expected: - self.fail( - f"warm {item['fixture']} has the wrong terminal invocation outcome" - ) - if item["fixture"] == "replay": - from lmi_tests.python_lifecycle import validate_replay_evidence - - self.errors.extend( - validate_replay_evidence(self.evidence, item["executionArn"]) - ) - if not saw_reuse: - self.fail("warm fixtures never reused an identified process", "PLACEMENT") - - def concurrency(self) -> None: - capacity = self.evidence["profile"]["nativeConcurrency"] - originals = [ - attempt for attempt in self.attempts.values() if attempt.get("attempt") == 1 - ] - if len(originals) != capacity or len(self.attempts) != capacity: - self.fail( - "concurrent telemetry must occupy exactly the configured native cohort", - "PLACEMENT", - ) - return - if ( - len({attempt["environmentId"] for attempt in originals}) != 1 - or len({attempt["processId"] for attempt in originals}) != capacity - ): - self.fail( - "concurrent Python requests need distinct processes in one original environment", - "PLACEMENT", - ) - ordered = sorted( - originals, - key=lambda attempt: self.exits[attempt["requestId"]].get("atMs", 0), - ) - for index, attempt in enumerate(ordered): - request_id = attempt["requestId"] - blocked = self.one(request_id, "blocked") - release = self.one(request_id, "control_released") - if blocked.get("gateId") != release.get("gateId") or not blocked.get( - "gateId" - ): - self.fail("cohort control release must identify its held gate") - if ( - not _number(blocked.get("atMs")) - < _number(release.get("atMs")) - <= _number(self.exits[request_id].get("atMs")) - ): - self.fail("invocation ended before its own barrier was released") - for peer in ordered[index + 1 :]: - peer_id = peer["requestId"] - boundary = _number(self.exits[request_id].get("atMs")) - peer_release = self.one(peer_id, "control_released") - if ( - not _number(self.enters[peer_id].get("atMs")) - < boundary - < _number(peer_release.get("atMs")) - ): - self.fail( - "ending one invocation did not leave the established peer held" - ) - if not any( - boundary - < _number(event.get("atMs")) - < _number(peer_release.get("atMs")) - for event in self.records(peer_id, "alive") - ): - self.fail( - "held peer made no observed progress after the other invocation ended" - ) - - def export_recovery(self) -> None: - victim, healthy = self.selected("victim"), self.selected("warm") - if any( - self.attempts[victim][key] != self.attempts[healthy][key] - for key in ("environmentId", "processId", "workerId") - ): - self.fail( - "export recovery must reuse the original worker/process", "PLACEMENT" - ) - if _number(self.attempts[healthy]["startedAtMs"]) <= _number( - self.exits[victim].get("atMs") - ): - self.fail("healthy export did not occur after the failed invocation ended") - if ( - self.attempts[victim].get("outcome") != "SUCCEEDED" - or self.attempts[healthy].get("outcome") != "SUCCEEDED" - ): - self.fail( - "export fixture failure must not change either successful handler outcome" - ) - - def held(self) -> None: - victim = self.selected("victim") - held = self.one(victim, "export_held") - release = self.one(victim, "export_released") - if not held.get("gateId") or held.get("gateId") != release.get("gateId"): - self.fail("held export and release must identify the same owned gate") - faults = _records(self.evidence.get("faultsCreated"), "faultsCreated") - owned = [ - fault - for fault in faults - if fault.get("id") == held.get("gateId") - and fault.get("ownerCaseId") == self.evidence.get("caseId") - ] - if ( - len(owned) != 1 - or owned[0].get("releasedAtMs") != release.get("atMs") - or release.get("mode") != "normal" - ): - self.fail( - "held-export release lacks case-owned normal measurement evidence", - "INCOMPLETE", - ) - if ( - not _number(held.get("atMs")) - < _number(release.get("atMs")) - <= _number(self.boundaries[victim].get("atMs")) - ): - self.fail( - "held export did not prevent SDK wrapper completion until normal release" - ) - markers = { - (event.get("traceId"), event.get("spanId")) - for event in self.records(victim, "otel_export") - } - if (held.get("traceId"), held.get("spanId")) not in markers: - self.fail( - "held export was not bound to an observed exporter span", "INCOMPLETE" - ) - else: - key = (held.get("traceId"), held.get("spanId")) - started = [ - event - for event in self.records(victim, "otel_export_started") - if (event.get("traceId"), event.get("spanId")) == key - ] - completed = [ - event - for event in self.records(victim, "otel_export") - if (event.get("traceId"), event.get("spanId")) == key - ] - if ( - len(started) != 1 - or not completed - or not _number(started[0].get("atMs")) - <= _number(held.get("atMs")) - < _number(release.get("atMs")) - <= min(_number(event.get("atMs")) for event in completed) - ): - self.fail("held-export gate was not inside the actual exporter call") - - -def validate_lifecycle_telemetry( - evidence: Mapping[str, Any], *, phase: str = "final" -) -> list[str]: - """Validate one mandatory view/configuration cell before releasing faults. - - Call the core lifecycle regression validator as well. Both phases preserve - observation checks; final resource retirement remains the core's concern. - """ - if phase not in ("observations", "final"): - return ["INCOMPLETE: unknown validation phase"] - validator = _Telemetry(evidence) - try: - if not validator.common(): - return validator.errors - validator.lifecycle() - if validator.errors: - return validator.errors - validator.spans() - if validator.scenario == "otel-warm-lifecycle": - validator.warm() - elif validator.scenario == "otel-concurrent": - validator.concurrency() - elif validator.scenario == "export-error-recovery": - validator.export_recovery() - elif validator.scenario == "held-export": - validator.held() - except (AttributeError, KeyError, TypeError, ValueError, OverflowError) as error: - validator.fail(f"malformed OTel regression evidence: {error}", "INCOMPLETE") - return validator.errors diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index da94d45ed..71d9e3496 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -1,86 +1,28 @@ -# Python-owned LMI regression suite - -## Problem - -Python-specific lifecycle requirements were mixed into the language-neutral -conformance catalog. That made Python exception, thread, cleanup and plugin -rules look like shared SDK requirements and coupled their evolution to the shared -package release. - -## Cause - -The implementation combined two separate responsibilities: reusable execution, -coordination and evidence handling, and the SDK-specific rules defining what a -Python fixture must do. - -## Options considered - -1. Keep a Python-only profile in the shared catalog. This leaves the ownership - problem in place. -2. Copy the whole coordinator and reporter into the SDK. This duplicates the - lease, journal, cancellation and retirement machinery. -3. Keep Python definitions and validators in this repository, using the shared - runner through its programmatic suite interface. - -## Chosen design - -Use option 3. The Python SDK owns `python_catalog.py`, `python_lifecycle.py` and -`python_otel.py`, their regression tests, and the corresponding evidence rules. -The local `python-lifecycle` suite has 28 baseline lifecycle cells and 48 OTel -cells. It reports `python-lmi-*` IDs under the `python-lmi` report namespace. -These results do not claim acceptance of shared conformance requirements. - -The shared repository owns only the language-neutral `standard` requirements -1–8 and `export-lifecycle` requirement 16. Its exporter profile runs 24 cells -across both modes, both views and native c1/c2. The Python workflow continues to -run those 24 shared cells alongside the 76 SDK-owned cells, using the same two -persistent functions. The separate shared standard profile is not implicitly -accepted by this combination. - -`SuiteDefinition` supplies local requirement metadata, the exact matrix and the -local validator to the common programmatic runner. It does not register a new -profile in the shared CLI or package Python-specific requirement resources in -shared wheels. The common runner still owns bounded scheduling, immutable -manifest snapshots, exact launch journals, common evidence checks, resource -leases, independent retirement auditing and strict JSON/JUnit reporting. - -Python-specific applicability, recovery policies and nonnegative resource-growth -budgets are declared in the local suite. Both OTel views and native c1/c2 remain -independent required cells. Missing capabilities, missing evidence and SDK bugs -stay failures; migration does not remove assertions or make the SDK implement a -capability it lacks. - -## Evidence and cleanup - -The adapter uses real public SDK hooks and real service history. It preserves -actual service operation timestamps and checkpoint results, immutable invocation -identities, captured SDK boundaries, exporter ownership and raw observations. -The validator selects the fixture's declared work gate separately from its -winner-admission handshake. Execution-view historical spans require matching -service-operation evidence. Contradictory span records remain failures. - -A persisted deadline measurement records the original cohort and bounds before -normal fault release. The full event ledger remains available; new user work and -SDK dispatch after the cutoff cannot be excused as cleanup. Completion of already -dispatched I/O is accounted for separately. - -Failed phases can retain partial observations before release. A separate final -snapshot may prove physical retirement without changing the original failure. -Every actual launch and owned resource must be accounted for; unknown requests, -missing completion or unpublished observations retain quarantine. Ownership uses -conditional S3 writes outside the expiring run prefix, with private storage and -native IAM/SigV4 authentication. - -## Integration and removed coupling - -CI automatically runs on trusted same-repository PRs through CodeBuild and keeps -manual dispatch. Fork and Dependabot PRs only run the unprivileged harness. The -conformance dependency is pinned to an exact reviewed revision in -`shared_contract.py` and the workflow. Setup can install that dependency before -importing any SDK-owned suite that uses it. - -Python-only shared requirement files, catalog branches, validators and their -self-tests are moved out of the shared repository. Their behavior is retained -here under local IDs. Existing SDK-local regression entry points remain available. -The driver-only catalogs and validators are not included in the Lambda bundle. -No production SDK fix or real-LMI acceptance is implied by local harness tests. +# Shared lifecycle suite ownership + +The Python adapter consumes the seven standard lifecycle requirements pinned by +`shared_contract.py`. Each YAML file in the shared repository has one `scenario`. +Normal return is requirement 2; normal failure is requirement 7. There is no +Python-owned replacement catalog, OTel profile or exporter matrix in this PR. + +The shared repository owns requirement descriptions, roles, minimum native +concurrency, scheduling, behavior validators and strict result aggregation. The +SDK repository maps each scenario to its real public API fixture and retains +source observations. `make_manifest` reads the shared matrix, uses c1 for +single-worker cases and c2 for overlap/recovery, and preserves every required role. + +The same shared runner handles admission, original launch identity, immutable +case observations and independent retirement auditing. The adapter never replaces +a failed original with a retry, translates a runtime deadline into invented +service evidence, or clears a resource merely because cleanup was attempted. + +The deployed instrumentation uses the public SDK observation hooks plus a +transparent service-client wrapper. It records operations and user-code entries; +it does not change production SDK behavior. Admission probes expose actual SDK +rejections, acceptance and pending work. Local harness tests validate that these +outcomes reach the shared validator accurately; they do not claim real-LMI +acceptance or fix the underlying SDK lifecycle behavior. + +The core-only Lambda bundle excludes driver code and conformance packages. CI +retains the dedicated test-account boundary, trusted-PR condition, exact dependency +pin, conditional ownership lease and final reconciliation path. diff --git a/lmi_tests/shared-telemetry.md b/lmi_tests/shared-telemetry.md deleted file mode 100644 index 355c6f502..000000000 --- a/lmi_tests/shared-telemetry.md +++ /dev/null @@ -1,80 +0,0 @@ -# Shared telemetry observations - -The LMI fixture uses the real public invocation/execution OTel factories and -providers. `ObservedPlugin` forwards all seven lifecycle hooks and propagation -metadata, binding the invocation owner captured when that actual plugin was -created. An export snapshots that owner and the actual completed span data when -the configured processor accepts the work. Background export never falls back -to whichever invocation happens to be current later. - -Process-owned runtimes are keyed by `(otelView, exportMode)`. Selecting a variant -does not mutate environment variables, a shared plugin configuration or another -invocation's client. Providers remain alive across invocation completion and warm -reuse. Completed owners, export records and flush records are pruned after their -observations are published; the process registry does not retain past traces or -their S3 clients. - -## Export modes - -`sync` uses the standard `SimpleSpanProcessor`. Its exporter runs on the caller -of `span.end()`, so a held export prevents that call and the SDK wrapper from -returning until the gate is released. - -`async-bounded-flush` queues the same exporter work with its captured owner. The -processor's configured wait is `DEFAULT_FLUSH_TIMEOUT_MS = 5000`, capped by the -timeout supplied through the real `TracerProvider.force_flush()` API. The shared -manifest separately declares a 10-second acceptance budget, allowing observation -and scheduling overhead. Neither value is a product SLA. Tests can inject a -shorter processor wait through the private runtime constructor; invocation input -cannot mutate the process-owned variant's timeout. - -A bounded wait returning does not settle its exports. The runtime records -`flush_started`, `flush_wait_completed` and actual `flush_completed` independently, -including the actual configured/effective timeout and pending export IDs. The -original invocation continues to own asynchronous work after wrapper return. -Success and failure are observed from actual exporter results, not inferred from -an elapsed timeout. - -`Trace.gate()` still returns `None`, preserving existing checkpoint serialization. -Its private `gate_release()` observer reports the first actual held poll and the -normal or emergency release receipt. Only calls that actually waited emit -canonical `export_held`/`export_released` pairs. Later exports after an open gate -remain in the ledger without being falsely labeled as held. Shared case aborts -take precedence over individual gate releases; legacy control order is unchanged. - -## Capture, publication and retirement - -`Trace.capture_observation()` reserves a per-request `captureSequence` and actual -boundary timestamp using a small lock independent of S3 publication. The SDK -return boundary is captured before resource sampling and post-call diagnostics. -`publish_observation()` retains that capture while the original `sourceSequence` -records publication order. The shared assembler sorts by actual time and capture -sequence; it does not turn delayed log arrival into delayed SDK completion. - -Canonical telemetry observations are published on a process-owned worker so -diagnostic S3 latency cannot extend the SDK's bounded flush wait. Publication is -still owned work: failures remain outstanding and cannot produce quiescence. -Legacy uppercase OTel records remain available to the existing SDK-local tests. - -`telemetry.active_work(trace)` and `runtime.pending_for_case(case_id)` report actual -pending export/flush IDs, publication count and publication errors. A canonical -`export_quiescence` signal is emitted only after invocation closure, actual work -completion and publication of all earlier observations. The controller must also -observe the actual wrapper and lifecycle retirement; the telemetry signal alone -does not settle a durable execution or authorize a deployment update. - -## Driver conversion - -`shared_telemetry.collect_telemetry(raw_events, attempts, *, view, -expected_sampled, caller_headers, observed_until_ms, export_mode=None)` converts -actual ledger spans with the pinned shared canonical model. Caller headers and -sampling expectations come from the admitted driver's Invoke configuration; -runtime carriers come from the observed process environment. Source sequence -gaps, missing wrapper/quiescence observations and unknown export owners remain -incomplete. The converter never fills in a missing export or completion. - -Only `telemetry.py`, `shared_exporter.py` and the lifecycle observation helpers run -inside Lambda. `shared_telemetry.py`, the shared conformance packages and the -scenario coordinator run on the driver. Local tests exercise public SDK handlers, -the real local durable service, both OTel views and both export modes. They do not -establish real-LMI placement or claim to fix the runtime defects tracked by #741. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 60648fe63..12db36cb9 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run Python LMI regressions and shared exporter conformance.""" +"""Run the seven shared LMI lifecycle requirements.""" from __future__ import annotations @@ -20,20 +20,7 @@ from lmi_tests.shared_lock import OwnershipError, RunLease -COVERAGE_PROFILES = ("python-lifecycle", "export-lifecycle") -BEHAVIOR_BUDGETS = { - "observationGraceMs": 1000, - "cleanupMs": 5000, - "recoveryMs": 5000, - "retryMs": 40000, - "progressMs": 15000, - "maxThreadGrowth": 1, - "maxFdGrowth": 8, - "maxRssGrowthBytes": 32 * 1024 * 1024, - "exportMs": 30000, - "exportCompletionMs": 30000, - "flushWaitMs": 10000, -} +COVERAGE_PROFILES = ("standard",) def verify_checkout(checkout: Path) -> None: @@ -62,8 +49,6 @@ def setup(checkout: Path) -> None: "install", "--editable", str(checkout / "packages/aws-durable-execution-conformance-tests"), - "--editable", - str(checkout / "packages/aws-durable-execution-conformance-tests-otel"), ], check=True, ) @@ -72,9 +57,8 @@ def setup(checkout: Path) -> None: def verify_install(checkout: Path) -> None: verify_checkout(checkout) import aws_durable_execution_conformance_tests as core - import aws_durable_execution_conformance_tests_otel as otel - for module in (core, otel): + for module in (core,): if not Path(module.__file__).resolve().is_relative_to(checkout.resolve()): raise ValueError( "Installed conformance packages are not from the pinned checkout; run shared setup" @@ -85,7 +69,6 @@ def make_manifest( deployment: Mapping[str, Any], coverage: str, directory: Path ) -> dict[str, Any]: from aws_durable_execution_conformance_tests.lmi.matrix import required_cases - from lmi_tests import python_catalog if coverage not in COVERAGE_PROFILES: raise ValueError("Unknown Python adapter coverage profile") @@ -116,27 +99,24 @@ def make_manifest( for key in ("c1", "c2") ] scenarios = [] - required_cells = ( - python_catalog.required_cases("lifecycle-otel") - if coverage == "python-lifecycle" - else required_cases("python", "lifecycle-otel", coverage) - ) - for required in required_cells: - case_id = f"{coverage}-{required.scenario}-c{required.native_concurrency}-{required.otel_view or 'baseline'}-{required.export_mode or 'sdk-default'}" + for required in required_cases("python", "lifecycle", coverage): + concurrency = required.minimum_native_concurrency + case_id = f"{required.requirement_id}-{required.scenario}" scenarios.append( { "case_id": case_id, "cohort_id": "cohort-1", "requirement_id": required.requirement_id, - "profile_id": f"c{required.native_concurrency}", + "profile_id": f"c{concurrency}", "scenario": required.scenario, - "otel_view": required.otel_view, - "export_mode": required.export_mode, "roles": list(required.roles), - "slots": required.minimum_native_concurrency, + "slots": concurrency, "exclusive": required.exclusive, "behavior_budgets": { - name: BEHAVIOR_BUDGETS[name] for name in required.budget_names + "observationGraceMs": 1000, + "admissionMs": 5000, + "cleanupMs": 5000, + "recoveryMs": 15000, }, } ) @@ -146,7 +126,7 @@ def make_manifest( "runtime": "python", "region": actual["region"], "coverageProfile": coverage, - "capabilityProfile": "lifecycle-otel", + "capabilityProfile": "lifecycle", "scenarioConcurrency": 8, "profiles": profiles, "scenarios": scenarios, @@ -168,7 +148,6 @@ def make_manifest( def run(deployment: Mapping[str, Any], directory: Path) -> int: - from lmi_tests import python_catalog from aws_durable_execution_conformance_tests.lmi.app import ( build_report, @@ -198,27 +177,17 @@ def run(deployment: Mapping[str, Any], directory: Path) -> int: plan = make_manifest(deployment, coverage, output) path = output / "manifest.json" save(path, plan) - suite = ( - python_catalog.suite_definition(plan["capabilityProfile"]) - if coverage == "python-lifecycle" - else None - ) - validated = ( - python_catalog.load_manifest(path) - if suite is not None - else load_manifest(path) - ) + validated = load_manifest(path) results = asyncio.run( live_results( validated, "lmi_tests.shared_adapter:create_adapter", audit_dir=output, - suite=suite, ) ) - report = build_report(validated, results, suite=suite) + report = build_report(validated, results) save(output / "evidence.json", results) - save(output / "coverage.json", coverage_inventory(validated, suite=suite)) + save(output / "coverage.json", coverage_inventory(validated)) print(render_console(report)) for format_name in ("json", "junit"): write_report(report, format_name, str(output / "report")) @@ -386,7 +355,7 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: "runId": deployment["run"], "errors": errors, "status": "OWNERSHIP_RETAINED", - "reason": "Fault release and Stop do not prove task/export quiescence; retained run requires reconciliation", + "reason": "Fault release and Stop do not prove task/request quiescence; retained run requires reconciliation", }, ) return 1 diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py index 9c67a9363..660cf434f 100644 --- a/lmi_tests/shared_adapter.py +++ b/lmi_tests/shared_adapter.py @@ -236,7 +236,9 @@ def retire() -> bool: context.evidence["retirement"] = final.get("retirement", {}) # Fresh completion facts prove physical retirement separately; they # must never replace the case's pre-release behavior observations. - context.retirement_evidence = final + from lmi_tests.shared_evidence import retirement_snapshot + + context.retirement_evidence = retirement_snapshot(final) return complete return await case.call(retire) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py index a81bc766d..300f07c0a 100644 --- a/lmi_tests/shared_contract.py +++ b/lmi_tests/shared_contract.py @@ -10,7 +10,7 @@ CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" # Updated only with the matching reviewed shared evidence contract. -CONFORMANCE_REVISION = "59352e1097d92fe034c7cd55a9618837b95b905b" +CONFORMANCE_REVISION = "cc1eae152d9d0a49ecb777db140339f290b43567" def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py index 282a1825a..f7efcb19f 100644 --- a/lmi_tests/shared_evidence.py +++ b/lmi_tests/shared_evidence.py @@ -21,7 +21,7 @@ TERMINAL = {"SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"} -_ACTIVE_FAULTS = {"deadline-recovery", "timeout-retry"} +_ACTIVE_FAULTS = {"deadline-during-step", "peer-recovery"} _OPERATION_TYPES = { "Step": "STEP", "Wait": "WAIT", @@ -42,8 +42,18 @@ def _milliseconds(value: Any) -> float: def _decoded(value: Any) -> Any: + if isinstance(value, Mapping) and "Payload" in value: + if value.get("Truncated") is True: + raise ValueError( + "Truncated service data cannot establish checkpoint equality" + ) + value = value["Payload"] if isinstance(value, str): - return json.loads(value) + try: + return json.loads(value) + except json.JSONDecodeError: + # Callback payloads may be plain UTF-8, rather than JSON results. + return value return value @@ -128,7 +138,6 @@ def observations(self) -> tuple[list[dict[str, Any]], list[str]]: if event.get("kind") in { "observation_error", "control_error", - "export_publication_error", }: errors.append( str(event.get("message", event.get("error", event["kind"]))) @@ -234,6 +243,9 @@ def histories( operation["status"] = "TIMED_OUT" elif event_type.endswith("Stopped"): operation["status"] = "STOPPED" + for operation in operations.values(): + operation.pop("startedAtMs", None) + operation.pop("endedAtMs", None) result = self.cloud.execution_results[arn] started = result.get("StartTimestamp") if started is None: @@ -259,7 +271,7 @@ def histories( history["error"] = _error(result["Error"]) histories.append(history) fixture = item.get("payload", {}).get("scenario") - if fixture in {"failure", "failure-inflight"}: + if fixture == "failure": expected[arn] = { "status": "FAILED", "error": { @@ -372,33 +384,6 @@ def attempts( if journal[request]["attempt"] == 1: selected[role] = request attempts.append(attempt) - if ( - self.context.scenario.scenario == "timeout-retry" - and "victim" in selected - and "retry" in selected - ): - original = next( - item for item in attempts if item["requestId"] == selected["victim"] - ) - retry = next( - item for item in attempts if item["requestId"] == selected["retry"] - ) - if ( - original.get("outcome") == "TIMED_OUT" - and retry["executionArn"] == original["executionArn"] - ): - events.append( - { - "kind": "service_retry", - "requestId": retry["requestId"], - "ownerRequestId": retry["requestId"], - "executionArn": retry["executionArn"], - "atMs": retry["startedAtMs"], - "source": "platform", - "previousRequestId": original["requestId"], - "sourceEvidence": "single-original-Invoke-and-native-service-continuation", - } - ) return attempts, selected, errors @@ -416,7 +401,7 @@ def _causal_order(events: list[dict[str, Any]]) -> None: event["sequence"] = sequence -def _quiescent(request: str, events: list[dict[str, Any]], *, telemetry: bool) -> bool: +def _quiescent(request: str, events: list[dict[str, Any]]) -> bool: local = [event for event in events if event.get("requestId") == request] if not any( event.get("kind") in {"wrapper_return", "wrapper_raise"} for event in local @@ -432,75 +417,147 @@ def _quiescent(request: str, events: list[dict[str, Any]], *, telemetry: bool) - ("io_start", "io_exit", "taskId"), ("checkpoint_call", "checkpoint_exit", "callId"), ("poll_call", "poll_exit", "callId"), - ("export_started", "export_completed", "exportId"), - ("flush_started", "flush_completed", "flushId"), ): begun = Counter(event.get(key) for event in local if event.get("kind") == start) ended = Counter(event.get(key) for event in local if event.get("kind") == end) if begun - ended: return False - if telemetry and not any( - event.get("kind") == "export_quiescence" - and event.get("pendingExportIds") == [] - and event.get("pendingFlushIds") == [] - and event.get("publicationCount") == 0 - for event in local - ): - return False return not any( event.get("kind") in {"observation_error", "control_error"} for event in local ) -def _deadline_measurement( - context: ScenarioContext, cloud: Cloud, selected: Mapping[str, str] -) -> tuple[dict[str, Any] | None, list[str]]: - """Read the persisted pre-release measurement without discarding full events.""" - path = cloud.artifacts_dir / "measurements/deadline.json" - in_memory = getattr(cloud, "deadline_measurement", None) - if not path.exists(): - if in_memory is not None: - return None, ["Pre-release deadline measurement file is missing"] - return None, [] # Legacy sources retain the strict full-tail validator. - try: - recorded = json.loads(path.read_text()) - victim = next( - item for item in cloud.invocations if item.get("role") == "victim" - ) - expected = { - "runId": context.run_id, - "caseId": context.scenario.case_id, - "cohortId": context.scenario.cohort_id, - "scenario": context.scenario.scenario, - "executionArn": victim["arn"], - "selectedRequests": dict(selected), +def replay_evidence(events, histories, selected): + """Compare actual invocation-start checkpoints with final service history.""" + first_resume = next( + ( + event + for event in events + if event.get("kind") == "invocation_start" + and event.get("requestId") == selected.get("resume") + ), + {}, + ) + before = [] + for saved in first_resume.get("operations", []): + if saved.get("operationType") == "EXECUTION" or saved.get("status") not in { + "SUCCEEDED", + "FAILED", + }: + continue + operation = { + "id": saved["operationId"], + "name": saved.get("name"), + "type": saved["operationType"], + "parentId": saved.get("parentId"), + "status": saved["status"], } - if not isinstance(recorded, dict) or any( - recorded.get(key) != value for key, value in expected.items() - ): - raise ValueError( - "stored measurement belongs to another run, case or original cohort" - ) - fields = ( - "selectedRequests", - "deadlineAtMs", - "cleanupCutoffAtMs", - "observationCutoffAtMs", - "observedAtMs", + if saved["status"] == "SUCCEEDED": + operation["result"] = _decoded(saved.get("serializedResult")) + else: + operation["error"] = saved.get("error") + before.append(operation) + victim = next( + (e for e in events if e.get("requestId") == selected.get("victim")), {} + ) + after = next( + ( + h["operations"] + for h in histories + if h["executionArn"] == victim.get("executionArn") + ), + [], + ) + bodies = [e for e in events if e.get("kind") == "body"] + executions = [ + { + "requestId": e["requestId"], + "operationId": e["operationId"], + "attemptId": f"{e['requestId']}:{e['attempt']}", + } + for e in bodies + ] + retry = [e for e in bodies if e.get("operation") == "retry"] + attempts = [] + for body in retry: + ending = next( + ( + e + for e in events + if e.get("kind") == "task_exit" + and e.get("operationId") == body["operationId"] + and e.get("requestId") == body["requestId"] + and e.get("attempt") == body["attempt"] + ), + {}, ) - if in_memory is not None and ( - not isinstance(in_memory, Mapping) - or any(in_memory.get(key) != recorded.get(key) for key in fields) - ): - raise ValueError( - "in-memory and persisted measurement identities/bounds differ" - ) - return { - **{key: recorded[key] for key in fields}, - "sourceEvidenceUri": path.resolve().as_uri(), - }, [] - except (OSError, ValueError, KeyError, StopIteration, TypeError) as error: - return None, [f"Invalid pre-release deadline measurement: {error}"] + attempts.append( + { + "attemptId": f"{body['requestId']}:{body['attempt']}", + "requestId": body["requestId"], + "outcome": ending.get("outcome"), + } + ) + return { + "before": before, + "after": after, + "operationExecutions": executions, + "retry": { + "operationId": retry[0]["operationId"] if retry else "", + "attempts": attempts, + }, + "sideEffects": [ + { + "operationId": body["operationId"], + "count": sum(e["operationId"] == body["operationId"] for e in bodies), + "expectedCount": 2 if body.get("operation") == "retry" else 1, + } + for index, body in enumerate(bodies) + if body["operationId"] not in {e["operationId"] for e in bodies[:index]} + ], + } + + +def assertion_events(events, scenario, selected): + """Expose one scenario-owned task boundary while retaining every task fact. + + The shared behavior contract's task_exit is singular. Other real tasks stay + in the envelope with their source kind, and are restored for the separate + all-task retirement audit. + """ + if scenario not in { + "normal-return", + "normal-failure", + "deadline-during-step", + "peer-recovery", + "warm-reuse", + }: + return events + result = [] + for event in events: + item = dict(event) + target = ( + "late-callback" + if scenario == "warm-reuse" + and event.get("requestId") == selected.get("victim") + else "root" + ) + if item["kind"] in {"task_start", "task_exit"} and item.get("taskId") != target: + item["sourceKind"] = item["kind"] + item["kind"] = "observed_" + item["kind"] + result.append(item) + return result + + +def retirement_snapshot(evidence): + """Restore every observed task for the common independent retirement audit.""" + from copy import deepcopy + + snapshot = deepcopy(evidence) + for event in snapshot.get("events", []): + if event.get("kind") in {"observed_task_start", "observed_task_exit"}: + event["kind"] = event["sourceKind"] + return snapshot def build_evidence( @@ -518,14 +575,11 @@ def build_evidence( (cloud.retired_at if phase == "final" else cloud.observation_ended_at) or 0 ) * 1000 complete = bool(attempts) and all( - _quiescent( - item["requestId"], events, telemetry=context.scenario.otel_view is not None - ) - for item in attempts + _quiescent(item["requestId"], events) for item in attempts ) if not complete: errors.append( - "Actual wrapper/task/export publication has not established a complete observation window" + "Actual wrapper/task publication has not established a complete observation window" ) latest = max((event["atMs"] for event in events), default=0) if ended < latest: @@ -535,21 +589,10 @@ def build_evidence( if not errors else {} ) - if context.scenario.export_mode and not errors: - watermarks["exportLifecycle"] = ended artifact = { "sdk": context.profile.artifact_id, "conformance": cloud.manifest.get("conformanceRevision", "unverified"), } - if context.scenario.otel_view: - artifact["plugin"] = next( - ( - f"sha256:{digest}" - for name, digest in cloud.manifest.get("wheels", {}).items() - if "_otel-" in name - ), - "", - ) profile = { "functionArn": context.profile.resource_id, "configurationDigest": context.profile.configuration_id, @@ -571,7 +614,7 @@ def build_evidence( "profile": profile, "attempts": attempts, "selectedRequests": selected, - "events": events, + "events": assertion_events(events, context.scenario.scenario, selected), "histories": histories, "expectedHistories": expected, "observationEndedAtMs": ended, @@ -582,50 +625,9 @@ def build_evidence( }, "collectionErrors": errors, } - if context.scenario.otel_view: - evidence["otelView"] = context.scenario.otel_view - if context.scenario.export_mode: - evidence["exportMode"] = context.scenario.export_mode - evidence["exportLifecycle"] = { - "budgets": dict(context.scenario.behavior_budgets) - } - else: - evidence["regression"] = {"budgets": dict(context.scenario.behavior_budgets)} - victim = next( - (item for item in cloud.invocations if item.get("role") == "victim"), None - ) - if victim: - fixture = victim.get("payload", {}).get("scenario") - suffix = ( - "-checkpoint" - if fixture == "checkpoint" - else "-cleanup" - if fixture == "suspend-cleanup" - else "-loser" - ) - if fixture in { - "checkpoint", - "suspend-cleanup", - "parallel", - "map", - "nested", - "return-inflight", - "failure-inflight", - "late-operation", - }: - evidence["regression"]["gateId"] = victim["marker"] + suffix - if context.scenario.scenario in _ACTIVE_FAULTS: - measurement, measurement_errors = _deadline_measurement( - context, cloud, selected - ) - errors.extend(measurement_errors) - if measurement is not None: - evidence["regression"]["deadlineMeasurement"] = measurement - if context.scenario.scenario in {"warm-lifecycle", "otel-warm-lifecycle"}: - evidence["regression"]["warmSequence"] = [ - {"executionArn": item["arn"], "fixture": item["payload"]["scenario"]} - for item in cloud.invocations - ] + evidence["budgets"] = dict(context.scenario.behavior_budgets) + if context.scenario.scenario == "suspension-retry": + evidence["replay"] = replay_evidence(events, histories, selected) for event in events: if event.get("kind") in {"checkpoint_call", "poll_call"}: evidence["sdkRequests"].append( @@ -649,27 +651,28 @@ def build_evidence( {"kind": "execution", "id": arn, "ownerCaseId": context.scenario.case_id} for arn in dict.fromkeys(item["executionArn"] for item in attempts) ) - for kind in ("export", "flush"): - created.extend( - { - "kind": kind, - "id": event[f"{kind}Id"], - "ownerCaseId": context.scenario.case_id, - } - for event in events - if event.get("kind") == f"{kind}_started" - ) evidence["resourcesCreated"] = created + # Coordination barriers are not injected victim faults. Retain their full + # journal separately; releasing a healthy peer is not evidence of victim cleanup. + evidence["controlRecords"] = list(cloud.control_records) + victim_faults = { + item.get("payload", {}).get("gate") + for item in cloud.invocations + if item.get("role") == "victim" + and item.get("payload", {}).get("scenario") == "deadline" + } faults = [] for gate in dict.fromkeys( record["gate"] for record in cloud.control_records if record["action"] == "hold" ): + if gate not in victim_faults: + continue fault: dict[str, Any] = {"id": gate, "ownerCaseId": context.scenario.case_id} receipts = [ event["atMs"] for event in events if event.get("gateId") == gate - and event.get("kind") in {"fault_released", "export_released"} + and event.get("kind") in {"fault_released"} and event.get("mode") == "normal" ] releases = [ @@ -692,24 +695,6 @@ def build_evidence( fault.setdefault("releasedAtMs", release["time"] * 1000) faults.append(fault) evidence["faultsCreated"] = faults - if context.scenario.otel_view: - from lmi_tests.shared_telemetry import collect_telemetry - - headers = { - request: item["trace_header"] - for item in cloud.invocations - for request in item.get("native_requests", []) - if item.get("trace_header") - } - evidence["telemetry"] = collect_telemetry( - list(sources.raw), - attempts, - view=context.scenario.otel_view, - expected_sampled=context.scenario.scenario != "runtime-carrier-unsampled", - caller_headers=headers, - observed_until_ms=ended, - export_mode=context.scenario.export_mode or "sync", - ) if phase == "final": retired = (cloud.retired_at or 0) * 1000 settled = [] @@ -721,13 +706,7 @@ def build_evidence( } if resource["kind"] == "invocation": item["outcome"] = ( - "QUIESCENT" - if _quiescent( - resource["id"], - events, - telemetry=context.scenario.otel_view is not None, - ) - else "UNKNOWN" + "QUIESCENT" if _quiescent(resource["id"], events) else "UNKNOWN" ) elif resource["kind"] == "execution": item["outcome"] = cloud.execution_results.get(resource["id"], {}).get( @@ -757,7 +736,7 @@ def build_evidence( "mode": "emergency" if any( event.get("gateId") == fault["id"] - and event.get("kind") in {"fault_released", "export_released"} + and event.get("kind") in {"fault_released"} and event.get("mode") == "emergency" for event in events ) diff --git a/lmi_tests/shared_exporter.py b/lmi_tests/shared_exporter.py deleted file mode 100644 index f0740e222..000000000 --- a/lmi_tests/shared_exporter.py +++ /dev/null @@ -1,489 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Real processor/exporter fixtures with immutable invocation ownership. - -Only diagnostic publication is deferred in synchronous mode. Export itself -remains on SimpleSpanProcessor's caller. The asynchronous mode queues the same -exporter work and bounds the SDK's flush wait independently of its completion. -""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from concurrent.futures import ThreadPoolExecutor -from contextlib import contextmanager -from contextvars import ContextVar -from dataclasses import dataclass, field -import queue -import threading -import time -from typing import Any, Protocol -import uuid - -from opentelemetry.sdk.trace import ReadableSpan, SpanProcessor -from opentelemetry.sdk.trace.export import SpanExporter, SpanExportResult - - -class Trace(Protocol): - payload: Mapping[str, Any] - identity: Mapping[str, Any] - - def emit(self, phase: str, **fields: Any) -> Any: ... - - def observe(self, kind: str, **fields: Any) -> Any: ... - - def capture_observation(self, kind: str, **fields: Any) -> dict[str, Any]: ... - - def publish_observation(self, observation: Mapping[str, Any]) -> Any: ... - - def gate(self, name: str, **fields: Any) -> Any: ... - - def gate_release(self, name: str, **fields: Any) -> Any: ... - - -@dataclass(eq=False) -class Owner: - trace: Trace - tracker: Tracker - export_behavior: str - gate: str | None - shared: bool - case_id: str - request_id: str - exports: list[Export] = field(default_factory=list) - publications: int = 0 - closed: bool = False - quiescent: bool = False - publication_errors: list[str] = field(default_factory=list) - - -CURRENT_OWNER: ContextVar[Owner | None] = ContextVar("lmi_export_owner", default=None) - - -@contextmanager -def owned(owner: Owner): - token = CURRENT_OWNER.set(owner) - try: - yield - finally: - CURRENT_OWNER.reset(token) - - -@dataclass(eq=False) -class Export: - owner: Owner - identity: str - spans: tuple[dict[str, Any], ...] - result: SpanExportResult | None = None - - -@dataclass(eq=False) -class Flush: - owner: Owner - identity: str - exports: tuple[Export, ...] - result: SpanExportResult | None = None - - -def snapshot(span: ReadableSpan) -> dict[str, Any]: - context = span.context - if context is None: - raise ValueError("Exported span has no span context") - return { - "name": span.name, - "trace_id": format(context.trace_id, "032x"), - "span_id": format(context.span_id, "016x"), - "parent_id": format(span.parent.span_id, "016x") if span.parent else None, - "sampled": context.trace_flags.sampled, - "start_ns": span.start_time, - "end_ns": span.end_time, - "status": span.status.status_code.name, - "kind": span.kind.name, - "attributes": dict(span.attributes or {}), - "service_name": span.resource.attributes.get("service.name"), - "links": [ - { - "trace_id": format(link.context.trace_id, "032x"), - "span_id": format(link.context.span_id, "016x"), - } - for link in span.links - ], - "exception_event_count": ( - None - if span.dropped_events - else sum(event.name == "exception" for event in span.events) - ), - } - - -# Test-processor configuration, separate from the admitted acceptance budget. -DEFAULT_FLUSH_TIMEOUT_MS = 5000 - - -class Tracker: - """Keep SDK work and evidence publication owned until both actually settle.""" - - def __init__( - self, - provider_id: str, - exporter_id: str, - view: str, - export_mode: str, - flush_timeout_ms: float = DEFAULT_FLUSH_TIMEOUT_MS, - ) -> None: - self.provider_id, self.exporter_id, self.view = provider_id, exporter_id, view - self.export_mode = export_mode - self.flush_timeout_ms = flush_timeout_ms - self.lock = threading.RLock() - self.changed = threading.Condition(self.lock) - self.exports: dict[str, Export] = {} - self.flushes: dict[str, Flush] = {} - self.owners: dict[str, Owner] = {} - self.publications: queue.Queue[tuple[Owner, str, dict[str, Any]] | None] = ( - queue.Queue() - ) - self.publisher = threading.Thread( - target=self._publish, name="lmi-otel-evidence", daemon=True - ) - self.publisher.start() - - def owner(self, trace: Trace) -> Owner: - payload = trace.payload - result = Owner( - trace=trace, - tracker=self, - export_behavior=str(payload.get("otel_export", "success")), - gate=payload.get("otel_export_gate"), - shared=bool(payload.get("coverageProfile")), - case_id=str(payload.get("caseId", trace.identity.get("marker", ""))), - request_id=str(trace.identity["request"]), - ) - with self.lock: - self.owners[result.request_id] = result - return result - - def observe(self, owner: Owner, kind: str, **fields: Any) -> None: - if not owner.shared: - return - with self.lock: - captured = time.time_ns() - data = { - "atMs": captured / 1_000_000, - "capturedAtNs": captured, - "otelView": self.view, - "exportMode": self.export_mode, - "providerId": self.provider_id, - "exporterId": self.exporter_id, - **fields, - } - observation = owner.trace.capture_observation(kind, **data) - owner.publications += 1 - owner.quiescent = False - self.owners[owner.request_id] = owner - self.publications.put((owner, kind, observation)) - - def _publish(self) -> None: - while (item := self.publications.get()) is not None: - owner, kind, data = item - try: - owner.trace.publish_observation(data) - except Exception as error: - # Unknown publication remains owned, never a fabricated success. - with self.lock: - owner.publication_errors.append(repr(error)) - else: - with self.lock: - owner.publications -= 1 - if ( - kind == "export_quiescence" - and owner.publications == 0 - and owner.quiescent - ): - self.owners.pop(owner.request_id, None) - else: - self._quiescence(owner) - finally: - self.publications.task_done() - del item, owner, data - self.publications.task_done() - - def pending(self, owner: Owner) -> list[str]: - return [ - identity - for identity, work in self.exports.items() - if work.owner.case_id == owner.case_id and work.result is None - ] - - def active(self, case_id: str) -> dict[str, Any]: - with self.lock: - owners = [ - owner for owner in self.owners.values() if owner.case_id == case_id - ] - return { - "pendingExportIds": [ - key - for key, work in self.exports.items() - if work.owner.case_id == case_id - ], - "pendingFlushIds": [ - key - for key, work in self.flushes.items() - if work.owner.case_id == case_id - ], - "publicationCount": sum(owner.publications for owner in owners), - "publicationErrors": [ - error for owner in owners for error in owner.publication_errors - ], - } - - def close_invocation(self, owner: Owner) -> None: - with self.lock: - owner.closed = True - owner.exports.clear() - self._quiescence(owner) - - def _quiescence(self, owner: Owner) -> None: - if ( - not owner.closed - or owner.publications - or owner.quiescent - or any(work.owner is owner for work in self.exports.values()) - or any(work.owner is owner for work in self.flushes.values()) - ): - return - owner.quiescent = True - if owner.shared: - # This signal follows publication of every earlier observation. - # Do not count the signal itself as outstanding prior publication. - owner.publications += 1 - self.publications.put( - ( - owner, - "export_quiescence", - owner.trace.capture_observation( - "export_quiescence", - pendingExportIds=[], - pendingFlushIds=[], - publicationCount=0, - ), - ) - ) - else: - self.owners.pop(owner.request_id, None) - - def accept(self, spans: Sequence[ReadableSpan]) -> Export: - owner = CURRENT_OWNER.get() - if owner is None or owner.tracker is not self: - raise RuntimeError( - "Export lost its original public-plugin invocation owner" - ) - work = Export(owner, uuid.uuid4().hex, tuple(snapshot(span) for span in spans)) - with self.lock: - self.exports[work.identity] = work - owner.exports.append(work) - self.observe(owner, "export_started", exportId=work.identity) - for span in work.spans: - self.observe( - owner, - "otel_export_started", - exportId=work.identity, - traceId=span["trace_id"], - spanId=span["span_id"], - ) - return work - - def complete( - self, work: Export, result: SpanExportResult, error: str | None - ) -> None: - with self.changed: - work.result = result - self.observe( - work.owner, - "export_completed", - exportId=work.identity, - outcome="succeeded" if result is SpanExportResult.SUCCESS else "failed", - **( - {"fixture": "bounded-export-error"} - if work.owner.export_behavior == "failure" - else {} - ), - **( - {"error": error or "SpanExportResult.FAILURE"} - if result is not SpanExportResult.SUCCESS - else {} - ), - ) - self.exports.pop(work.identity) - for flush in tuple(self.flushes.values()): - self._complete_flush(flush) - self.changed.notify_all() - self._quiescence(work.owner) - - def _complete_flush(self, flush: Flush) -> None: - if flush.result is not None or any( - work.result is None for work in flush.exports - ): - return - flush.result = ( - SpanExportResult.SUCCESS - if all(work.result is SpanExportResult.SUCCESS for work in flush.exports) - else SpanExportResult.FAILURE - ) - self.observe( - flush.owner, - "flush_completed", - flushId=flush.identity, - outcome="succeeded" - if flush.result is SpanExportResult.SUCCESS - else "failed", - pendingExportIds=self.pending(flush.owner), - ) - self.flushes.pop(flush.identity) - self.changed.notify_all() - - def force_flush(self, timeout_millis: float) -> bool: - owner = CURRENT_OWNER.get() - if owner is None or owner.tracker is not self: - raise RuntimeError("Flush lost its public-plugin invocation owner") - allowed = min(float(timeout_millis), self.flush_timeout_ms) - if allowed <= 0: - raise ValueError( - "Flush wait must have a positive provider and scenario budget" - ) - deadline = time.monotonic() + allowed / 1000 - with self.changed: - flush = Flush(owner, uuid.uuid4().hex, tuple(owner.exports)) - owner.exports.clear() - self.flushes[flush.identity] = flush - self.observe( - owner, - "flush_started", - flushId=flush.identity, - timeoutMs=allowed, - configuredTimeoutMs=self.flush_timeout_ms, - pendingExportIds=self.pending(owner), - ) - self._complete_flush(flush) - while ( - flush.result is None and (remaining := deadline - time.monotonic()) > 0 - ): - self.changed.wait(remaining) - outcome = ( - "timeout" - if flush.result is None - else "completed" - if flush.result is SpanExportResult.SUCCESS - else "error" - ) - self.observe( - owner, - "flush_wait_completed", - flushId=flush.identity, - outcome=outcome, - pendingExportIds=self.pending(owner), - **({"error": "Observed export failure"} if outcome == "error" else {}), - ) - return flush.result is SpanExportResult.SUCCESS - - def shutdown(self) -> None: - self.publications.join() - self.publications.put(None) - self.publisher.join() - - -class LedgerExporter(SpanExporter): - def __init__(self, tracker: Tracker) -> None: - self.tracker = tracker - self.shutdown_calls = 0 - - def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: - return self.export_owned(self.tracker.accept(spans)) - - def export_owned(self, work: Export) -> SpanExportResult: - owner, result, failure = work.owner, SpanExportResult.FAILURE, None - try: - if owner.export_behavior == "held": - if not owner.gate: - raise ValueError("Held export requires its case-owned gate") - owner.trace.emit("OTEL_EXPORT_HELD", view=self.tracker.view) - if owner.shared: - - def held(at_ms): - self.tracker.observe( - owner, - "export_held", - exportId=work.identity, - gateId=owner.gate, - atMs=at_ms, - ) - - def released(release): - if release.held_at_ms is not None: - self.tracker.observe( - owner, - "export_released", - exportId=work.identity, - gateId=owner.gate, - mode=release.mode, - atMs=release.observed_at_ms, - ) - - owner.trace.gate_release( - owner.gate, on_hold=held, on_release=released - ) - else: - owner.trace.gate(owner.gate) - for span in work.spans: - owner.trace.emit( - "OTEL_EXPORT", - view=self.tracker.view, - outcome=owner.export_behavior, - span=span, - ) - self.tracker.observe( - owner, - "otel_export", - exportId=work.identity, - outcome=owner.export_behavior, - traceId=span["trace_id"], - spanId=span["span_id"], - span=span, - ) - result = ( - SpanExportResult.FAILURE - if owner.export_behavior == "failure" - else SpanExportResult.SUCCESS - ) - return result - except Exception as error: - failure = repr(error) - owner.trace.emit("OTEL_EXPORT_ERROR", view=self.tracker.view, error=failure) - return result - finally: - self.tracker.complete(work, result, failure) - - def shutdown(self) -> None: - self.shutdown_calls += 1 - self.tracker.shutdown() - - -class AsyncSpanProcessor(SpanProcessor): - def __init__(self, exporter: LedgerExporter) -> None: - self.exporter = exporter - self.executor = ThreadPoolExecutor( - max_workers=1, thread_name_prefix="lmi-otel-export" - ) - - def on_start(self, span, parent_context=None) -> None: - pass - - def on_end(self, span: ReadableSpan) -> None: - work = self.exporter.tracker.accept((span,)) - self.executor.submit(self.exporter.export_owned, work) - - def force_flush(self, timeout_millis: int = 30000) -> bool: - return self.exporter.tracker.force_flush(timeout_millis) - - def shutdown(self) -> None: - self.executor.shutdown(wait=True) - self.exporter.shutdown() diff --git a/lmi_tests/shared_observations.py b/lmi_tests/shared_observations.py index 61d40af9c..7cde20d9c 100644 --- a/lmi_tests/shared_observations.py +++ b/lmi_tests/shared_observations.py @@ -25,7 +25,7 @@ class ObservationSink(Protocol): - """Shared with telemetry without a circular import of fixture.Trace.""" + """Record SDK observations without importing fixture.Trace.""" def observe(self, kind: str, **fields: Any) -> Mapping[str, Any]: ... @@ -335,6 +335,7 @@ def on_user_function_end(self, info: UserFunctionEndInfo) -> None: operationId=info.operation_id, parentId=info.parent_id, outcome=info.outcome.value, + attempt=task.operation.attempt, ) diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py index b2878dde7..2e33ef809 100644 --- a/lmi_tests/shared_scenarios.py +++ b/lmi_tests/shared_scenarios.py @@ -1,12 +1,11 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Drive existing public-API fixtures through coordinated service boundaries.""" +"""Drive the seven shared lifecycle scenarios through real SDK handlers.""" from __future__ import annotations from collections import Counter -from collections.abc import Callable, Mapping -from copy import deepcopy +from collections.abc import Callable import hashlib import time from typing import Any @@ -16,37 +15,18 @@ from lmi_tests.cloud import Cloud from lmi_tests.evidence import CollectionError, PlacementError, overlap, select - -_SIMPLE = { - "native-concurrency": "barrier", - "replay-callback-retry": "replay", - "warm-lifecycle": "success", - "early-completion-parallel": "parallel", - "early-completion-map": "map", - "early-completion-nested": "nested", - "checkpoint-settlement": "checkpoint", - "deadline-recovery": "deadline", - "timeout-retry": "deadline", - "pending-cleanup": "suspend-cleanup", - "nested-progress": "nested-progress", - "return-inflight": "return-inflight", - "failure-inflight": "failure-inflight", - "late-operation": "late-operation", - "runtime-carrier-sampled": "success", - "runtime-carrier-unsampled": "success", - "otel-warm-lifecycle": "success", - "otel-concurrent": "barrier", - "export-error-recovery": "success", - "export-success": "success", - "export-failure-recovery": "success", - "held-export": "success", +FIXTURES = { + "native-overlap": "barrier", + "normal-return": "success", + "deadline-during-step": "deadline", + "peer-recovery": "deadline", + "suspension-retry": "replay", + "warm-reuse": "warm-owner", + "normal-failure": "failure", } -_COHORT = {"native-concurrency", "nested-progress", "otel-concurrent"} -_ANCHORED = {"pending-cleanup", "return-inflight", "failure-inflight"} def canonical(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - """Read actual observation records without replacing missing boundaries.""" return [ { "requestId": event.get("request"), @@ -61,8 +41,6 @@ def canonical(events: list[dict[str, Any]]) -> list[dict[str, Any]]: class ScenarioDriver: - """Case-local orchestration; the shared package owns acceptance assertions.""" - def __init__( self, context: ScenarioContext, @@ -71,24 +49,22 @@ def __init__( release_capacity: Callable[[], None] | None = None, reserve_capacity: Callable[[], None] | None = None, ) -> None: - self.context = context - self.cloud = cloud + self.context, self.cloud = context, cloud self.items: dict[str, dict[str, Any]] = {} - self.release_capacity = release_capacity - self.reserve_capacity = reserve_capacity + self.release_capacity, self.reserve_capacity = ( + release_capacity, + reserve_capacity, + ) self.boundary: dict[str, Any] | None = None - self.peer_gate: str | None = None - self.cohort_gate: str | None = None - self.cohort_gates: dict[str, str] = {} - self.target_environment: str | None = None @property def scenario(self) -> str: return self.context.scenario.scenario def seconds(self, budget: str, fallback: float) -> float: - value = self.context.scenario.behavior_budgets.get(budget) - return value / 1000 if value is not None else fallback + return ( + self.context.scenario.behavior_budgets.get(budget, fallback * 1000) / 1000 + ) def gate(self, label: str) -> str: owner = hashlib.sha256( @@ -104,78 +80,35 @@ def launch( gate: str | None = None, resume_roles: tuple[str, ...] = (), target_environment: str | None = None, - exporter: str = "success", ) -> dict[str, Any]: if role in self.items: raise CollectionError("A later launch cannot replace a selected original") - payload = self.context.dispatcher(role) - sampled = "0" if self.scenario == "runtime-carrier-unsampled" else "1" - # This header is sent on the real caller's HTTP Invoke request. It never - # supplies the runtime carrier through function input. - header = ( - f"Root=1-{int(time.time()):08x}-{uuid.uuid4().hex[:24]};" - f"Parent={uuid.uuid4().hex[:16]};Sampled={sampled}" - if self.context.scenario.otel_view - else None - ) item = self.cloud.start( fixture, gate=gate, target_environment=target_environment, - otel_view=self.context.scenario.otel_view, - trace_header=header, - otel_export=exporter, role=role, resume_roles=resume_roles, - dispatcher_payload=payload, + dispatcher_payload=self.context.dispatcher(role), marker=self.gate(role) + "-" + uuid.uuid4().hex[:12], ) self.items[role] = item return item def start(self) -> None: - if self.scenario not in _SIMPLE: + if self.scenario not in FIXTURES: raise NotImplementedError(f"No Python fixture for {self.scenario}") - if self.scenario in _COHORT: - self.cohort_gate = self.gate("cohort") - for role in ("victim", "peer")[: self.cloud.concurrency]: - gate = ( - self.gate(f"cohort-{role}") - if self.scenario == "otel-concurrent" - else self.cohort_gate - ) - self.cohort_gates[role] = gate - self.launch(role, _SIMPLE[self.scenario], gate=gate) + if self.scenario == "native-overlap": + for role in self.context.scenario.roles: + self.launch(role, "barrier", gate=self.gate("cohort")) return - if self.scenario in _ANCHORED and self.cloud.concurrency == 2: - self.peer_gate = self.gate("peer") - peer = self.launch("peer", "barrier", gate=self.peer_gate) - self.target_environment = self.cloud.phase(peer, "BLOCKED")[0][ - "environment" - ] - resume_roles = ( - ("resume",) - if self.scenario in {"replay-callback-retry", "pending-cleanup"} - else ("retry",) - if self.scenario == "timeout-retry" - else () - ) - exporter = ( - "held" - if self.scenario == "held-export" - else "failure" - if self.scenario in {"export-error-recovery", "export-failure-recovery"} - else "success" - ) self.launch( "victim", - _SIMPLE[self.scenario], + FIXTURES[self.scenario], gate=self.gate("fault") - if self.scenario in {"deadline-recovery", "timeout-retry"} + if self.scenario in {"deadline-during-step", "peer-recovery"} else None, - resume_roles=resume_roles, - target_environment=self.target_environment, - exporter=exporter, + resume_roles=("resume",) if self.scenario == "suspension-retry" else (), ) def wait_overlap( @@ -183,7 +116,7 @@ def wait_overlap( ) -> list[dict[str, Any]]: markers = {self.items[role]["marker"] for role in roles} - def established() -> list[dict[str, Any]] | None: + def established(): events = [ event for event in self.cloud.refresh() if event["marker"] in markers ] @@ -209,47 +142,27 @@ def established() -> list[dict[str, Any]] | None: def place(self) -> None: victim = self.items["victim"] - if self.scenario in _COHORT: - self.wait_overlap(tuple(self.items)) - return - if self.scenario in _ANCHORED and self.cloud.concurrency == 2: - admission = self.cloud.poll( - lambda: [ - event - for event in self.cloud.for_item(victim) - if event["phase"] in {"PLACEMENT_ACCEPTED", "PLACEMENT_MISS"} - ], - category=PlacementError, - message="Victim placement admission was not observed", - )[0] - if admission["phase"] != "PLACEMENT_ACCEPTED": - raise PlacementError("Original victim missed the selected environment") - if self.scenario in {"deadline-recovery", "timeout-retry"}: + if self.scenario == "native-overlap": + self.wait_overlap(("victim", "peer")) + elif self.scenario in {"deadline-during-step", "peer-recovery"}: self.boundary = self.cloud.phase(victim, "BLOCKED")[0] - if self.scenario == "deadline-recovery" and self.cloud.concurrency == 2: - self.peer_gate = self.gate("peer") - self.launch("peer", "barrier", gate=self.peer_gate) + if self.scenario == "peer-recovery": + self.launch("peer", "barrier", gate=self.gate("peer")) self.wait_overlap(("victim", "peer"), self.boundary["environment"]) else: self.cloud.phase(victim, "WRAPPER_ENTER") - def _finish(self, item: dict[str, Any], status: str = "SUCCEEDED") -> None: - self.cloud.finish(item, status) - def _callback(self, item: dict[str, Any]) -> None: - # Public callback waiting observes real service history. Waiting does not - # create a replacement durable execution. self.cloud.poll( lambda: [ - event - for event in self.cloud.for_item(item) - if event["phase"] == "WRAPPER_RETURN" - and event.get("status") == "PENDING" + e + for e in self.cloud.for_item(item) + if e["phase"] == "WRAPPER_RETURN" and e.get("status") == "PENDING" ], seconds=self.cloud.manifest.get("driverTimeout", 120), - message="Callback fixture did not actually suspend", + message="Callback fixture did not suspend", ) - if self.release_capacity is not None: + if self.release_capacity: self.release_capacity() try: callback = item["runner"].wait_for_callback( @@ -258,10 +171,8 @@ def _callback(self, item: dict[str, Any]) -> None: timeout=self.cloud.manifest.get("driverTimeout", 120), ) finally: - if self.reserve_capacity is not None: + if self.reserve_capacity: self.reserve_capacity() - self.cloud.check_active() - # Retain every earlier retry/wait invocation before sending completion. self.cloud.for_item(item) item["runner"].send_callback_success(callback, result=item["marker"].encode()) self.cloud.callback_records.append( @@ -273,251 +184,61 @@ def _callback(self, item: dict[str, Any]) -> None: } ) - def _peer_progress(self, boundary: Mapping[str, Any]) -> None: - if "peer" not in self.items: - return - self.wait_overlap(("victim", "peer"), self.target_environment) - self.cloud.poll( - lambda: [ - event - for event in self.cloud.for_item(self.items["peer"]) - if event["phase"] == "ALIVE" and event["time"] >= boundary["time"] - ], - message="Selected healthy peer did not progress during the held boundary", - ) - - def _finish_peer(self) -> None: - if self.peer_gate is not None: - self.cloud.release(self.peer_gate) - self._finish(self.items["peer"]) - - def _warm(self) -> None: - count = 3 if self.scenario == "warm-lifecycle" else 2 - sequence = ["success", "failure", "replay"] * count - for index, fixture in enumerate(sequence): - if index == 0: - item = self.items["victim"] - else: - if self.scenario == "warm-lifecycle": - role = "warm" if index == 1 else f"warm-{index}" - else: - role = "warm" if index == 5 else f"warm-{index}" - item = self.launch( - role, - fixture, - resume_roles=(f"{role}-resume",) if fixture == "replay" else (), - ) - if fixture == "replay": - self._callback(item) - self._finish(item, "FAILED" if fixture == "failure" else "SUCCEEDED") - def _deadline(self) -> None: - victim = self.items["victim"] - blocked = self.boundary - if blocked is None: + if self.boundary is None: raise CollectionError("Deadline fixture lost its original placement record") - if self.scenario == "deadline-recovery": - self._finish_peer() - self.cloud.poll( - lambda: time.time() >= blocked["deadline"] - 1, - seconds=self.cloud.manifest["invocationTimeout"] + 5, - ) - gate = self.gate("recovery") - roles = ("recovery", "recovery-peer")[: self.cloud.concurrency] - for role in roles: - self.launch(role, "barrier", gate=gate) - # Demand the original environment before eventual service-history - # delivery consumes the measurement window. - self.wait_overlap(roles, blocked["environment"]) - self.cloud.release(gate) - for role in roles: - self._finish(self.items[role]) - self.cloud.platform_timeout(victim, blocked["request"]) - cutoff = blocked["deadline"] + self.seconds("cleanupMs", 5) + victim = self.items["victim"] + # The fixture cooperatively exits real held I/O after the runtime deadline; + # no control release is allowed to establish the cleanup assertion. self.cloud.poll( - lambda: time.time() > cutoff + self.seconds("observationGraceMs", 1), - seconds=self.cloud.manifest["invocationTimeout"] + 30, - ) - if self.scenario == "timeout-retry": - self.cloud.poll( - lambda: ( - len( - { - event["request"] - for event in self.cloud.for_item(victim) - if event["phase"] == "WRAPPER_ENTER" - } - ) - >= 2 - ), - seconds=self.seconds("retryMs", 40), - message="No real service retry of the original execution was observed", - ) - # Close the measurement before the normal release. The later execution - # result cannot rewrite the original deadline, cohort or late-activity - # evidence, and no outcome is invented for a still-running retry. - self.cloud.history(victim) - events = self.cloud.refresh(validate_controls=False) - selected = { - attempt["role"]: attempt["requestId"] - for attempt in self.context.attempts - if attempt["attempt"] == 1 and attempt.get("requestId") - } - if selected.get("victim") != blocked["request"] or not set( - self.context.scenario.roles - ) <= set(selected): - raise CollectionError( - "Deadline measurement could not bind every original selected request" - ) - self.cloud.record_deadline_measurement( - { - "runId": self.context.run_id, - "caseId": self.context.scenario.case_id, - "cohortId": self.context.scenario.cohort_id, - "scenario": self.scenario, - "executionArn": victim["arn"], - "selectedRequests": selected, - "deadlineAtMs": blocked["deadline"] * 1000, - "cleanupCutoffAtMs": cutoff * 1000, - "observationCutoffAtMs": ( - cutoff + self.seconds("observationGraceMs", 1) - ) - * 1000, - "observedAtMs": time.time() * 1000, - "events": deepcopy(events), - "histories": deepcopy(self.cloud.raw_histories), - "sourceUris": sorted(self.cloud.source_uris), - } + lambda: time.time() > self.boundary["deadline"], + seconds=self.cloud.manifest["invocationTimeout"] + 5, ) - self.cloud.release(self.gate("fault")) - self.cloud.wait_terminal(victim) - - def _held_export(self) -> None: - victim = self.items["victim"] - self.cloud.phase(victim, "OTEL_EXPORT_HELD") - if self.context.scenario.export_mode == "async-bounded-flush": - self.cloud.poll( - lambda: [ - event - for event in canonical(self.cloud.for_item(victim)) - if event.get("kind") == "flush_wait_completed" - and event.get("outcome") == "timeout" - ], - message="Asynchronous held export did not reach the real bounded flush timeout", + if self.scenario == "peer-recovery": + recovery = self.launch( + "recovery", + "barrier", + gate=self.gate("recovery"), + target_environment=self.boundary["environment"], ) - self.cloud.phase(victim, "WRAPPER_RETURN") - self.cloud.release(victim["marker"] + "-export") - self._finish(victim) + self.wait_overlap(("peer", "recovery"), self.boundary["environment"]) + self.cloud.release(self.gate("recovery")) + self.cloud.finish(recovery) + self.cloud.release(self.gate("peer")) + self.cloud.finish(self.items["peer"]) + self.cloud.platform_timeout(victim, self.boundary["request"]) self.cloud.poll( - lambda: [ - event - for event in canonical(self.cloud.for_item(victim)) - if event.get("kind") == "export_completed" - ], - message="Held exporter did not actually finish after its case-owned release", + lambda: ( + time.time() + > self.boundary["deadline"] + + self.seconds("cleanupMs", 5) + + self.seconds("observationGraceMs", 1) + ), + seconds=self.cloud.manifest["invocationTimeout"] + 30, ) + self.cloud.wait_terminal(victim) def execute(self) -> None: victim = self.items["victim"] - if self.scenario in _COHORT: - if self.cohort_gate is None: - raise CollectionError("Missing cohort barrier") - # Separate gates let telemetry observe one request ending while its - # peer remains held. Ordinary cohorts share one case-owned barrier. - if self.scenario != "otel-concurrent": - self.cloud.release(self.cohort_gate) - for role, item in self.items.items(): - if self.scenario == "otel-concurrent": - self.cloud.release(self.cohort_gates[role]) - if self.scenario == "nested-progress": - self.cloud.phase( - item, "PROGRESS", seconds=self.seconds("progressMs", 20) - ) - self._finish(item) - if ( - self.scenario == "otel-concurrent" - and role == "victim" - and "peer" in self.items - ): - returned = self.cloud.phase(item, "WRAPPER_RETURN")[-1] - self.cloud.poll( - lambda: [ - event - for event in self.cloud.for_item(self.items["peer"]) - if event["phase"] == "ALIVE" - and event["time"] >= returned["time"] - ], - message="Ending the original OTel invocation did not preserve its held peer", - ) - elif self.scenario in {"warm-lifecycle", "otel-warm-lifecycle"}: - self._warm() - elif self.scenario == "replay-callback-retry": - self._callback(victim) - self._finish(victim) - elif self.scenario.startswith("early-completion-"): - winner = self.cloud.phase(victim, "WINNER_SELECTED")[0] - cutoff = ( - winner["time"] - + self.seconds("cleanupMs", 5) - + self.seconds("observationGraceMs", 1) - ) - self.cloud.poll( - lambda: time.time() >= cutoff, seconds=self.seconds("cleanupMs", 5) + 30 - ) - # Do not release the losing branch to manufacture earlier cleanup. - self.cloud.for_item(victim) - elif self.scenario in {"deadline-recovery", "timeout-retry"}: + if self.scenario == "native-overlap": + self.cloud.release(self.gate("cohort")) + for item in self.items.values(): + self.cloud.finish(item) + elif self.scenario in {"deadline-during-step", "peer-recovery"}: self._deadline() - elif self.scenario == "checkpoint-settlement": - self.cloud.phase(victim, "ACK_HELD") - self.cloud.phase(victim, "BLOCKED") - self.cloud.release(victim["marker"] + "-checkpoint") - self._finish(victim) - elif self.scenario == "pending-cleanup": - blocked = self.cloud.phase(victim, "BLOCKED")[0] - self._peer_progress(blocked) - self.cloud.release(victim["marker"] + "-cleanup") - self._finish(victim) - self._finish_peer() - elif self.scenario in {"return-inflight", "failure-inflight"}: - self.cloud.phase(victim, "WINNER_SELECTED") - exit_boundary = self.cloud.phase(victim, "USER_EXIT")[0] - self._peer_progress(exit_boundary) - self.cloud.release(victim["marker"] + "-loser") - self._finish( - victim, "FAILED" if self.scenario == "failure-inflight" else "SUCCEEDED" - ) - self._finish_peer() - elif self.scenario == "late-operation": - self.cloud.phase(victim, "WINNER_SELECTED") - self.cloud.release(victim["marker"] + "-loser") - self.cloud.poll( - lambda: [ - event - for event in self.cloud.for_item(victim) - if event["phase"] in {"LATE_REJECTED", "LATE_ACCEPTED"} - ], - message="Residual child never attempted the late public durable operation", - ) - self._finish(victim) - elif self.scenario == "held-export": - self._held_export() + elif self.scenario == "suspension-retry": + self._callback(victim) + self.cloud.finish(victim) + elif self.scenario == "warm-reuse": + self.cloud.finish(victim) + self.cloud.finish(self.launch("warm", "warm-probe")) else: - self._finish(victim) - if self.scenario in {"export-error-recovery", "export-failure-recovery"}: - self._finish(self.launch("warm", "success")) + self.cloud.finish( + victim, "FAILED" if self.scenario == "normal-failure" else "SUCCEEDED" + ) self.cloud.observation_ended_at = time.time() def collect(self) -> None: - if not self.scenario.startswith("early-completion-") and self.scenario not in { - "deadline-recovery", - "timeout-retry", - }: - self.cloud.poll( - self.quiescent, - seconds=self.cloud.manifest.get("driverTimeout", 120), - message="Completed scenario evidence still has unsettled task/export publication", - ) self.cloud.wait(self.seconds("observationGraceMs", 1)) self.cloud.collect(full_run=False) self.cloud.observation_ended_at = time.time() @@ -547,8 +268,6 @@ def quiescent(self) -> bool: ("io_start", "io_exit", "taskId"), ("checkpoint_call", "checkpoint_exit", "callId"), ("poll_call", "poll_exit", "callId"), - ("export_started", "export_completed", "exportId"), - ("flush_started", "flush_completed", "flushId"), ): starts = Counter( (item.get("requestId"), item.get(identity)) @@ -571,17 +290,6 @@ def quiescent(self) -> bool: } if not entered <= root_ends: return False - if self.context.scenario.otel_view: - published = { - item.get("requestId") - for item in observations - if item.get("kind") == "export_quiescence" - and item.get("pendingExportIds") == [] - and item.get("pendingFlushIds") == [] - and item.get("publicationCount") == 0 - } - if not entered <= published: - return False return True def retire(self) -> bool: @@ -596,7 +304,7 @@ def retire(self) -> bool: self.cloud.poll( self.quiescent, seconds=self.cloud.manifest.get("driverTimeout", 120), - message="Case-owned task/request/export work is not quiescent", + message="Case-owned task/request work is not quiescent", ) self.cloud.collect(full_run=False) self.cloud.retired_at = time.time() diff --git a/lmi_tests/shared_telemetry.py b/lmi_tests/shared_telemetry.py deleted file mode 100644 index 217bfc43f..000000000 --- a/lmi_tests/shared_telemetry.py +++ /dev/null @@ -1,207 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Driver-only conversion of actual OTel ledger data to the shared wire model. - -This module is not part of the Lambda fixture bundle. It imports the pinned -conformance model only when converting observed spans, not on runtime startup. -""" - -from __future__ import annotations - -from collections.abc import Mapping, Sequence -from datetime import UTC, datetime -from typing import Any - - -def _canonical_span(raw: Mapping[str, Any]) -> dict[str, Any]: - from aws_durable_execution_conformance_tests_otel.model import ( - Span, - SpanLink, - span_to_dict, - ) - - return span_to_dict( - Span( - trace_id=raw["trace_id"], - span_id=raw["span_id"], - name=raw["name"], - start_time=datetime.fromtimestamp(raw["start_ns"] / 1_000_000_000, UTC), - end_time=datetime.fromtimestamp(raw["end_ns"] / 1_000_000_000, UTC), - parent_span_id=raw["parent_id"], - status=raw["status"], - kind=raw["kind"], - attributes=raw["attributes"], - service_name=raw["service_name"], - links=tuple(SpanLink(**link) for link in raw["links"]), - exception_event_count=raw["exception_event_count"], - ) - ) - - -def collect_telemetry( - raw_events: Sequence[Mapping[str, Any]], - attempts: Sequence[Mapping[str, Any]], - *, - view: str, - expected_sampled: bool, - caller_headers: Mapping[str, str], - observed_until_ms: float, - export_mode: str | None = None, -) -> dict[str, Any]: - """Preserve exporter ownership and report incomplete collection explicitly. - - Expected sampling and caller headers come from the admitted driver's actual - Invoke configuration. They are never inferred from the observed runtime - carrier or from whether telemetry happened to arrive. - """ - by_request = {attempt["requestId"]: attempt for attempt in attempts} - observations = [ - (record, record["observation"]) - for record in raw_events - if record.get("phase") == "CONFORMANCE" - and isinstance(record.get("observation"), Mapping) - ] - providers: dict[str, dict[str, Any]] = {} - bindings, carriers, spans, outcomes = [], [], [], [] - complete = bool(attempts) - for request_id, attempt in by_request.items(): - records = [ - record for record in raw_events if record.get("request") == request_id - ] - local = [ - (record, observation) - for record, observation in observations - if record.get("request") == request_id - ] - numbers = [record.get("sequence") for record in records] - complete &= bool(numbers) and all(type(number) is int for number in numbers) - if numbers and all(type(number) is int for number in numbers): - sequences = [number for number in numbers if isinstance(number, int)] - complete &= sorted(sequences) == list(range(1, max(sequences) + 1)) - complete &= any( - record.get("phase") in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - for record in records - ) - quiescent = [ - observation - for _, observation in local - if observation.get("kind") == "export_quiescence" - ] - complete &= bool(quiescent) and all( - item.get("pendingExportIds") == [] - and item.get("pendingFlushIds") == [] - and item.get("publicationCount") == 0 - for item in quiescent - ) - entries = [ - (record, item) for record, item in local if item.get("kind") == "otel_enter" - ] - if any(item.get("otelView") != view for _, item in entries): - raise ValueError("The observed OTel view differs from the admitted view") - if export_mode is not None and any( - item.get("exportMode") != export_mode for _, item in entries - ): - raise ValueError( - "The observed exporter mode differs from the admitted mode" - ) - completed = [ - item for _, item in local if item.get("kind") == "export_completed" - ] - started = [item for _, item in local if item.get("kind") == "export_started"] - complete &= len(entries) == 1 and {item["exportId"] for item in started} == { - item["exportId"] for item in completed - } - for _, observation in local: - if observation.get("kind") == "provider_observed": - provider = { - key: observation[key] - for key in ( - "providerId", - "exporterId", - "createdAtMs", - "environmentId", - "processId", - ) - } - previous = providers.setdefault(provider["providerId"], provider) - if previous != provider: - raise ValueError( - "The observed provider identity changed across requests" - ) - if observation.get("kind") == "otel_export": - raw_span = observation["span"] - spans.append( - { - "requestId": request_id, - "exportId": observation["exportId"], - "sampled": raw_span["sampled"], - "span": _canonical_span(raw_span), - } - ) - if len(entries) == 1: - source, entry = entries[0] - bindings.append( - {"requestId": request_id, "providerId": entry["providerId"]} - ) - carriers.append( - { - "requestId": request_id, - "environmentId": source["environment"], - "processId": source["process"], - "workerId": source.get("workerId", source["process"]), - "source": entry["carrierSource"], - "header": entry.get("carrier"), - "expectedSampled": expected_sampled, - **( - {"callerHeader": caller_headers[request_id]} - if request_id in caller_headers - else {} - ), - } - ) - failures = [item for item in completed if item.get("outcome") == "failed"] - if failures: - failed = failures[0] - outcomes.append( - { - "requestId": request_id, - "status": "export-error", - "error": failed.get("error"), - "fixture": failed.get("fixture"), - } - ) - else: - outcomes.append( - { - "requestId": request_id, - "status": "delivered" - if started - else "unsampled" - if not expected_sampled - else "missing", - } - ) - unknown = { - record.get("request") - for record, item in observations - if item.get("kind") == "otel_export" - } - set(by_request) - if unknown: - raise ValueError("Telemetry contains an unjournaled original invocation") - return { - "providers": list(providers.values()), - "providerBindings": bindings, - "views": { - view: { - "collection": { - "status": "complete" if complete else "incomplete", - "scope": "cohort", - "requestIds": list(by_request), - "throughMs": observed_until_ms, - }, - "carriers": carriers, - "spans": spans, - "exportOutcomes": outcomes, - } - }, - } diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py index fcbcdbbb8..d01164263 100644 --- a/lmi_tests/summary.py +++ b/lmi_tests/summary.py @@ -1,43 +1,34 @@ -"""Small, dependency-free Actions summary; no executed scenario means no pass.""" +"""Summarize the shared lifecycle report without inventing cloud acceptance.""" import json from pathlib import Path -import xml.etree.ElementTree as ET def summarize(directory): directory = Path(directory) manifest = directory / "manifest.json" - lines = ["LMI cloud regressions for Python SDK #741.", ""] - if manifest.exists(): - config = json.loads(manifest.read_text()) + config = json.loads(manifest.read_text()) if manifest.exists() else None + lines = ["Python shared LMI lifecycle conformance."] + if config: lines.append( - f"Commit `{config['commit']}`; {config['runtime']}; one deployment, environment concurrency {list(config['concurrencies'].values())}." + f"Commit `{config['commit']}`; {config['runtime']}; native concurrency {list(config['concurrencies'].values())}." ) + shared = directory / "shared" + path = shared / "standard/report.json" + report = json.loads(path.read_text()) if path.exists() else None + if report and config is None: lines.append( - f"Timeouts: invocation {config['invocationTimeout']}s (all scenarios), durable execution {config['executionTimeout']}s, driver {config['driverTimeout']}s; cleanup grace {config['cleanupGrace']}s." + "Deployment identity is missing; current cloud acceptance is unavailable." + ) + elif report and config and report.get("run", {}).get("name") != config["run"]: + lines.append( + "Report belongs to another run; current cloud acceptance is unavailable." + ) + elif report and report.get("results"): + lines.extend(f"- {case['id']}: {case['status']}" for case in report["results"]) + lines.append( + f"Shared runner exit code: {report.get('ci', {}).get('exit_code', 'unknown')}." ) - report = directory / "cloud.xml" - if report.exists(): - for case in ET.parse(report).iter("testcase"): - category = next( - ( - p.get("value") - for p in case.findall("properties/property") - if p.get("name") == "lmi_outcome" - ), - None, - ) - problem = next( - ( - case.find(k) - for k in ("error", "failure", "skipped") - if case.find(k) is not None - ), - None, - ) - state = category or (problem.tag.upper() if problem is not None else "PASS") - lines.append(f"- {case.get('name')}: {state}") else: lines.append( "Cloud scenarios did not run. Inspect provisioning/build diagnostics." @@ -45,11 +36,17 @@ def summarize(directory): for path in sorted(directory.glob("*-error.json")): data = json.loads(path.read_text()) lines.append(f"- {path.stem}: {data['type']}") - cleanup = directory / "cleanup.json" + receipt = shared / "ownership-released.json" + released = ( + config is not None + and receipt.exists() + and json.loads(receipt.read_text()) + == {"runId": config["run"], "lease": config.get("sharedLease")} + ) lines.append( - "Run work released; persistent stack, functions, bucket, and code retained." - if cleanup.exists() - else "Run cleanup not confirmed; persistent resources are retained. Inspect cleanup diagnostics before updating." + "Run ownership released after retirement; persistent resources retained." + if released + else "Run cleanup not confirmed; ownership and persistent resources are retained." ) return "\n\n".join(lines) diff --git a/lmi_tests/telemetry.py b/lmi_tests/telemetry.py deleted file mode 100644 index adbee4125..000000000 --- a/lmi_tests/telemetry.py +++ /dev/null @@ -1,246 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Real public OTel plugins with immutable runtime and invocation ownership.""" - -from __future__ import annotations - -from collections.abc import Callable -from importlib.metadata import version -import os -import threading -import time -from typing import Any -import uuid -from weakref import WeakKeyDictionary - -from opentelemetry import context as otel_context, trace as otel_trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor -from opentelemetry.sdk.trace.sampling import ALWAYS_ON - -from aws_durable_execution_sdk_python_otel import ( - ExecutionOtelPluginFactory, - InvocationOtelPluginFactory, - OtelPluginConfig, -) -from aws_durable_execution_sdk_python.plugin import ( - DurableInstrumentationPlugin, - DurableInstrumentationPluginFactory, - InvocationStartInfo, -) -from lmi_tests.shared_exporter import ( - AsyncSpanProcessor, - CURRENT_OWNER, - DEFAULT_FLUSH_TIMEOUT_MS, - LedgerExporter, - Owner, - Trace, - Tracker, - owned, -) - -_RUNTIMES: dict[tuple[str, str], Runtime] = {} -_RUNTIME_LOCK = threading.Lock() - - -class ObservedPlugin(DurableInstrumentationPlugin): - """Forward the real public hooks, binding their immutable invocation owner.""" - - def __init__(self, delegate: DurableInstrumentationPlugin, owner: Owner) -> None: - self.delegate, self.owner = delegate, owner - - def _call(self, name: str, info: Any) -> Any: - with owned(self.owner): - return getattr(self.delegate, name)(info) - - def on_invocation_start(self, info) -> None: - self._call("on_invocation_start", info) - - def on_invocation_end(self, info) -> None: - self._call("on_invocation_end", info) - - def on_operation_start(self, info) -> None: - self._call("on_operation_start", info) - - def on_operation_end(self, info) -> None: - self._call("on_operation_end", info) - - def on_operation_change(self, info) -> None: - self._call("on_operation_change", info) - - def on_user_function_start(self, info) -> None: - self._call("on_user_function_start", info) - - def on_user_function_end(self, info) -> None: - self._call("on_user_function_end", info) - - def provide_propagation_metadata(self, info): - return self._call("provide_propagation_metadata", info) - - -class EvidenceFactory: - __durable_registration_api__ = 1 - exclusive_group = "aws-durable-execution-otel-view" - - def __init__(self, view: str, provider: TracerProvider) -> None: - kind = ( - InvocationOtelPluginFactory - if view == "invocation" - else ExecutionOtelPluginFactory - ) - self.delegate = kind( - OtelPluginConfig(tracer_provider=provider, enrich_logger=True) - ) - self.instances: WeakKeyDictionary[DurableInstrumentationPlugin, str] = ( - WeakKeyDictionary() - ) - self.lock = threading.Lock() - - def create_plugin(self, info: InvocationStartInfo) -> DurableInstrumentationPlugin: - owner = CURRENT_OWNER.get() - if owner is None: - raise RuntimeError("Factory lost the invocation evidence owner") - plugin = self.delegate.create_plugin(info) - with self.lock: - identity = self.instances.setdefault(plugin, uuid.uuid4().hex) - owner.trace.emit( - "OTEL_FACTORY", - plugin_instance=identity, - plugin_execution=info.execution_arn, - ) - owner.tracker.observe( - owner, - "otel_factory", - pluginInstanceId=identity, - pluginExecutionArn=info.execution_arn, - ) - return ObservedPlugin(plugin, owner) - - -class Runtime: - """An immutable provider variant shared across invocations in one process.""" - - def __init__( - self, - view: str, - export_mode: str = "sync", - *, - flush_timeout_ms: float = DEFAULT_FLUSH_TIMEOUT_MS, - ) -> None: - if view not in {"invocation", "execution"} or export_mode not in { - "sync", - "async-bounded-flush", - }: - raise ValueError("Unknown OTel view or exporter mode") - self.identity = uuid.uuid4().hex - self.created_at_ms = time.time_ns() / 1_000_000 - self.view, self.export_mode = view, export_mode - self.tracker = Tracker( - self.identity, uuid.uuid4().hex, view, export_mode, flush_timeout_ms - ) - self.exporter = LedgerExporter(self.tracker) - self.provider = TracerProvider(sampler=ALWAYS_ON, shutdown_on_exit=False) - self.provider.add_span_processor( - SimpleSpanProcessor(self.exporter) - if export_mode == "sync" - else AsyncSpanProcessor(self.exporter) - ) - self.factory = EvidenceFactory(view, self.provider) - - def pending_for_case(self, case_id: str) -> dict[str, Any]: - return self.tracker.active(case_id) - - -def active_work(trace: Trace) -> dict[str, Any]: - """Actual outstanding work/publication across immutable runtime variants.""" - case_id = str(trace.payload.get("caseId", trace.identity.get("marker", ""))) - result: dict[str, Any] = { - "pendingExportIds": [], - "pendingFlushIds": [], - "publicationCount": 0, - "publicationErrors": [], - } - with _RUNTIME_LOCK: - runtimes = tuple(_RUNTIMES.values()) - for runtime in runtimes: - state = runtime.pending_for_case(case_id) - for field in ("pendingExportIds", "pendingFlushIds", "publicationErrors"): - result[field].extend(state[field]) - result["publicationCount"] += state["publicationCount"] - return result - - -def _context_snapshot() -> dict[str, str | None]: - context = otel_trace.get_current_span().get_span_context() - return { - "traceId": format(context.trace_id, "032x") if context.is_valid else None, - "spanId": format(context.span_id, "016x") if context.is_valid else None, - } - - -def invoke( - trace: Trace, - invoke_handler: Callable[ - [list[DurableInstrumentationPluginFactory]], dict[str, Any] - ], -) -> dict[str, Any]: - view = trace.payload.get("otel_view", trace.payload.get("otelView")) - export_mode = str(trace.payload.get("exportMode", "sync")) - if view not in {"invocation", "execution"} or export_mode not in { - "sync", - "async-bounded-flush", - }: - raise ValueError("Unknown OTel view or exporter mode") - key = (view, export_mode) - with _RUNTIME_LOCK: - if key not in _RUNTIMES: - _RUNTIMES[key] = Runtime(view, export_mode) - runtime = _RUNTIMES[key] - owner = runtime.tracker.owner(trace) - before, snapshot = otel_context.get_current(), _context_snapshot() - carrier = os.environ.get("_X_AMZN_TRACE_ID") - trace.emit( - "OTEL_ENTER", - view=view, - export_mode=runtime.export_mode, - provider=runtime.identity, - carrier=carrier, - carrier_source="runtime_environment", - core_version=version("aws-durable-execution-sdk-python"), - otel_version=version("aws-durable-execution-sdk-python-otel"), - ) - runtime.tracker.observe( - owner, - "provider_observed", - createdAtMs=runtime.created_at_ms, - environmentId=trace.identity["environment"], - processId=trace.identity["process"], - ) - runtime.tracker.observe( - owner, - "otel_enter", - contextBefore=snapshot, - carrier=carrier, - carrierSource="process-environment", - ) - try: - with owned(owner): - return invoke_handler([runtime.factory]) - finally: - try: - # The SDK caller captures its real return before this diagnostic I/O. - trace.emit( - "OTEL_EXIT", - view=view, - provider=runtime.identity, - context_restored=otel_context.get_current() == before, - exporter_shutdowns=runtime.exporter.shutdown_calls, - ) - runtime.tracker.observe( - owner, - "otel_exit", - contextAfter=_context_snapshot(), - exporterShutdowns=runtime.exporter.shutdown_calls, - ) - finally: - runtime.tracker.close_invocation(owner) diff --git a/lmi_tests/telemetry_evidence.py b/lmi_tests/telemetry_evidence.py deleted file mode 100644 index 2d298cba5..000000000 --- a/lmi_tests/telemetry_evidence.py +++ /dev/null @@ -1,134 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Independent assertions over recorded native headers and exported OTel spans.""" - -import re -from typing import Any - -from lmi_tests.evidence import CollectionError, PlacementError, select - - -def parse_header(header: object) -> dict[str, str | bool] | None: - if not isinstance(header, str): - return None - fields = dict( - piece.strip().split("=", 1) for piece in header.split(";") if "=" in piece - ) - root, parent, sampled = (fields.get(key) for key in ("Root", "Parent", "Sampled")) - if ( - not root - or not re.fullmatch(r"1-[0-9a-f]{8}-[0-9a-f]{24}", root) - or not parent - or not re.fullmatch(r"[0-9a-f]{16}", parent) - or sampled not in {"0", "1"} - ): - return None - return { - "trace_id": root[2:].replace("-", ""), - "parent_id": parent, - "sampled": sampled == "1", - } - - -def explicit_sampling(header: object) -> bool | None: - if not isinstance(header, str): - return None - values = [ - part.strip()[8:] - for part in header.split(";") - if part.strip().startswith("Sampled=") - ] - return None if not values or values[-1] not in {"0", "1"} else values[-1] == "1" - - -def invocation( - events: list[dict[str, Any]], - *, - native: bool = True, - expected_outcome: str = "success", -) -> None: - starts = select(events, "OTEL_ENTER") - if not starts: - raise CollectionError("Missing OTel invocation entry and carrier evidence") - for start in starts: - local = [event for event in events if event["request"] == start["request"]] - ends = select(local, "OTEL_EXIT") - if len(ends) != 1: - raise CollectionError("Missing or duplicate OTel invocation exit evidence") - assert ends[0]["context_restored"], "OTel context leaked after invocation End" - assert ends[0]["exporter_shutdowns"] == 0, ( - "Invocation shut down a shared exporter" - ) - factories = select(local, "OTEL_FACTORY") - assert len(factories) == 1, "Expected one plugin instance per invocation" - exports = select(local, "OTEL_EXPORT") - header = parse_header(start.get("carrier")) - if native and header is None: - raise CollectionError( - "Real LMI did not provide the required valid X-Ray carrier" - ) - if explicit_sampling(start.get("carrier")) is False: - assert not exports, "An explicit Sampled=0 invocation exported spans" - continue - roots = [ - event["span"] for event in exports if event["span"]["name"] == "Invocation" - ] - if len(roots) != 1: - raise CollectionError( - "Expected one exported Invocation span for a sampled request" - ) - for event in exports: - span = event["span"] - assert event["outcome"] == expected_outcome, "Unexpected export result" - assert ( - span["attributes"].get("durable.execution.arn") == start["execution"] - ), "Exported span belongs to another execution" - assert span["start_ns"] <= span["end_ns"], "Span duration is negative" - assert span["sampled"], "Recording export has an unsampled span context" - if header is not None: - assert span["trace_id"] == header["trace_id"], ( - "Runtime X-Ray Root was lost" - ) - if header is not None: - assert roots[0]["parent_id"] == header["parent_id"], ( - "Runtime X-Ray Parent was lost" - ) - instances = [event["plugin_instance"] for event in select(events, "OTEL_FACTORY")] - assert len(instances) == len(set(instances)), ( - "Plugin instance reused across invocations" - ) - - -def terminal(events: list[dict[str, Any]], *, failed: bool = False) -> None: - """Workflow is emitted once at the terminal invocation, not once per replay.""" - sampled = [ - entry - for entry in select(events, "OTEL_ENTER") - if explicit_sampling(entry.get("carrier")) is not False - ] - if not sampled: - return - spans = [event["span"] for event in select(events, "OTEL_EXPORT")] - workflows = [span for span in spans if span["name"] == "Workflow"] - assert len(workflows) == 1, "Workflow was missing or duplicated across replay" - assert workflows[0]["status"] == ("ERROR" if failed else "OK") - - -def warm_reuse(events: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: - starts = select(events, "OTEL_ENTER") - cohorts = {} - for start in starts: - key = (start["environment"], start["process"], start["pid"], start["view"]) - cohorts.setdefault(key, []).append(start) - reused = [ - values - for values in cohorts.values() - if len({e["execution"] for e in values}) > 1 - ] - if not reused: - raise PlacementError("No observed sequential reuse of a Python worker process") - for values in reused: - assert len({entry["provider"] for entry in values}) == 1, ( - "Provider was rebuilt per invocation" - ) - return reused diff --git a/lmi_tests/tests/e2e/conftest.py b/lmi_tests/tests/e2e/conftest.py deleted file mode 100644 index 899949e98..000000000 --- a/lmi_tests/tests/e2e/conftest.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Explicit cloud opt-in; infrastructure/collection exceptions stay JUnit errors.""" - -import json - -import pytest - -from lmi_tests.cloud import Cloud -from lmi_tests.deploy import ARTIFACTS, CONCURRENCIES -from lmi_tests.evidence import CollectionError, PlacementError, ProvisioningError - - -def pytest_addoption(parser): - parser.addoption( - "--lmi-cloud", action="store_true", help="Use a verified real LMI deployment" - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - report = outcome.get_result() - if call.excinfo: - category = type(call.excinfo.value).__name__ - if isinstance( - call.excinfo.value, (CollectionError, PlacementError, ProvisioningError) - ): - report.user_properties.append(("lmi_outcome", category)) - item.user_properties.append(("lmi_outcome", category)) - elif isinstance(call.excinfo.value, AssertionError): - report.user_properties.append(("lmi_outcome", "RegressionAssertion")) - item.user_properties.append(("lmi_outcome", "RegressionAssertion")) - - -@pytest.fixture(scope="session") -def deployment(request): - if not request.config.getoption("--lmi-cloud"): - pytest.skip("Real LMI deployment required; run hatch run lmi:cloud") - manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) - cloud = Cloud(manifest) - cloud.verify() - return manifest - - -@pytest.fixture(scope="session", params=CONCURRENCIES) -def configuration(request, deployment): - return {"key": request.param, "pending": None} - - -@pytest.fixture -def cloud(deployment, configuration): - # A failed teardown cannot silently hand a still-busy function to another - # case. Retry only retirement; never retry the regression assertions. - if configuration["pending"] is not None: - configuration["pending"].settle_case() - driver = Cloud(deployment, configuration["key"]) - configuration["pending"] = driver - try: - yield driver - finally: - try: - driver.settle_case() - configuration["pending"] = None - finally: - driver.collect(full_run=False) diff --git a/lmi_tests/tests/e2e/test_lifecycle.py b/lmi_tests/tests/e2e/test_lifecycle.py deleted file mode 100644 index 525dc15eb..000000000 --- a/lmi_tests/tests/e2e/test_lifecycle.py +++ /dev/null @@ -1,301 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Real LMI regressions: #741 assertions intentionally fail until the SDK fix.""" - -import time -import uuid - -import pytest - -from lmi_tests import evidence - - -def test_checkpoint_replay_callback_and_retry(cloud): - item = cloud.start("replay") - callback = item["runner"].wait_for_callback( - item["arn"], name="completion", timeout=90 - ) - item["runner"].send_callback_success(callback, result=item["marker"].encode()) - history = cloud.finish(item) - evidence.replay(cloud.for_item(item), history, item["marker"]) - - -def capacity_barrier(cloud, environment=None, seconds=30): - count = cloud.concurrency - gate = "capacity-" + uuid.uuid4().hex - items = [cloud.start("barrier", gate=gate) for _ in range(count)] - env, events = wait_overlap(cloud, items, count, environment, seconds) - cloud.release(gate) - for item in items: - cloud.finish(item) - evidence.lifecycle(cloud.for_item(item)) - return env, evidence.select(events, "BLOCKED") - - -def wait_overlap(cloud, items, count, environment=None, seconds=30): - markers = {i["marker"] for i in items} - - def observed(): - events = [e for e in cloud.refresh() if e["marker"] in markers] - if environment is not None and any( - e["environment"] != environment for e in evidence.select(events, "BLOCKED") - ): - raise evidence.PlacementError( - "Recovery probes reached a replacement/unrelated environment" - ) - try: - return evidence.overlap(events, markers, count, environment), events - except evidence.PlacementError: - return None - - return cloud.poll( - observed, - seconds=seconds, - category=AssertionError if environment is not None else evidence.PlacementError, - message="#741: full original-environment worker capacity did not recover" - if environment is not None - else "Live worker overlap was not established in the required environment", - ) - - -def test_actual_environment_process_concurrency(cloud): - # c1 is a genuine baseline; c2 must establish different PIDs/process UUIDs - # and distinct durable execution ARNs inside the same /tmp environment. - capacity_barrier(cloud) - - -def test_warm_success_failure_and_suspension_cleanup(cloud): - items = [] - for scenario in ["success", "failure", "replay"] * 3: - item = cloud.start(scenario) - items.append(item) - if scenario == "replay": - callback = item["runner"].wait_for_callback( - item["arn"], name="completion", timeout=90 - ) - item["runner"].send_callback_success( - callback, result=item["marker"].encode() - ) - cloud.finish(item, "FAILED" if scenario == "failure" else "SUCCEEDED") - markers = {i["marker"] for i in items} - evidence.warm_resources([e for e in cloud.refresh() if e["marker"] in markers]) - - -@pytest.mark.parametrize("scenario", ["parallel", "map", "nested"]) -def test_early_completion_reclaims_losing_branches(cloud, scenario): - item = cloud.start(scenario) - winner = cloud.phase(item, "WINNER_SELECTED")[0] - # Observation is deliberately completed BEFORE any driver release signal. - cutoff = winner["time"] + cloud.manifest["cleanupGrace"] - cloud.poll(lambda: time.time() >= cutoff + 1, seconds=15) - events = cloud.for_item(item) - evidence.early_completion(events, cloud.manifest["cleanupGrace"]) - cloud.finish(item) - - -def test_real_synchronous_checkpoint_settles_before_branch_join(cloud): - item = cloud.start("checkpoint") - cloud.phase(item, "BLOCKED") - held = cloud.for_item(item) - evidence.checkpoint_held(held, item["marker"] + "-checkpoint") - cloud.release(item["marker"] + "-checkpoint") - cloud.finish(item) - evidence.lifecycle(cloud.for_item(item)) - - -def test_invocation_deadline_isolation_and_capacity_recovery(cloud): - fault = cloud.start("deadline", gate="fault-" + uuid.uuid4().hex) - blocked = cloud.phase(fault, "BLOCKED")[0] - if cloud.concurrency > 1: - healthy_gate = "healthy-" + uuid.uuid4().hex - healthy = cloud.start("barrier", gate=healthy_gate) - cloud.phase(healthy, "BLOCKED") - wait_overlap(cloud, [fault, healthy], 2, blocked["environment"]) - cloud.release(healthy_gate) - cloud.finish(healthy) - evidence.lifecycle(cloud.for_item(healthy)) - # Queue recovery demand at the runtime deadline, before waiting for the - # eventually consistent service history. History delivery latency must not - # consume the worker recovery budget. - cloud.poll( - lambda: time.time() >= blocked["deadline"] - 1, - seconds=cloud.manifest["invocationTimeout"] + 5, - ) - # Put every worker slot under demand without releasing the fault. A different - # environment cannot satisfy recovery, even if the service replaces capacity. - recovery_started = time.time() - try: - _env, probes = capacity_barrier(cloud, blocked["environment"], seconds=15) - finally: - cloud.platform_timeout(fault, blocked["request"]) - bound = blocked["deadline"] + cloud.manifest["cleanupGrace"] - if recovery_started > bound: - raise evidence.CollectionError( - "Driver was too late to measure the recovery budget" - ) - assert max(e["time"] for e in probes) <= bound, ( - f"#741: worker recovery exceeded deadline + grace; probes started at {recovery_started}" - ) - cloud.poll(lambda: time.time() > bound + 1, seconds=15) - evidence.deadline( - cloud.for_item(fault), blocked["request"], cloud.manifest["cleanupGrace"] - ) - - -def test_service_timeout_retry_does_not_repeat_completed_step(cloud): - fault = cloud.start("deadline", gate="retry-" + uuid.uuid4().hex) - first = cloud.phase(fault, "BLOCKED")[0] - cloud.platform_timeout(fault, first["request"]) - # Diagnose stale side effects before waiting for a retry that may itself be - # unable to start because the old attempt has pinned the only worker. - cutoff = first["deadline"] + cloud.manifest["cleanupGrace"] - cloud.poll(lambda: time.time() > cutoff + 1, seconds=15) - evidence.deadline( - cloud.for_item(fault), first["request"], cloud.manifest["cleanupGrace"] - ) - - def retry_observed(): - entries = evidence.select(cloud.for_item(fault), "WRAPPER_ENTER") - return entries if len({e["request"] for e in entries}) >= 2 else None - - # Only a retry of the SAME durable execution satisfies this assertion. A new - # execution started by pytest must not be mislabeled as a service replay. - cloud.poll( - retry_observed, - seconds=40, - message="Service retry precondition unavailable: no second request for the same execution", - ) - events = cloud.for_item(fault) - completed = [ - e for e in evidence.select(events, "BODY") if e["operation"] == "success" - ] - assert len(completed) == 1, "Completed side effect repeated across timeout/retry" - assert {e["execution"] for e in events} == {fault["arn"]} - evidence.deadline(events, first["request"], cloud.manifest["cleanupGrace"]) - # Interrupted blocked-io effects are recorded per request, not incorrectly - # constrained to exactly-once under at-least-once step semantics. - - -def place_with_healthy_anchor(cloud, scenario): - if cloud.concurrency == 1: - return None, None, cloud.start(scenario) - for _attempt in range(3): - gate = "anchor-" + uuid.uuid4().hex - anchor = cloud.start("barrier", gate=gate) - environment = cloud.phase(anchor, "BLOCKED")[0]["environment"] - victim = cloud.start(scenario, target_environment=environment) - admission = cloud.poll( - lambda: [ - e - for e in cloud.for_item(victim) - if e["phase"] in {"PLACEMENT_ACCEPTED", "PLACEMENT_MISS"} - ], - seconds=15, - category=evidence.PlacementError, - message="Victim placement was not established before fault admission", - )[0] - if admission["phase"] == "PLACEMENT_ACCEPTED": - return anchor, gate, victim - # Only a rejected precondition is retried. The victim has checkpointed - # admission=False and returns without executing the scenario. - cloud.finish(victim) - finish_anchor(cloud, anchor, gate) - raise evidence.PlacementError( - "No same-environment pair after three placement attempts" - ) - - -def healthy_during(cloud, anchor, victim, boundary): - if anchor is None: - return - wait_overlap(cloud, [anchor, victim], 2) - cloud.poll( - lambda: [ - e - for e in cloud.for_item(anchor) - if e["phase"] == "ALIVE" and e["time"] >= boundary["time"] - ], - message="Healthy peer did not progress while victim cleanup was held", - ) - - -def finish_anchor(cloud, anchor, gate): - if anchor is not None: - cloud.release(gate) - cloud.finish(anchor) - evidence.lifecycle(cloud.for_item(anchor)) - - -def test_pending_waits_for_root_finally_and_then_replays(cloud): - anchor, anchor_gate, item = place_with_healthy_anchor(cloud, "suspend-cleanup") - blocked = cloud.phase(item, "BLOCKED")[0] - healthy_during(cloud, anchor, item, blocked) - gate = item["marker"] + "-cleanup" - evidence.scope_exit(cloud.for_item(item), gate) - cloud.release(gate) - history = cloud.finish(item) - evidence.suspension_cleanup(cloud.for_item(item), history) - finish_anchor(cloud, anchor, anchor_gate) - - -def test_nested_single_lane_pools_progress_for_all_runtime_workers(cloud): - gate = "progress-" + uuid.uuid4().hex - items = [ - cloud.start("nested-progress", gate=gate) for _ in range(cloud.concurrency) - ] - wait_overlap(cloud, items, len(items)) - cloud.release(gate) - for item in items: - cloud.phase( - item, - "PROGRESS", - seconds=20, - category=AssertionError, - message="Nested child/map/parallel work starved behind its own branch pool", - ) - cloud.finish(item) - evidence.progress(cloud.for_item(item), item["marker"]) - - -@pytest.mark.parametrize( - "scenario,status", - [("return-inflight", "SUCCEEDED"), ("failure-inflight", "FAILED")], -) -def test_root_return_or_failure_settles_inflight_work(cloud, scenario, status): - anchor, anchor_gate, item = place_with_healthy_anchor(cloud, scenario) - cloud.phase(item, "WINNER_SELECTED") - root_exit = cloud.phase(item, "USER_EXIT")[0] - healthy_during(cloud, anchor, item, root_exit) - gate = item["marker"] + "-loser" - evidence.held_loser(cloud.for_item(item)) - evidence.scope_exit(cloud.for_item(item), gate) - cloud.release(gate) - cloud.finish(item, status) - if status == "FAILED": - failure = cloud.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ - "Error" - ] - assert ( - failure["ErrorType"] == "ValueError" - and failure["ErrorMessage"] == "expected:" + item["marker"] - ) - evidence.scope_exit(cloud.for_item(item), gate, status) - finish_anchor(cloud, anchor, anchor_gate) - - -def test_abandoned_child_rejects_late_durable_operation(cloud): - item = cloud.start("late-operation") - cloud.phase(item, "WINNER_SELECTED") - evidence.held_loser(cloud.for_item(item)) - cloud.release(item["marker"] + "-loser") - cloud.poll( - lambda: [ - e - for e in cloud.for_item(item) - if e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - ], - message="Residual branch did not settle after release", - ) - history = cloud.history(item) - evidence.late_operation(cloud.for_item(item), history) - cloud.finish(item) diff --git a/lmi_tests/tests/e2e/test_observability.py b/lmi_tests/tests/e2e/test_observability.py deleted file mode 100644 index cdd8b57bf..000000000 --- a/lmi_tests/tests/e2e/test_observability.py +++ /dev/null @@ -1,140 +0,0 @@ -"""Native LMI observability acceptance; controlled carriers stay in unit tests.""" - -import time -import uuid - -import pytest - -from lmi_tests import evidence, telemetry_evidence -from lmi_tests.tests.e2e import test_lifecycle as lifecycle - - -@pytest.fixture(params=["invocation", "execution"]) -def view(request): - return request.param - - -def header(sampled="1"): - return ( - f"Root=1-{int(time.time()):08x}-{uuid.uuid4().hex[:24]};" - f"Parent={uuid.uuid4().hex[:16]};Sampled={sampled}" - ) - - -def finish(cloud, item, *, scenario="success", export_outcome="success"): - if scenario == "replay": - callback = item["runner"].wait_for_callback( - item["arn"], name="completion", timeout=90 - ) - item["runner"].send_callback_success(callback, result=item["marker"].encode()) - history = cloud.finish(item, "FAILED" if scenario == "failure" else "SUCCEEDED") - events = cloud.for_item(item) - telemetry_evidence.invocation(events, expected_outcome=export_outcome) - telemetry_evidence.terminal(events, failed=scenario == "failure") - first = evidence.select(events, "OTEL_ENTER")[0] - actual = telemetry_evidence.parse_header(first["carrier"]) - sent = telemetry_evidence.parse_header(item["trace_header"]) - assert actual["trace_id"] == sent["trace_id"], ( - "Lambda did not preserve the caller's X-Ray Root" - ) - assert actual["sampled"] == sent["sampled"], ( - "Lambda did not preserve explicit sampling" - ) - if scenario == "replay": - evidence.replay(events, history, item["marker"]) - return events - - -@pytest.mark.parametrize("sampled", ["0", "1"]) -def test_native_runtime_carrier_and_sampling(cloud, view, sampled): - item = cloud.start("success", otel_view=view, trace_header=header(sampled)) - finish(cloud, item) - - -def test_warm_process_cleanup_and_replay(cloud, view): - events = [] - for scenario in ["success", "failure", "replay"] * 2: - item = cloud.start(scenario, otel_view=view, trace_header=header()) - events.extend(finish(cloud, item, scenario=scenario)) - telemetry_evidence.warm_reuse(events) - telemetry_evidence.invocation(events) - - -def test_concurrent_worker_attribution_and_peer_end(cloud, view): - items = [ - cloud.start( - "barrier", - gate="otel-" + uuid.uuid4().hex, - otel_view=view, - trace_header=header(), - ) - for _ in range(cloud.concurrency) - ] - lifecycle.wait_overlap(cloud, items, cloud.concurrency) - for index, item in enumerate(items): - blocked = cloud.phase(item, "BLOCKED")[0] - cloud.release(blocked["gate"]) - finish(cloud, item) - for peer in items[index + 1 :]: - peer_events = cloud.for_item(peer) - assert not evidence.select(peer_events, "OTEL_EXIT"), ( - "Ending A ended a held peer" - ) - assert evidence.select(peer_events, "ALIVE"), ( - "No healthy-peer progress evidence" - ) - - -def test_export_failure_then_healthy_invocation(cloud, view): - failed = cloud.start( - "success", otel_view=view, trace_header=header(), otel_export="failure" - ) - finish(cloud, failed, export_outcome="failure") - healthy = cloud.start("success", otel_view=view, trace_header=header()) - finish(cloud, healthy) - - -def test_held_export_must_settle_before_successful_cleanup(cloud, view): - item = cloud.start( - "success", otel_view=view, trace_header=header(), otel_export="held" - ) - cloud.phase(item, "OTEL_EXPORT_HELD") - assert not evidence.select(cloud.for_item(item), "OTEL_EXIT") - cloud.release(item["marker"] + "-export") - finish(cloud, item, export_outcome="held") - - -@pytest.mark.parametrize( - "case", - [ - "deadline", - "timeout-retry", - "pending-cleanup", - "late-operation", - "return-inflight", - "failure-inflight", - ], -) -def test_plugin_enabled_lifecycle_boundaries(cloud, view, case, monkeypatch): - # Run the original assertions with real OTel registration. Runtime defects - # remain failures; neither cleanup nor instrumentation turns them into passes. - original = cloud.start - - def start(*args, **kwargs): - return original(*args, **kwargs, otel_view=view, trace_header=header()) - - monkeypatch.setattr(cloud, "start", start) - if case == "deadline": - lifecycle.test_invocation_deadline_isolation_and_capacity_recovery(cloud) - elif case == "timeout-retry": - lifecycle.test_service_timeout_retry_does_not_repeat_completed_step(cloud) - elif case == "pending-cleanup": - lifecycle.test_pending_waits_for_root_finally_and_then_replays(cloud) - elif case == "late-operation": - lifecycle.test_abandoned_child_rejects_late_durable_operation(cloud) - else: - lifecycle.test_root_return_or_failure_settles_inflight_work( - cloud, case, "FAILED" if case == "failure-inflight" else "SUCCEEDED" - ) - for item in cloud.invocations: - telemetry_evidence.invocation(cloud.for_item(item)) diff --git a/lmi_tests/tests/e2e/test_shared_driver_local.py b/lmi_tests/tests/e2e/test_shared_driver_local.py index 7e51503c3..d5816900d 100644 --- a/lmi_tests/tests/e2e/test_shared_driver_local.py +++ b/lmi_tests/tests/e2e/test_shared_driver_local.py @@ -24,6 +24,7 @@ class LocalTrace: def __init__(self): self.events = [] + self.shared = False self.payload = {} self.lock = Lock() @@ -107,9 +108,9 @@ def test_shared_callback_driver_resumes_the_original_public_sdk_execution(): "local-run", Scenario( "callback-case", - "python-lmi-5", + "lmi-lifecycle-5", profile.profile_id, - "replay-callback-retry", + "suspension-retry", roles=("victim", "resume"), ), profile, diff --git a/lmi_tests/tests/e2e/test_shared_evidence_local.py b/lmi_tests/tests/e2e/test_shared_evidence_local.py index 05f020ce4..795d8576e 100644 --- a/lmi_tests/tests/e2e/test_shared_evidence_local.py +++ b/lmi_tests/tests/e2e/test_shared_evidence_local.py @@ -4,9 +4,11 @@ import json import time + +import pytest from pathlib import Path -from lmi_tests.python_catalog import validate +from aws_durable_execution_conformance_tests.lmi.app import validate from aws_durable_execution_conformance_tests.lmi.coordinator import ( Profile, Scenario, @@ -24,7 +26,7 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( ): ledger = install_ledger(monkeypatch) payload = shared_payload("replay") - payload["sharedScenario"] = "replay-callback-retry" + payload["sharedScenario"] = "suspension-retry" payload["behaviorBudgets"] = {"observationGraceMs": 1} with DurableFunctionTestRunner(handler=fixture.handler) as runner: arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) @@ -34,6 +36,29 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( history = runner.get_execution_history( arn, include_execution_data=True ).to_dict()["Events"] + evidence, cloud = assemble( + ledger, + result, + history, + payload, + tmp_path, + "lmi-lifecycle-5", + "suspension-retry", + ) + assert evidence["collectionErrors"] == [] + assert validate(evidence) == [] + bodies = [event for event in evidence["events"] if event["kind"] == "body"] + assert len([event for event in bodies if event["operation"] == "success"]) == 1 + # Losing a real source object must not turn into an inferred complete stream. + cloud.events.pop(next(iter(cloud.events))) + incomplete = build_evidence(cloud.case_context, cloud, phase="observations") + assert incomplete["collectionErrors"] + assert incomplete["collection"]["watermarks"] == {} + + +def assemble(ledger, result, history, payload, tmp_path, requirement_id, scenario): + raw = ledger.records() + arn = next(e["execution"] for e in raw if e["phase"] == "WRAPPER_ENTER") profile = Profile( "c1", "local-function", 1, 1, "actual-local-sdk", "local-config", "reuse-worker" ) @@ -41,11 +66,13 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( "unit", Scenario( "case", - "python-lmi-5", + requirement_id, "c1", - "replay-callback-retry", - roles=("victim", "resume"), - coverage_profile="python-lifecycle", + scenario, + roles=("victim", "resume") + if scenario == "suspension-retry" + else ("victim",), + coverage_profile="standard", behavior_budgets={"observationGraceMs": 1}, ), profile, @@ -57,10 +84,11 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( ) roles = {} for index, event in enumerate(entries): - role = "victim" if index == 0 else "resume" + role = "resume" if index and scenario == "suspension-retry" else "victim" roles[event["request"]] = role context.record_attempt(event["request"], role=role, requestId=event["request"]) cloud = object.__new__(Cloud) + cloud.case_context = context cloud.manifest = { "commit": "tested-commit", "invocationTimeout": 30, @@ -89,12 +117,46 @@ def test_real_sdk_replay_and_service_history_normalize_for_shared_acceptance( cloud.retired_at = cloud.observation_ended_at cloud.artifacts_dir = Path(tmp_path) evidence = build_evidence(context, cloud, phase="final") - assert evidence["collectionErrors"] == [] - assert validate(evidence) == [] - bodies = [event for event in evidence["events"] if event["kind"] == "body"] - assert len([event for event in bodies if event["operation"] == "success"]) == 1 - # Losing a real source object must not turn into an inferred complete stream. - cloud.events.pop(next(iter(cloud.events))) - incomplete = build_evidence(context, cloud, phase="observations") - assert incomplete["collectionErrors"] - assert incomplete["collection"]["watermarks"] == {} + return evidence, cloud + + +@pytest.mark.parametrize( + "requirement_id,scenario,fixture_name,outcome", + [ + ("lmi-lifecycle-2", "normal-return", "success", "SUCCEEDED"), + ("lmi-lifecycle-7", "normal-failure", "failure", "FAILED"), + ], +) +def test_actual_disposal_outcome_and_late_admission_findings_reach_shared_validation( + monkeypatch, tmp_path, requirement_id, scenario, fixture_name, outcome +): + ledger = install_ledger(monkeypatch) + payload = shared_payload(fixture_name) + payload["sharedScenario"] = scenario + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(payload), execution_timeout=15) + arn = next( + e["execution"] for e in ledger.records() if e["phase"] == "WRAPPER_ENTER" + ) + history = runner.get_execution_history( + arn, include_execution_data=True + ).to_dict()["Events"] + assert result.status.value == outcome + evidence, _ = assemble( + ledger, result, history, payload, tmp_path, requirement_id, scenario + ) + victim = evidence["selectedRequests"]["victim"] + original = next( + item for item in evidence["attempts"] if item["requestId"] == victim + ) + assert original["outcome"] == outcome + probes = [e for e in evidence["events"] if e["kind"] == "admission_probe"] + assert any(e["probe"] == "checkpoint" for e in probes) + errors = validate(evidence) + if len(probes) == 2 and all(e["outcome"] == "REJECTED" for e in probes): + assert errors == [] + else: + # Test the harness, not an assumed SDK fix: actual acceptance or missing + # completion must remain a failed shared result, never fabricated rejection. + assert errors + assert any(e["outcome"] != "REJECTED" for e in probes) or len(probes) < 2 diff --git a/lmi_tests/tests/e2e/test_shared_exports_local.py b/lmi_tests/tests/e2e/test_shared_exports_local.py deleted file mode 100644 index faba730e8..000000000 --- a/lmi_tests/tests/e2e/test_shared_exports_local.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Actual fixture exports reach shared acceptance through the local service.""" - -import json -import time - -import pytest - -from aws_durable_execution_conformance_tests.lmi.app import validate -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - Scenario, - ScenarioContext, -) -from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner -from lmi_tests import fixture, telemetry -from lmi_tests.cloud import Cloud -from lmi_tests.shared_evidence import build_evidence -from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload -from lmi_tests.tests.unit.test_shared_telemetry import HEADER - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) -@pytest.mark.parametrize( - "scenario", ["export-success", "export-failure-recovery", "held-export"] -) -def test_real_fixture_exports_satisfy_shared_mode_contract( - monkeypatch, tmp_path, view, mode, scenario -): - ledger = install_ledger(monkeypatch) - monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) - telemetry._RUNTIMES.clear() - factory = telemetry.Runtime - # Test-only processor configuration. The deployed fixture defaults to 5 s; - # its separately declared acceptance budget is 10 s. - monkeypatch.setattr( - telemetry, - "Runtime", - lambda selected_view, selected_mode="sync": factory( - selected_view, selected_mode, flush_timeout_ms=20 - ), - ) - budgets = {"exportCompletionMs": 5000, "observationGraceMs": 1} - if mode == "async-bounded-flush": - budgets["flushWaitMs"] = 100 - roles = ("victim", "warm") if scenario == "export-failure-recovery" else ("victim",) - context = ScenarioContext( - "unit", - Scenario( - "case", - "lmi-lifecycle-16", - "c1", - scenario, - roles=roles, - otel_view=view, - coverage_profile="export-lifecycle", - export_mode=mode, - behavior_budgets=budgets, - ), - Profile("c1", "local-function", 1, 1, "actual-local-sdk", "local-config"), - ) - cloud = object.__new__(Cloud) - cloud.manifest = { - "commit": "tested-commit", - "conformanceRevision": "local-pinned-conformance", - "invocationTimeout": 30, - "wheels": { - "aws_durable_execution_sdk_python_otel-local.whl": "local-plugin-source" - }, - } - cloud.invocations, cloud.control_records = [], [] - cloud.request_roles, cloud.raw_histories, cloud.execution_results = {}, {}, {} - cloud.artifacts_dir = tmp_path - cloud._save = lambda name, data: (tmp_path / name).write_text(json.dumps(data)) - held_count = 0 - try: - with DurableFunctionTestRunner( - fixture.handler, poll_interval=0.01, skip_time=False - ) as runner: - for role in roles: - payload = shared_payload("success") - payload.update( - marker=role, - role=role, - coverageProfile="export-lifecycle", - sharedScenario=scenario, - otelView=view, - exportMode=mode, - behaviorBudgets=budgets, - otel_export="held" - if scenario == "held-export" - else "failure" - if scenario == "export-failure-recovery" and role == "victim" - else "success", - otel_export_gate="export-gate", - ) - prefix = payload["controlPrefix"] - for gate in ("release-all", "export-gate"): - if prefix + gate not in ledger.objects: - started = time.time() - ledger.put_object( - Bucket="private-evidence", Key=prefix + gate, Body=b"hold" - ) - cloud.control_records.append( - { - "gate": gate, - "action": "hold", - "started": started, - "time": time.time(), - } - ) - context.record_attempt(role, role=role) - arn = runner.run_async( - input=payload, execution_name=role, execution_timeout=15 - ) - try: - if scenario == "held-export": - ledger.wait_for("blocked", timeout=3) - if mode == "async-bounded-flush": - waited = ledger.wait_for("flush_wait_completed", timeout=3) - assert waited["observation"]["outcome"] == "timeout" - ledger.wait_for("wrapper_return", timeout=3) - assert not any( - record["observation"]["kind"] == "export_completed" - for record in ledger.observations() - ) - else: - assert not any( - record["observation"]["kind"] == "wrapper_return" - for record in ledger.observations() - ) - finally: - if scenario == "held-export": - started = time.time() - ledger.put_object( - Bucket="private-evidence", - Key=prefix + "export-gate", - Body=b"release", - ) - cloud.control_records.append( - { - "gate": "export-gate", - "action": "release", - "started": started, - "time": time.time(), - } - ) - result = runner.wait_for_result(arn, timeout=10) - assert result.status.value == "SUCCEEDED" - history = runner.get_execution_history( - arn, include_execution_data=True - ).to_dict()["Events"] - deadline = time.monotonic() + 5 - while time.monotonic() < deadline: - complete = [ - record - for record in ledger.observations() - if record["execution"] == arn - and record["observation"]["kind"] == "export_quiescence" - ] - if complete: - break - time.sleep(0.001) - assert complete, "No observed export/publication quiescence" - entry = next( - record - for record in ledger.records() - if record["execution"] == arn and record["phase"] == "WRAPPER_ENTER" - ) - request = entry["request"] - context.update_attempt(role, requestId=request, executionArn=arn) - cloud.request_roles[request] = role - cloud.invocations.append( - { - "marker": role, - "arn": arn, - "payload": payload, - "role": role, - "trace_header": HEADER, - "native_requests": [request], - } - ) - cloud.raw_histories[arn] = history - cloud.execution_results[arn] = { - "Status": result.execution_status.value, - "Result": result.result, - } - held_count += sum( - record["execution"] == arn - and record["observation"]["kind"] == "export_held" - for record in ledger.observations() - ) - for gate in ("release-all", "export-gate"): - started = time.time() - ledger.put_object( - Bucket="private-evidence", Key=prefix + gate, Body=b"release" - ) - cloud.control_records.append( - { - "gate": gate, - "action": "release", - "started": started, - "time": time.time(), - } - ) - raw = ledger.records() - cloud.events = {str(index): record for index, record in enumerate(raw)} - cloud.source_uris = { - "memory://actual-sdk-ledger/" + key - for key in ledger.objects - if "/events/" in key - } - (tmp_path / "events.json").write_text(json.dumps(raw)) - time.sleep(0.002) - cloud.observation_ended_at = time.time() - cloud.retired_at = time.time() - evidence = build_evidence(context, cloud, phase="final") - assert evidence["collectionErrors"] == [] - assert validate(evidence) == [] - if scenario == "held-export": - # Later legitimate exports happen after the gate has opened. They - # stay in evidence without being falsely labeled as held work. - assert held_count == 1 - assert ( - len( - [ - event - for event in evidence["events"] - if event["kind"] == "export_started" - ] - ) - > held_count - ) - finally: - for runtime in telemetry._RUNTIMES.values(): - runtime.provider.shutdown() - telemetry._RUNTIMES.clear() diff --git a/lmi_tests/tests/e2e/test_shared_observations_local.py b/lmi_tests/tests/e2e/test_shared_observations_local.py index 5f48e9a8d..49fa61e08 100644 --- a/lmi_tests/tests/e2e/test_shared_observations_local.py +++ b/lmi_tests/tests/e2e/test_shared_observations_local.py @@ -83,89 +83,30 @@ def test_actual_service_replay_keeps_operation_identity_and_skips_completed_bodi assert "PENDING" in {item["outcome"] for item in values(ledger, "wrapper_return")} -def test_actual_checkpoint_response_settles_before_join_observation(monkeypatch): +def test_warm_invocation_observes_late_work_under_its_original_request(monkeypatch): ledger = install_ledger(monkeypatch) - payload = shared_payload("checkpoint") - prefix = payload["controlPrefix"] - ledger.objects[prefix + "marker-checkpoint"] = b"hold" - ledger.objects[prefix + "release-all"] = b"hold" + owner = shared_payload("warm-owner") + owner["sharedScenario"] = "warm-reuse" + warm = {**owner, "scenario": "warm-probe", "role": "warm", "marker": "warm-marker"} with DurableFunctionTestRunner(handler=fixture.handler) as runner: - arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) - held = ledger.wait_for("ack_held")["observation"] - ledger.wait_for("blocked") - assert not [ - item - for item in values(ledger, "checkpoint_exit") - if item["callId"] == held["callId"] - ] - assert values(ledger, "branch_join") == [] - ledger.objects[prefix + "marker-checkpoint"] = b"release" - result = runner.wait_for_result(arn, timeout=20) - assert result.result == '"marker"' - assert values(ledger, "observation_error") == [] - exited = next( - item - for item in values(ledger, "checkpoint_exit") - if item["callId"] == held["callId"] + first = runner.run(input=json.dumps(owner), execution_timeout=15) + second = runner.run(input=json.dumps(warm), execution_timeout=15) + assert first.result == '"marker"' + assert second.result == '"warm-marker"' + raw = ledger.records() + original = next( + e for e in raw if e["phase"] == "WRAPPER_RETURN" and e["role"] == "victim" ) - joined = values(ledger, "branch_join")[0] - assert held["operationId"] in { - op.operation_id - for op in result.get_context("checkpoint-pool") - .child_operations[0] - .child_operations - } - assert joined["scopeId"] == result.get_context("checkpoint-pool").operation_id - assert ( - exited["atMs"] <= joined["atMs"] <= values(ledger, "wrapper_return")[0]["atMs"] + entered = next( + e for e in raw if e["phase"] == "WRAPPER_ENTER" and e["role"] == "warm" ) - - -def test_actual_nested_public_calls_supply_configuration_ids(monkeypatch): - ledger = install_ledger(monkeypatch) - payload = shared_payload("nested-progress") - payload["gate"] = "entry" - ledger.objects[payload["controlPrefix"] + "entry"] = b"release" - ledger.objects[payload["controlPrefix"] + "release-all"] = b"hold" - with DurableFunctionTestRunner(handler=fixture.handler) as runner: - result = runner.run(input=json.dumps(payload), execution_timeout=30) - assert result.result == '"marker"' - assert values(ledger, "observation_error") == [] - configured = values(ledger, "operation_config") - assert len(configured) == 2 and all( - item["maxConcurrency"] == 1 for item in configured + ended = next( + e for e in raw if e["phase"] == "WRAPPER_RETURN" and e["role"] == "warm" ) - started = {item["operationId"]: item for item in values(ledger, "operation_start")} - assert {started[item["operationId"]]["name"] for item in configured} == { - "inner-map", - "outer-parallel", - } - assert values(ledger, "progress")[0]["value"] == [ - ["marker:0", "marker:1"], - "marker", - ] - - -def test_actual_parent_closure_is_captured_before_late_child_attempt(monkeypatch): - ledger = install_ledger(monkeypatch) - payload = shared_payload("late-operation") - prefix = payload["controlPrefix"] - ledger.objects[prefix + "marker-loser"] = b"hold" - ledger.objects[prefix + "marker-loser-started"] = b"hold" - ledger.objects[prefix + "release-all"] = b"hold" - with DurableFunctionTestRunner(handler=fixture.handler) as runner: - arn = runner.run_async(input=json.dumps(payload), execution_timeout=30) - winner = ledger.wait_for("winner_selected")["observation"] - ledger.objects[prefix + "marker-loser"] = b"release" - runner.wait_for_result(arn, timeout=20) - assert values(ledger, "observation_error") == [] - attempted = values(ledger, "late_attempt")[0] - closed = next( - item - for item in values(ledger, "scope_closed") - if item["scopeId"] == attempted["scopeId"] + late = next( + e for e in raw if e.get("observation", {}).get("kind") == "late_activity" ) - assert closed["atMs"] <= winner["atMs"] < attempted["atMs"] - # Observe the actual SDK result, without fixing or hiding a lifecycle defect. - outcomes = values(ledger, "late_rejected") + values(ledger, "late_accepted") - assert len(outcomes) == 1 and outcomes[0]["scopeId"] == attempted["scopeId"] + assert late["request"] == original["request"] + assert late["execution"] == original["execution"] + assert late["observation"]["observedDuringRequestId"] == entered["request"] + assert original["time"] < entered["time"] <= late["time"] <= ended["time"] diff --git a/lmi_tests/tests/e2e/test_shared_telemetry_local.py b/lmi_tests/tests/e2e/test_shared_telemetry_local.py deleted file mode 100644 index 912f18a85..000000000 --- a/lmi_tests/tests/e2e/test_shared_telemetry_local.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Public SDK/plugin integration on the local durable service, without AWS.""" - -from dataclasses import replace -import time - -import pytest - -from aws_durable_execution_sdk_python import durable_execution -from aws_durable_execution_sdk_python.config import Duration -from aws_durable_execution_sdk_python.execution import DurableExecutionInvocationInput -from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner -from lmi_tests import telemetry -from lmi_tests.shared_telemetry import collect_telemetry -from lmi_tests.tests.unit.test_shared_telemetry import ( - HEADER, - SharedTrace, - attempt, - observations, - wait_quiescence, -) - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("export_mode", ["sync", "async-bounded-flush"]) -def test_public_local_runner_replay_preserves_export_ownership( - monkeypatch, view, export_mode -): - monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) - monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) - traces, bodies = [], [] - telemetry._RUNTIMES.clear() - - def business(event, context): - value = context.step( - lambda _: bodies.append(event["marker"]) or event["marker"], name="saved" - ) - context.wait(Duration.from_seconds(1), name="pause") - return value - - def entry(event, context): - invocation = ( - event - if hasattr(event, "durable_execution_arn") - else DurableExecutionInvocationInput.from_json_dict(event) - ) - # The local runtime context is a test double. Give each simulated native - # request its own ID before entering the actual public SDK handler. - context = replace(context, aws_request_id="local-" + str(len(traces))) - trace = SharedTrace( - "replay", view=view, mode=export_mode, profile="python-lifecycle" - ) - trace.identity["request"] = context.aws_request_id - trace.identity["execution"] = invocation.durable_execution_arn - traces.append(trace) - trace.emit("WRAPPER_ENTER") - - def call(plugins): - try: - return durable_execution(business, plugins=plugins)(event, context) - finally: - trace.sdk_completed_at = time.time() - - result = telemetry.invoke(trace, call) - trace.emit( - "WRAPPER_RETURN", time=trace.sdk_completed_at, status=result["Status"] - ) - return result - - try: - with DurableFunctionTestRunner( - entry, poll_interval=0.01, skip_time=False - ) as runner: - execution = runner.run_async( - input={"marker": "saved"}, execution_timeout=15 - ) - result = runner.wait_for_result(execution, timeout=15) - history = runner.get_execution_history( - execution, include_execution_data=True - ) - assert result.status.value == "SUCCEEDED" - assert bodies == ["saved"] - assert len(traces) >= 2 - assert len(history.events) > 1 - for trace in traces: - wait_quiescence(trace) - assert ( - observations(trace, "otel_factory")[0]["pluginExecutionArn"] - == execution - ) - raw = [record for trace in traces for record in trace.events] - data = collect_telemetry( - raw, - [attempt(trace) for trace in traces], - view=view, - export_mode=export_mode, - expected_sampled=True, - caller_headers={trace.identity["request"]: HEADER for trace in traces}, - observed_until_ms=time.time_ns() / 1_000_000, - ) - assert data["views"][view]["collection"]["status"] == "complete" - spans = data["views"][view]["spans"] - assert sum(record["span"]["name"] == "Invocation" for record in spans) == len( - traces - ) - assert sum(record["span"]["name"] == "Workflow" for record in spans) == 1 - assert len({binding["providerId"] for binding in data["providerBindings"]}) == 1 - finally: - for runtime in telemetry._RUNTIMES.values(): - runtime.provider.shutdown() - telemetry._RUNTIMES.clear() diff --git a/lmi_tests/tests/regressions/conftest.py b/lmi_tests/tests/regressions/conftest.py deleted file mode 100644 index cb64f770c..000000000 --- a/lmi_tests/tests/regressions/conftest.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - - -def pytest_addoption(parser): - parser.addoption("--run-lmi-regressions", action="store_true") - - -@pytest.fixture(autouse=True) -def opt_in(request): - if not request.config.getoption("--run-lmi-regressions"): - pytest.skip("#741 regressions are opt-in and expected red until the SDK fix") diff --git a/lmi_tests/tests/regressions/test_issue741.py b/lmi_tests/tests/regressions/test_issue741.py deleted file mode 100644 index 494194bb8..000000000 --- a/lmi_tests/tests/regressions/test_issue741.py +++ /dev/null @@ -1,168 +0,0 @@ -"""Assert fixed behavior, not the bug. All blocking fixture work has a final release.""" - -from concurrent.futures import ThreadPoolExecutor, wait -import json -import threading -import time -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_sdk_python import durable_execution -from aws_durable_execution_sdk_python.config import ( - CompletionConfig, - MapConfig, - ParallelConfig, -) -from aws_durable_execution_sdk_python.exceptions import InvocationError -from aws_durable_execution_sdk_python.execution import ( - DurableExecutionInvocationInputWithClient, - InitialExecutionState, -) -from aws_durable_execution_sdk_python.lambda_service import ( - CheckpointOutput, - CheckpointUpdatedExecutionState, - ContextDetails, - ExecutionDetails, - Operation, - OperationStatus, - OperationType, - StepDetails, -) - - -class MemoryClient: - def checkpoint(self, *, updates, **_kwargs): - changed = [ - Operation( - operation_id=u.operation_id, - operation_type=u.operation_type, - status={ - "START": OperationStatus.STARTED, - "SUCCEED": OperationStatus.SUCCEEDED, - "FAIL": OperationStatus.FAILED, - }[u.action.value], - name=u.name, - parent_id=u.parent_id, - sub_type=u.sub_type, - step_details=StepDetails(result=u.payload, error=u.error) - if u.operation_type == OperationType.STEP - else None, - context_details=ContextDetails(result=u.payload, error=u.error) - if u.operation_type == OperationType.CONTEXT - else None, - ) - for u in updates - ] - return CheckpointOutput( - "next-token", CheckpointUpdatedExecutionState(operations=changed) - ) - - -def invocation(): - return DurableExecutionInvocationInputWithClient( - durable_execution_arn="arn:test:execution/local", - checkpoint_token="token", - initial_execution_state=InitialExecutionState( - operations=[ - Operation( - operation_id="local", - operation_type=OperationType.EXECUTION, - status=OperationStatus.STARTED, - execution_details=ExecutionDetails(input_payload=json.dumps({})), - ) - ], - next_marker="", - ), - service_client=MemoryClient(), - ) - - -def lambda_context(remaining): - context = Mock() - context.aws_request_id = "local-request" - context.client_context = context.identity = context.tenant_id = None - context.invoked_function_arn = "test-function" - context._epoch_deadline_time_in_ms = int(time.time() * 1000) + remaining - context.get_remaining_time_in_millis.side_effect = lambda: ( - context._epoch_deadline_time_in_ms - int(time.time() * 1000) - ) - return context - - -def test_expired_invocation_starts_no_step_body(): - effects = [] - handler = durable_execution( - lambda _, c: c.step(lambda _: effects.append("write"), name="write") - ) - try: - handler(invocation(), lambda_context(-1000)) - except InvocationError: - pass # invocation interruption is permitted; a user operation failure is not - assert effects == [], ( - "#741: already-expired invocation still executed user side effect" - ) - - -@pytest.mark.parametrize("mode", ["parallel", "map", "nested"]) -def test_early_result_does_not_pin_wrapper_past_deadline(mode): - started, release, computed = threading.Event(), threading.Event(), threading.Event() - - def slow(child): - def io(_): - started.set() - assert release.wait(6), "emergency fixture timeout" - return "loser" - - return child.step(io, name="slow") - - def fast(child): - def io(_): - assert started.wait(3) - return "winner" - - return child.step(io, name="fast") - - def race(context): - if mode == "map": - return context.map( - [0, 1], - lambda c, item, _idx, _all: slow(c) if item == 0 else fast(c), - name="race", - config=MapConfig( - max_concurrency=2, - completion_config=CompletionConfig(min_successful=1), - ), - ) - return context.parallel( - [slow, fast], - name="race", - config=ParallelConfig( - max_concurrency=2, completion_config=CompletionConfig.first_successful() - ), - ) - - @durable_execution - def handler(_, context): - if mode == "nested": - context.map( - [0], - lambda c, _item, _idx, _all: c.run_in_child_context(race, name="child"), - name="outer", - ) - else: - race(context) - computed.set() - return "winner" - - with ThreadPoolExecutor(1) as caller: - future = caller.submit(handler, invocation(), lambda_context(1000)) - try: - assert computed.wait(2), "Fixture failed to select a winner" - done, _ = wait([future], timeout=2) - assert done, ( - "#741: user result computed but wrapper still joins losing branch past deadline" - ) - finally: - release.set() - future.result(timeout=3) diff --git a/lmi_tests/tests/regressions/test_late_operation.py b/lmi_tests/tests/regressions/test_late_operation.py deleted file mode 100644 index 53acf1143..000000000 --- a/lmi_tests/tests/regressions/test_late_operation.py +++ /dev/null @@ -1,9 +0,0 @@ -"""A stopped parent must reject new work created in a residual branch's finally.""" - -from lmi_tests import evidence -from lmi_tests.tests.unit.test_java_parity import run_inflight_case - - -def test_abandoned_child_cannot_start_another_step(): - events, history = run_inflight_case("late-operation", "SUCCEEDED") - evidence.late_operation(events, history) diff --git a/lmi_tests/tests/unit/conftest.py b/lmi_tests/tests/unit/conftest.py deleted file mode 100644 index cf4c61583..000000000 --- a/lmi_tests/tests/unit/conftest.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Keep Linux-only resource sampling outside portable harness unit tests.""" - -import sys -import threading - -import pytest - -from lmi_tests import fixture - - -@pytest.fixture(autouse=True) -def portable_unit_resource_snapshot(monkeypatch): - if sys.platform == "linux": - return - - def snapshot(): - # Real thread lifetime remains observable. RSS/FD cloud assertions still - # use /proc in the deployed Linux fixture; these tests do not measure it. - return { - "threads": [ - {"id": thread.ident, "name": thread.name} - for thread in threading.enumerate() - ], - "fds": 0, - "rss": 0, - } - - monkeypatch.setattr(fixture, "resources", snapshot) - monkeypatch.setattr("lmi_tests.tests.unit.test_java_parity.resources", snapshot) diff --git a/lmi_tests/tests/unit/support.py b/lmi_tests/tests/unit/support.py new file mode 100644 index 000000000..5640071e9 --- /dev/null +++ b/lmi_tests/tests/unit/support.py @@ -0,0 +1,80 @@ +"""Assert fixed behavior, not the bug. All blocking fixture work has a final release.""" + +import json +import time +from unittest.mock import Mock + + +from aws_durable_execution_sdk_python.execution import ( + DurableExecutionInvocationInputWithClient, + InitialExecutionState, +) +from aws_durable_execution_sdk_python.lambda_service import ( + CheckpointOutput, + CheckpointUpdatedExecutionState, + ContextDetails, + ExecutionDetails, + Operation, + OperationStatus, + OperationType, + StepDetails, +) + + +class MemoryClient: + def checkpoint(self, *, updates, **_kwargs): + changed = [ + Operation( + operation_id=u.operation_id, + operation_type=u.operation_type, + status={ + "START": OperationStatus.STARTED, + "SUCCEED": OperationStatus.SUCCEEDED, + "FAIL": OperationStatus.FAILED, + }[u.action.value], + name=u.name, + parent_id=u.parent_id, + sub_type=u.sub_type, + step_details=StepDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.STEP + else None, + context_details=ContextDetails(result=u.payload, error=u.error) + if u.operation_type == OperationType.CONTEXT + else None, + ) + for u in updates + ] + return CheckpointOutput( + "next-token", CheckpointUpdatedExecutionState(operations=changed) + ) + + +def invocation(): + return DurableExecutionInvocationInputWithClient( + durable_execution_arn="arn:test:execution/local", + checkpoint_token="token", + initial_execution_state=InitialExecutionState( + operations=[ + Operation( + operation_id="local", + operation_type=OperationType.EXECUTION, + status=OperationStatus.STARTED, + execution_details=ExecutionDetails(input_payload=json.dumps({})), + ) + ], + next_marker="", + ), + service_client=MemoryClient(), + ) + + +def lambda_context(remaining): + context = Mock() + context.aws_request_id = "local-request" + context.client_context = context.identity = context.tenant_id = None + context.invoked_function_arn = "test-function" + context._epoch_deadline_time_in_ms = int(time.time() * 1000) + remaining + context.get_remaining_time_in_millis.side_effect = lambda: ( + context._epoch_deadline_time_in_ms - int(time.time() * 1000) + ) + return context diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index 8909008f8..6ae0495c6 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -189,12 +189,33 @@ def test_requires_request_specific_service_timeout(driver, message): def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): assert "did not run" in summarize(tmp_path) - (tmp_path / "cloud.xml").write_text( - '' + (tmp_path / "manifest.json").write_text( + json.dumps( + { + "run": "current", + "commit": "sdk", + "runtime": "python3.14", + "concurrencies": {"c1": 1, "c2": 2}, + } + ) + ) + shared = tmp_path / "shared/standard" + shared.mkdir(parents=True) + (shared / "report.json").write_text( + json.dumps( + { + "run": {"name": "current"}, + "results": [ + {"id": "lmi-lifecycle-1", "status": "FAILED"}, + {"id": "lmi-lifecycle-2", "status": "UNCOVERED"}, + ], + "ci": {"exit_code": 1}, + } + ) ) summary = summarize(tmp_path) - assert "placement: PlacementError" in summary - assert "not-run: SKIPPED" in summary + assert "lmi-lifecycle-1: FAILED" in summary + assert "lmi-lifecycle-2: UNCOVERED" in summary assert "PASS" not in summary (tmp_path / "deploy-error.json").write_text( json.dumps({"type": "ProvisioningError"}) @@ -202,57 +223,6 @@ def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): assert "ProvisioningError" in summarize(tmp_path) -@pytest.mark.parametrize( - "scenario", ["parallel", "return-inflight", "failure-inflight", "late-operation"] -) -def test_controls_exist_before_invocation_and_shared_barriers_are_not_reset( - driver, scenario -): - driver.lam.invoke = Mock( - return_value={"StatusCode": 202, "DurableExecutionArn": "arn"} - ) - calls = [] - driver.s3.put_object.side_effect = lambda **kw: calls.append( - ("hold", kw["Key"], kw["Body"]) - ) - driver.lam.invoke.side_effect = lambda **_: ( - calls.append(("invoke",)) or {"StatusCode": 202, "DurableExecutionArn": "arn"} - ) - item = driver.start(scenario) - assert calls == [ - ("hold", "runs/unit/control/" + item["marker"] + "-loser", b"hold"), - ("hold", "runs/unit/control/" + item["marker"] + "-loser-started", b"hold"), - ("invoke",), - ] - calls.clear() - driver.start("barrier", gate="shared") - driver.start("barrier", gate="shared") - assert calls == [ - ("hold", "runs/unit/control/shared", b"hold"), - ("invoke",), - ("invoke",), - ] - assert "release-all" not in driver.gates - - -def test_finally_control_is_initialized_before_invocation(driver): - calls = [] - driver.s3.put_object.side_effect = lambda **kw: calls.append( - ("hold", kw["Key"], kw["Body"]) - ) - driver.lam.invoke = Mock( - side_effect=lambda **_: ( - calls.append(("invoke",)) - or {"StatusCode": 202, "DurableExecutionArn": "arn"} - ) - ) - item = driver.start("suspend-cleanup") - assert calls == [ - ("hold", "runs/unit/control/" + item["marker"] + "-cleanup", b"hold"), - ("invoke",), - ] - - def test_runs_share_functions_but_never_controls_or_event_prefixes(driver): other = module.Cloud({**driver.manifest, "run": "other"}, "c1") other.s3 = driver.s3 @@ -414,3 +384,46 @@ def test_case_collection_is_scoped_but_final_collection_is_complete(driver): driver.collect() driver.refresh.assert_called_with(markers=[], validate_controls=False) driver.logs.get_paginator.assert_called_once_with("filter_log_events") + + +def test_summary_rejects_other_run_reports_and_requires_the_matching_release_receipt( + tmp_path, +): + config = { + "commit": "sdk", + "runtime": "python3.14", + "concurrencies": {"c1": 1, "c2": 2}, + "run": "current", + "sharedLease": {"token": "fence"}, + } + (tmp_path / "manifest.json").write_text(json.dumps(config)) + shared = tmp_path / "shared" + (shared / "standard").mkdir(parents=True) + (shared / "standard/report.json").write_text( + json.dumps( + { + "run": {"name": "previous"}, + "results": [{"id": "lmi-lifecycle-1", "status": "PASSED"}], + } + ) + ) + receipt = shared / "ownership-released.json" + receipt.write_text( + json.dumps({"runId": "previous", "lease": config["sharedLease"]}) + ) + summary = summarize(tmp_path) + assert "another run" in summary and "PASSED" not in summary + assert "Run cleanup not confirmed" in summary + receipt.write_text(json.dumps({"runId": "current", "lease": config["sharedLease"]})) + assert "Run ownership released after retirement" in summarize(tmp_path) + + +def test_summary_does_not_accept_a_report_without_deployment_identity(tmp_path): + shared = tmp_path / "shared/standard" + shared.mkdir(parents=True) + (shared / "report.json").write_text( + json.dumps({"results": [{"id": "lmi-lifecycle-1", "status": "PASSED"}]}) + ) + summary = summarize(tmp_path) + assert "Deployment identity is missing" in summary + assert "PASSED" not in summary diff --git a/lmi_tests/tests/unit/test_control.py b/lmi_tests/tests/unit/test_control.py index d67394c95..1ba8259a4 100644 --- a/lmi_tests/tests/unit/test_control.py +++ b/lmi_tests/tests/unit/test_control.py @@ -1,10 +1,8 @@ """Regressions for the real-cloud 403 and premature-loser false positives.""" import io -import json import threading import time -from concurrent.futures import ThreadPoolExecutor from types import SimpleNamespace from unittest.mock import Mock @@ -13,10 +11,8 @@ from botocore.stub import Stubber import pytest -from lmi_tests.fixture import Trace, workflow -from lmi_tests import evidence, fixture -from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context -from aws_durable_execution_sdk_python import durable_execution +from lmi_tests.fixture import Trace +from lmi_tests import fixture @pytest.fixture @@ -140,54 +136,3 @@ def test_emitted_records_are_partitioned_by_case_and_request(trace): trace.s3.put_object.call_args.kwargs["Key"] == "runs/unit/events/case/request/000001.json" ) - - -@pytest.mark.parametrize("scenario", ["parallel", "map", "nested"]) -def test_full_fixture_establishes_a_real_blocked_loser(scenario): - """Exercise fixture wiring and SDK APIs; valid evidence survives future SDK fixes.""" - objects = { - "runs/unit/control/release-all": b"hold", - "runs/unit/control/local-loser": b"hold", - "runs/unit/control/local-loser-started": b"hold", - } - events = [] - computed = threading.Event() - lock = threading.Lock() - - def put(**kwargs): - with lock: - objects[kwargs["Key"]] = kwargs["Body"] - if kwargs["Key"].startswith("runs/unit/events/"): - event = json.loads(kwargs["Body"]) - events.append(event) - if event["phase"] == "WINNER_SELECTED": - computed.set() - - def get(**kwargs): - with lock: - return {"Body": io.BytesIO(objects[kwargs["Key"]])} - - trace = Trace.__new__(Trace) - trace.capture_lock, trace.capture_sequence = threading.Lock(), 0 - trace.s3, trace.bucket = Mock(), "test-bucket" - trace.s3.put_object.side_effect, trace.s3.get_object.side_effect = put, get - trace.identity = {"run": "unit", "marker": "local", "request": "request"} - trace.lock, trace.sequence, trace.deadline = threading.Lock(), 0, time.time() + 30 - trace.payload, trace.shared = {"scenario": scenario}, False - trace.control_prefix = "runs/unit/control/" - handler = durable_execution( - lambda _e, c: workflow({"marker": "local", "scenario": scenario}, c, trace) - ) - with ThreadPoolExecutor(1) as caller: - future = caller.submit(handler, invocation(), lambda_context(30000)) - try: - assert computed.wait(3), ( - "The controlled fixture did not reach winner selection" - ) - with lock: - snapshot = list(events) - evidence.held_loser(snapshot) - assert any(e["phase"] == "EFFECT" for e in snapshot) - finally: - put(Key="runs/unit/control/local-loser", Body=b"release") - assert future.result(timeout=3)["Status"] == "SUCCEEDED" diff --git a/lmi_tests/tests/unit/test_evidence.py b/lmi_tests/tests/unit/test_evidence.py index a0dbf9685..db1552ce5 100644 --- a/lmi_tests/tests/unit/test_evidence.py +++ b/lmi_tests/tests/unit/test_evidence.py @@ -22,32 +22,6 @@ def event(phase, time=1, **fields): } -def resource_snapshot(extra=False): - return { - "threads": [{"id": 1, "name": "MainThread"}] - + ([{"id": 2, "name": "dex-handler_0"}] if extra else []), - "fds": 8, - "rss": 1000, - } - - -def lifecycle_events(): - return [ - event("WRAPPER_ENTER", resources=resource_snapshot()), - event("WINNER_SELECTED", 2), - event("USER_EXIT", 3), - event("WRAPPER_RETURN", 4, status="SUCCEEDED", resources=resource_snapshot()), - ] - - -def race_events(): - return lifecycle_events() + [ - event("BLOCKED", 1.1, gate="a-loser"), - event("EFFECT", 1.2, gate="a-loser"), - event("WINNER_READY", 1.3), - ] - - def alive(events): return events + [ {**e, "phase": "ALIVE", "time": max(v["time"] for v in events) + 1} @@ -89,218 +63,3 @@ def test_different_environment_and_sequential_invocations_cannot_pass(): evidence.overlap(alive(events), {"a", "b"}, 2) with pytest.raises(evidence.PlacementError): evidence.overlap(alive([a, b]), {"a", "b"}, 2, environment="original-env") - - -@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "PENDING"]) -def test_lifecycle_accepts_clean_success_failure_and_suspension(status): - events = lifecycle_events() - events[-1]["status"] = status - evidence.lifecycle(events) - - -@pytest.mark.parametrize( - "phase", - [ - "BODY", - "EFFECT", - "CHECKPOINT_CALL", - "CHECKPOINT_ACK", - "CHECKPOINT_EXIT", - "POLL_EXIT", - ], -) -def test_late_activity_fails(phase): - with pytest.raises(AssertionError, match="Late invocation"): - evidence.lifecycle(lifecycle_events() + [event(phase, 5)]) - - -def test_return_with_threads_or_without_user_exit_fails(): - events = lifecycle_events() - events[-1]["resources"] = resource_snapshot(True) - with pytest.raises(AssertionError, match="workers survived"): - evidence.lifecycle(events) - with pytest.raises(AssertionError, match="User code still running"): - evidence.lifecycle([e for e in lifecycle_events() if e["phase"] != "USER_EXIT"]) - - -def test_winner_without_wrapper_return_and_emergency_release_fail(): - with pytest.raises(AssertionError, match="wrapper pinned"): - evidence.early_completion( - [e for e in race_events() if e["phase"] != "WRAPPER_RETURN"], 5 - ) - with pytest.raises(AssertionError, match="emergency"): - evidence.early_completion(race_events() + [event("ESCAPE", 3)], 5) - with pytest.raises(AssertionError, match="grace"): - evidence.early_completion(race_events(), 1) - evidence.early_completion(race_events(), 5) - - -def test_early_completion_rejects_the_cloud_false_positive(): - with pytest.raises(evidence.CollectionError, match="not held"): - evidence.early_completion( - [e for e in race_events() if e["phase"] != "EFFECT"], 5 - ) - with pytest.raises(evidence.CollectionError, match="not held"): - evidence.early_completion( - race_events() + [event("IO_EXIT", 1.25, gate="a-loser")], 5 - ) - with pytest.raises(evidence.CollectionError, match="403"): - evidence.early_completion( - race_events() - + [event("CONTROL_ERROR", gate="a-loser", error="403 Forbidden")], - 5, - ) - # SDK cancellation after the winning step is ready, before the public - # parallel call returns, is legitimate cleanup rather than a fixture error. - evidence.early_completion( - race_events() + [event("IO_EXIT", 1.5, gate="a-loser")], 5 - ) - - -def test_checkpoint_hold_requires_real_blocking_and_correlates_the_request(): - held = [event("ACK_HELD", 1), event("BLOCKED", 2, gate="checkpoint")] - evidence.checkpoint_held(held, "checkpoint") - with pytest.raises(evidence.CollectionError, match="not held"): - evidence.checkpoint_held(held[:1], "checkpoint") - with pytest.raises(AssertionError, match="before the held"): - evidence.checkpoint_held(held + [event("WRAPPER_RETURN", 3)], "checkpoint") - # An unrelated request's return is not evidence that this waiter settled. - evidence.checkpoint_held( - held + [event("WRAPPER_RETURN", 3, request="other")], "checkpoint" - ) - - -@pytest.mark.parametrize( - "phase", ["BODY", "EFFECT", "CHECKPOINT_CALL", "CHECKPOINT_ACK"] -) -def test_expired_invocation_cannot_continue_activity(phase): - with pytest.raises(AssertionError, match="#741"): - evidence.deadline([event("WRAPPER_ENTER"), event(phase, 16)], "r1", 5) - # A distinct attempt may repeat interrupted at-least-once work. - evidence.deadline([event("WRAPPER_ENTER"), event(phase, 16, request="r2")], "r1", 5) - - -def test_warm_reuse_is_required_and_resource_growth_fails(): - with pytest.raises(evidence.PlacementError): - evidence.warm_resources(lifecycle_events()) - events = [] - for index in range(3): - events += [ - {**e, "request": f"r{index}", "time": e["time"] + index * 10} - for e in lifecycle_events() - ] - evidence.warm_resources(events) - events[-1]["resources"]["fds"] = 100 - with pytest.raises(AssertionError, match="descriptors"): - evidence.warm_resources(events) - - -def test_replay_rejects_repeated_bodies_changed_identity_and_missing_history(): - events = lifecycle_events() - events[-1]["status"] = "PENDING" - events += [ - {**e, "request": "r2", "time": e["time"] + 10} for e in lifecycle_events() - ] - events += [ - event("BODY", operation=name, value="a", attempt=attempt) - for name, attempt in [ - ("success", 1), - ("failure", 1), - ("retry", 1), - ("retry", 2), - ] - ] - events += [ - event("STORED_FAILURE", message="expected:a", error_type="ValueError") - for _ in range(2) - ] - history = [ - {"Name": "success", "Id": "s", "EventType": "StepSucceeded"}, - {"Name": "failure", "Id": "f", "EventType": "StepFailed"}, - *[ - {"EventType": kind} - for kind in ("WaitSucceeded", "CallbackSucceeded", "ContextSucceeded") - ], - ] - evidence.replay(events, history, "a") - for bad_events, bad_history in [ - (events + [event("BODY", operation="success", value="a")], history), - (events, history[1:]), - ( - events, - history - + [{"Name": "success", "Id": "different", "EventType": "StepStarted"}], - ), - ]: - with pytest.raises(AssertionError): - evidence.replay(bad_events, bad_history, "a") - - -@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "PENDING"]) -def test_scope_exit_rejects_return_before_owned_work_exits(status): - events = lifecycle_events() + [event("BLOCKED", 1.1)] - events[3]["status"] = status - with pytest.raises(AssertionError, match="still active"): - evidence.scope_exit(events, "gate", status) - evidence.scope_exit(events + [event("IO_EXIT", 3.5)], "gate", status) - - -def test_suspension_requires_finally_before_pending_and_preserves_replay(): - events = [ - event("WRAPPER_ENTER", resources=resource_snapshot()), - event("BODY", 1.1, operation="success"), - event("CLEANUP_ENTER", 1.2), - event("CLEANUP_EXIT", 2), - event("USER_EXIT", 2.1), - event("WRAPPER_RETURN", 2.2, status="PENDING", resources=resource_snapshot()), - event("WRAPPER_ENTER", 3, request="r2", resources=resource_snapshot()), - event("USER_EXIT", 4, request="r2"), - event( - "WRAPPER_RETURN", - 5, - request="r2", - status="SUCCEEDED", - resources=resource_snapshot(), - ), - ] - history = [{"EventType": "WaitSucceeded"}] - evidence.suspension_cleanup(events, history) - wrong = [ - {**e, "sequence": 30} if e["phase"] == "CLEANUP_EXIT" else e for e in events - ] - with pytest.raises(AssertionError, match="finally"): - evidence.suspension_cleanup(wrong, history) - with pytest.raises(AssertionError, match="repeated"): - evidence.suspension_cleanup( - events + [event("BODY", 3.1, request="r2", operation="success")], history - ) - - -def test_progress_requires_correct_nested_results_within_budget(): - events = lifecycle_events() + [ - event("PROGRESS_BEGIN", 1.1), - event("PROGRESS", 2, value=[["a:0", "a:1"], "a"]), - ] - evidence.progress(events, "a") - with pytest.raises(AssertionError, match="crossed"): - evidence.progress(events, "different") - with pytest.raises(AssertionError, match="budget"): - evidence.progress(events, "a", seconds=0.1) - - -def test_late_rejection_after_a_side_effect_is_not_a_pass(): - events = race_events() + [ - event("LATE_ATTEMPT", 2.1), - event("LATE_REJECTED", 2.2, error="OrphanedChildException"), - ] - evidence.late_operation(events, []) - with pytest.raises(AssertionError, match="late user side effect"): - evidence.late_operation( - events + [event("BODY", 2.15, operation="late-work")], [] - ) - with pytest.raises(AssertionError, match="accepted"): - evidence.late_operation(events + [event("LATE_ACCEPTED", 2.2)], []) - with pytest.raises(AssertionError, match="reached the service"): - evidence.late_operation( - events, [{"Name": "late-work", "EventType": "StepStarted"}] - ) diff --git a/lmi_tests/tests/unit/test_fixture.py b/lmi_tests/tests/unit/test_fixture.py index 96e2919c6..fda57fd63 100644 --- a/lmi_tests/tests/unit/test_fixture.py +++ b/lmi_tests/tests/unit/test_fixture.py @@ -1,4 +1,3 @@ -from concurrent.futures import ThreadPoolExecutor import json import subprocess import sys @@ -15,7 +14,7 @@ from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner from lmi_tests.fixture import ObservedClient, workflow from lmi_tests import fixture -from lmi_tests.tests.regressions.test_issue741 import ( +from lmi_tests.tests.unit.support import ( MemoryClient, invocation, lambda_context, @@ -25,6 +24,7 @@ class LocalTrace: def __init__(self): self.events = [] + self.shared = False self.payload = {"scenario": "checkpoint", "marker": "local"} def emit(self, phase, **fields): @@ -89,8 +89,6 @@ def test_checkpoint_proxy_preserves_real_result_and_errors(): assert [e["phase"] for e in trace.events] == [ "CHECKPOINT_CALL", "CHECKPOINT_ACK", - "ACK_HELD", - "GATE", "CHECKPOINT_EXIT", ] assert "secret" not in json.dumps(trace.events) @@ -124,16 +122,6 @@ def test_replay_fixture_uses_real_public_apis_and_preserves_outcomes(): assert all(e["message"] == "expected:local" for e in failures) -def test_parallel_checkpoints_finish_with_local_runner(): - trace = LocalTrace() - handler = durable_execution(lambda e, c: workflow(e, c, trace)) - with DurableFunctionTestRunner(handler) as runner, ThreadPoolExecutor(1) as caller: - result = caller.submit( - runner.run, input={"marker": "local", "scenario": "checkpoint"} - ).result(timeout=10) - assert result.status.value == "SUCCEEDED" - - def test_cloud_entrypoint_unwraps_runtime_event_and_observes_real_wrapper(monkeypatch): trace = LocalTrace() trace.payload["scenario"] = "success" diff --git a/lmi_tests/tests/unit/test_java_parity.py b/lmi_tests/tests/unit/test_java_parity.py deleted file mode 100644 index 999971ccf..000000000 --- a/lmi_tests/tests/unit/test_java_parity.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Public-API counterparts of Java's root-finally and executor-starvation guards.""" - -import io -import json -import subprocess -import sys -import threading -import time -import uuid - -import pytest - -from aws_durable_execution_sdk_python import durable_execution -from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner -from lmi_tests import evidence -from lmi_tests.fixture import Trace, nested_progress, resources, workflow - - -class Controls: - def __init__(self): - self.condition = threading.Condition() - self.objects = {"runs/unit/control/release-all": b"hold"} - self.events = [] - - def put_object(self, *, Key, Body, **_kwargs): - with self.condition: - self.objects[Key] = Body - if Key.startswith("runs/unit/events/"): - self.events.append(json.loads(Body)) - self.condition.notify_all() - - def get_object(self, *, Key, **_kwargs): - with self.condition: - return {"Body": io.BytesIO(self.objects[Key])} - - def wait(self, phase): - with self.condition: - assert self.condition.wait_for( - lambda: any(e["phase"] == phase for e in self.events), timeout=5 - ), phase - return next(e for e in self.events if e["phase"] == phase) - - def snapshot(self): - with self.condition: - return list(self.events) - - def trace(self, marker, scenario): - trace = Trace.__new__(Trace) - trace.capture_lock, trace.capture_sequence = threading.Lock(), 0 - trace.s3, trace.bucket = self, "unit" - trace.payload = {"marker": marker, "scenario": scenario} - trace.shared = False - trace.control_prefix = "runs/unit/control/" - trace.identity = { - "run": "unit", - "marker": marker, - "request": uuid.uuid4().hex, - "environment": "local", - "process": "local", - "pid": 1, - } - trace.lock, trace.sequence, trace.deadline = ( - threading.Lock(), - 0, - time.time() + 30, - ) - return trace - - -def test_pending_waits_for_finally_then_replays_without_repeating_body(): - controls = Controls() - controls.put_object(Key="runs/unit/control/local-cleanup", Body=b"hold") - - def entry(event, context): - trace = controls.trace("local", "suspend-cleanup") - trace.emit("WRAPPER_ENTER", resources=resources()) - result = durable_execution(lambda e, c: workflow(e, c, trace))(event, context) - trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) - return result - - with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: - arn = runner.run_async(input={"marker": "local", "scenario": "suspend-cleanup"}) - try: - controls.wait("BLOCKED") - evidence.scope_exit(controls.snapshot(), "local-cleanup") - assert not [ - e for e in controls.snapshot() if e["phase"] == "WRAPPER_RETURN" - ], "PENDING returned before finally completed" - finally: - controls.put_object(Key="runs/unit/control/local-cleanup", Body=b"release") - result = runner.wait_for_result(arn, timeout=10) - history = runner.get_execution_history(arn).to_dict()["Events"] - assert result.status.value == "SUCCEEDED" - evidence.suspension_cleanup(controls.snapshot(), history) - - -@pytest.mark.parametrize( - "scenario", ["suspend-cleanup", "return-inflight", "failure-inflight"] -) -def test_placement_miss_runs_no_fault_or_finally_gate(scenario): - controls = Controls() - trace = controls.trace("local", scenario) - handler = durable_execution(lambda event, context: workflow(event, context, trace)) - with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: - result = runner.run( - input={ - "marker": "local", - "scenario": scenario, - "target_environment": "another-environment", - }, - execution_timeout=10, - ) - assert result.status.value == "SUCCEEDED" - phases = {e["phase"] for e in controls.snapshot()} - assert "PLACEMENT_MISS" in phases - assert not phases & {"BODY", "BLOCKED", "WINNER_READY", "CLEANUP_ENTER"} - - -def test_only_missed_admission_retries_environment_pair(monkeypatch): - from unittest.mock import Mock - from lmi_tests.tests.e2e import test_lifecycle - - cloud = Mock() - cloud.concurrency = 2 - first, missed, second, admitted = [ - dict(marker=name) for name in ("first", "missed", "second", "admitted") - ] - cloud.start.side_effect = [first, missed, second, admitted] - cloud.phase.side_effect = [[{"environment": "old"}], [{"environment": "new"}]] - cloud.for_item.side_effect = [ - [{"phase": "PLACEMENT_MISS"}], - [{"phase": "PLACEMENT_ACCEPTED"}], - ] - cloud.poll.side_effect = lambda predicate, **_: predicate() - finish = Mock() - monkeypatch.setattr(test_lifecycle, "finish_anchor", finish) - anchor, _gate, victim = test_lifecycle.place_with_healthy_anchor( - cloud, "return-inflight" - ) - assert anchor is second and victim is admitted - cloud.finish.assert_called_once_with(missed) - assert finish.call_count == 1 - assert cloud.start.call_args_list[1].kwargs["target_environment"] == "old" - assert cloud.start.call_args_list[3].kwargs["target_environment"] == "new" - - -def run_inflight_case(scenario, status): - controls = Controls() - for suffix in ("loser", "loser-started"): - controls.put_object(Key="runs/unit/control/local-" + suffix, Body=b"hold") - - def entry(event, context): - trace = controls.trace("local", scenario) - trace.emit("WRAPPER_ENTER", resources=resources()) - result = durable_execution(lambda e, c: workflow(e, c, trace))(event, context) - trace.emit("WRAPPER_RETURN", status=result["Status"], resources=resources()) - return result - - with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: - arn = runner.run_async(input={"marker": "local", "scenario": scenario}) - try: - controls.wait("USER_EXIT") - evidence.held_loser(controls.snapshot()) - evidence.scope_exit(controls.snapshot(), "local-loser") - finally: - controls.put_object(Key="runs/unit/control/local-loser", Body=b"release") - result = runner.wait_for_result(arn, timeout=5) - history = runner.get_execution_history(arn).to_dict()["Events"] - assert result.status.value == status - if status == "FAILED": - assert ( - result.error.message == "expected:local" - and result.error.type == "ValueError" - ) - evidence.scope_exit(controls.snapshot(), "local-loser", status) - return controls.snapshot(), history - - -@pytest.mark.parametrize( - "scenario,status", - [("return-inflight", "SUCCEEDED"), ("failure-inflight", "FAILED")], -) -def test_inflight_return_and_failure_use_real_sdk_scopes(scenario, status): - run_inflight_case(scenario, status) - - -def run_shared_roots(nested): - """Runs in a child process so a future executor deadlock cannot hang pytest.""" - rendezvous = threading.Barrier(2, timeout=3) - - class QuietTrace: - def __init__(self): - self.payload = {} - - def emit(self, *_args, **_kwargs): - pass - - def body(self, _name, value, _step=None): - return value - - @durable_execution - def handler(event, context): - context.step(lambda _: rendezvous.wait(), name="root-rendezvous") - if nested: - nested_progress(context, QuietTrace(), event["marker"]) - return context.step(lambda _: event["marker"], name="result") - - with DurableFunctionTestRunner(handler, poll_interval=0.01) as runner: - arns = [ - runner.run_async(input={"marker": marker}, execution_name=marker) - for marker in ("left", "right") - ] - for marker, arn in zip(("left", "right"), arns): - result = runner.wait_for_result(arn, timeout=5) - assert result.status.value == "SUCCEEDED" - assert json.loads(result.result) == marker - - -@pytest.mark.parametrize("nested", [False, True]) -def test_shared_decorator_roots_progress_without_starving_children(nested): - try: - result = subprocess.run( - [ - sys.executable, - "-c", - "from lmi_tests.tests.unit.test_java_parity import run_shared_roots; " - f"run_shared_roots({nested!r})", - ], - capture_output=True, - text=True, - timeout=15, - ) - except subprocess.TimeoutExpired: - pytest.fail( - "Shared root/nested work exceeded its bounded executor progress test" - ) - assert result.returncode == 0, result.stdout + result.stderr diff --git a/lmi_tests/tests/unit/test_lifecycle_mapping.py b/lmi_tests/tests/unit/test_lifecycle_mapping.py new file mode 100644 index 000000000..4a4130d62 --- /dev/null +++ b/lmi_tests/tests/unit/test_lifecycle_mapping.py @@ -0,0 +1,90 @@ +"""All seven shared contracts reach their actual SDK fixture and preserve audit facts.""" + +from copy import deepcopy +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_conformance_tests.lmi.matrix import required_cases +from lmi_tests.shared_scenarios import ScenarioDriver, FIXTURES +from lmi_tests.shared_evidence import _decoded, assertion_events, retirement_snapshot +from lmi_tests.tests.unit.test_shared_adapter import ScriptCloud, context + + +@pytest.mark.parametrize( + "cell", required_cases("python", "lifecycle"), ids=lambda c: c.scenario +) +def test_each_shared_case_uses_its_required_original_roles(cell): + cloud = ScriptCloud(cell.minimum_native_concurrency) + driver = ScenarioDriver( + context( + scenario=cell.scenario, + roles=cell.roles, + native=cell.minimum_native_concurrency, + ), + cloud, + ) + driver.wait_overlap = Mock(return_value=[]) + driver.start() + driver.place() + driver.execute() + launches = [action for action in cloud.actions if action[0] == "start"] + assert {item[2]["role"] for item in launches} == set(cell.roles) - {"resume"} + assert ( + next(item for item in launches if item[2]["role"] == "victim")[1] + == FIXTURES[cell.scenario] + ) + assert all( + "otel_view" not in item[2] and "export_mode" not in item[2] for item in launches + ) + if cell.scenario == "peer-recovery": + actions = cloud.actions + recovery_finished = next( + i for i, a in enumerate(actions) if a == ("finish", "recovery", "SUCCEEDED") + ) + peer_released = actions.index(("release", driver.gate("peer"))) + assert recovery_finished < peer_released + assert ("release", driver.gate("fault")) not in actions + if cell.scenario == "normal-failure": + assert ("finish", "victim", "FAILED") in cloud.actions + + +def test_scalar_and_wrapped_service_payloads_have_the_same_persisted_meaning(): + assert ( + _decoded({"Truncated": False, "Payload": '"value"'}) + == _decoded('"value"') + == "value" + ) + assert ( + _decoded({"Truncated": False, "Payload": "callback bytes"}) == "callback bytes" + ) + assert _decoded('{"Payload":"business field"}') == {"Payload": "business field"} + with pytest.raises(ValueError, match="Truncated"): + _decoded({"Truncated": True, "Payload": '"partial"'}) + + +@pytest.mark.parametrize( + "scenario,target", + [ + ("normal-return", "root"), + ("normal-failure", "root"), + ("warm-reuse", "late-callback"), + ], +) +def test_singular_assertion_boundary_does_not_discard_other_task_lifetimes( + scenario, target +): + original = [ + {"kind": kind, "taskId": task, "requestId": "victim", "atMs": time} + for task in ("root", "user-task-1", "late-callback") + for kind, time in (("task_start", 1), ("task_exit", 2)) + ] + retained = deepcopy(original) + events = assertion_events(original, scenario, {"victim": "victim"}) + assert [e["taskId"] for e in events if e["kind"] == "task_exit"] == [target] + assert len(events) == len(original) + assert original == retained + restored = retirement_snapshot({"events": events})["events"] + assert [ + {k: v for k, v in e.items() if k != "sourceKind"} for e in restored + ] == original diff --git a/lmi_tests/tests/unit/test_python_lifecycle.py b/lmi_tests/tests/unit/test_python_lifecycle.py deleted file mode 100644 index c1e1c2f21..000000000 --- a/lmi_tests/tests/unit/test_python_lifecycle.py +++ /dev/null @@ -1,1189 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""Complete retained controls for every Python lifecycle coverage cell.""" - -from copy import deepcopy -from typing import Any - -import pytest - -from lmi_tests.python_lifecycle import ( - SCENARIOS, - validate_lifecycle_events, - validate_lifecycle_regressions, - validate_replay_evidence, -) - - -def resources() -> dict[str, Any]: - return { - "threads": [{"id": "runtime", "name": "MainThread"}], - "fds": 10, - "rssBytes": 1_000_000, - } - - -class _Builder: - def __init__(self, scenario: str, native: int) -> None: - self.data: dict[str, Any] = { - "schemaVersion": 1, - "coverageProfile": "python-lifecycle", - "scenario": scenario, - "runtime": "python", - "runId": "run", - "caseId": "case", - "cohortId": "cohort-1", - "requirementId": f"python-lmi-{SCENARIOS[scenario]}", - "artifact": { - "sdk": "sdk@sha256:tested", - "conformance": "conformance@sha256:tested", - }, - "profile": { - "functionArn": f"arn:function:c{native}", - "configurationDigest": "sha256:configuration", - "nativeConcurrency": native, - "timeoutMs": 500, - "recoveryPolicy": "reuse-worker", - }, - "attempts": [], - "events": [], - "selectedRequests": {}, - "histories": [], - "expectedHistories": {}, - "sdkRequests": [], - "faultsCreated": [], - "resourcesCreated": [], - "observationEndedAtMs": 20_000, - "collection": { - "watermarks": { - "lifecycle": 20_000, - "sdkRequests": 20_000, - "history": 20_000, - }, - "evidenceUris": ["s3://evidence/run/case/raw.json"], - }, - "retirement": {"retiredAtMs": 21_000, "resources": [], "faults": []}, - "regression": {"budgets": {"observationGraceMs": 5}}, - } - self.requests: dict[str, dict[str, Any]] = {} - self.histories: dict[str, dict[str, Any]] = {} - - def request( - self, - role: str, - start: int = 100, - end: int = 1_000, - *, - outcome: str = "SUCCEEDED", - process: int = 1, - execution: str | None = None, - fixture: str = "success", - ) -> dict[str, Any]: - arn = execution or f"arn:execution:{role}" - marker = f"marker:{arn}" - request = { - "requestId": f"request:{role}", - "executionArn": arn, - "environmentId": "environment-1", - "workerId": f"worker-{process}", - "processId": f"process-{process}", - "pid": process + 100, - "role": role, - "attempt": 1, - "startedAtMs": start, - "endedAtMs": end, - "outcome": outcome, - "marker": marker, - "fixture": fixture, - } - self.requests[role] = request - self.data["attempts"].append(request) - self.data["selectedRequests"][role] = request["requestId"] - self.histories.setdefault( - arn, - { - "executionArn": arn, - "ownerCaseId": "case", - "status": "SUCCEEDED", - "result": marker, - "operations": [], - }, - ) - self.event(role, "wrapper_enter", start, resources=resources()) - if outcome != "TIMED_OUT": - self.event(role, "user_exit", end - 2) - self.event( - role, "wrapper_return", end, outcome=outcome, resources=resources() - ) - else: - request["deadlineAtMs"] = start + 500 - self.event( - role, "runtime_deadline", start, source="runtime", remainingTimeMs=500 - ) - self.event(role, "platform_timeout", start + 500, source="platform") - self.event(role, "task_exit", start + 550, taskId="root") - self.op(role, "success", "SUCCEEDED") - if execution is None: - self.body(role, "success", start + 5) - if outcome == "FAILED": - self.histories[arn].pop("result", None) - self.histories[arn].update( - status="FAILED", - error={"type": "ValueError", "message": "expected:" + marker}, - ) - return request - - def event(self, role: str, kind: str, time: int, **extra: Any) -> dict[str, Any]: - request = self.requests[role] - item = { - "kind": kind, - "requestId": request["requestId"], - "ownerRequestId": request["requestId"], - "executionArn": request["executionArn"], - "atMs": time, - "sequence": 0, - **extra, - } - self.data["events"].append(item) - return item - - def op( - self, - role: str, - name: str, - status: str = "SUCCEEDED", - kind: str = "STEP", - **extra: Any, - ) -> dict[str, Any]: - history = self.histories[self.requests[role]["executionArn"]] - existing = next( - (item for item in history["operations"] if item["name"] == name), None - ) - if existing is not None: - return existing - operation = { - "id": f"{history['executionArn']}:{name}", - "name": name, - "type": kind, - "parentId": None, - "status": status, - **extra, - } - history["operations"].append(operation) - return operation - - def body(self, role: str, name: str, time: int, *, attempt: int = 1) -> None: - operation = self.op(role, name, "FAILED" if name == "failure" else "SUCCEEDED") - self.event( - role, - "body", - time, - operation=name, - operationId=operation["id"], - attempt=attempt, - value=self.requests[role]["marker"], - ) - - def block( - self, - role: str, - start: int, - end: int, - *, - gate: str | None = None, - release: int | None = None, - ) -> str: - gate = gate or f"gate:{role}" - self.data["faultsCreated"].append({"id": gate, "ownerCaseId": "case"}) - self.data["retirement"]["faults"].append( - { - "id": gate, - "ownerCaseId": "case", - "releasedAtMs": release or 20_500, - "mode": "normal", - } - ) - self.event(role, "io_start", start - 10, taskId=gate, gateId=gate) - self.event(role, "blocked", start, gateId=gate) - self.event(role, "effect", start + 20, gateId=gate) - self.event(role, "alive", start + 40, gateId=gate) - self.event(role, "io_exit", end, taskId=gate, gateId=gate) - if release is not None: - self.event(role, "fault_released", release, gateId=gate, mode="normal") - return gate - - def replay( - self, - role: str, - start: int, - *, - resume_role: str = "resume", - fixture: str = "replay", - ) -> None: - original = self.request( - role, start, start + 100, outcome="PENDING", fixture=fixture - ) - self.request( - resume_role, - start + 150, - start + 300, - execution=original["executionArn"], - fixture=fixture, - ) - self.body(role, "failure", start + 10) - self.body(role, "retry", start + 20, attempt=1) - self.body(role, "retry", start + 30, attempt=2) - error = {"type": "ValueError", "message": "expected:" + original["marker"]} - self.event(role, "stored_failure", start + 15, error=error) - self.event(resume_role, "stored_failure", start + 160, error=deepcopy(error)) - for name, kind in ( - ("pause", "WAIT"), - ("completion", "CALLBACK"), - ("nested", "CONTEXT"), - ): - self.op(role, name, kind=kind) - - def finish(self) -> dict[str, Any]: - for request in self.data["attempts"]: - local = sorted( - [ - item - for item in self.data["events"] - if item["requestId"] == request["requestId"] - ], - key=lambda item: item["atMs"], - ) - for index, item in enumerate(local): - item["sequence"] = index - self.data["resourcesCreated"].append( - { - "id": request["requestId"], - "kind": "invocation", - "ownerCaseId": "case", - } - ) - self.data["histories"] = list(self.histories.values()) - for arn, history in self.histories.items(): - self.data["expectedHistories"][arn] = { - key: deepcopy(history[key]) - for key in ("status", "result", "error") - if key in history - } - self.data["resourcesCreated"].append( - {"id": arn, "kind": "execution", "ownerCaseId": "case"} - ) - self.data["retirement"]["resources"] = [ - { - **item, - "settledAtMs": 20_900, - "evidenceUri": "s3://evidence/run/case/retired.json", - "outcome": "QUIESCENT" if item["kind"] == "invocation" else "SUCCEEDED", - } - for item in self.data["resourcesCreated"] - ] - self.data["sdkRequests"] = [ - {**item, "method": "CheckpointDurableExecution", "outcome": "DISPATCHED"} - for item in self.data["events"] - if item["kind"] == "checkpoint_call" - ] - return self.data - - -def build_evidence(scenario: str, native: int = 1) -> dict[str, Any]: - """Synthetic positive controls, never claims of deployed SDK acceptance.""" - b = _Builder(scenario, native) - budgets = b.data["regression"]["budgets"] - if scenario == "replay-callback-retry": - b.replay("victim", 100) - elif scenario == "warm-lifecycle": - budgets.update( - maxThreadGrowth=1, maxFdGrowth=8, maxRssGrowthBytes=32 * 1024 * 1024 - ) - sequence = [] - for index, fixture in enumerate(["success", "failure", "replay"] * 3): - role = "victim" if index == 0 else "warm" if index == 1 else f"warm-{index}" - start = 100 + index * 400 - if fixture == "replay": - b.replay(role, start, resume_role=f"{role}-resume") - else: - b.request( - role, - start, - start + 300, - outcome="FAILED" if fixture == "failure" else "SUCCEEDED", - fixture=fixture, - ) - sequence.append( - {"executionArn": b.requests[role]["executionArn"], "fixture": fixture} - ) - b.data["regression"]["warmSequence"] = sequence - elif scenario in ("deadline-recovery", "timeout-retry"): - budgets.update(cleanupMs=100) - victim = b.request("victim", end=600, outcome="TIMED_OUT") - b.block("victim", 200, 640) - if scenario == "timeout-retry": - budgets["retryMs"] = 500 - b.request("retry", 750, 1_000, execution=victim["executionArn"]) - b.event( - "retry", - "service_retry", - 751, - source="platform", - previousRequestId=victim["requestId"], - ) - else: - budgets["recoveryMs"] = 100 - if native == 2: - b.request("peer", 110, 500, process=2) - b.block("peer", 210, 450, release=440) - b.request("recovery", 650, 1_000) - b.block("recovery", 670, 800, release=790) - if native == 2: - b.request("recovery-peer", 660, 1_010, process=2) - b.block("recovery-peer", 680, 810, release=800) - elif scenario == "pending-cleanup": - original = b.request("victim", end=500, outcome="PENDING") - b.request("resume", 600, 900, execution=original["executionArn"]) - gate = b.block("victim", 220, 400, release=390) - b.data["regression"]["gateId"] = gate - b.event("victim", "cleanup_enter", 180) - b.event("victim", "cleanup_exit", 410) - b.op("victim", "pause", kind="WAIT") - if native == 2: - b.request("peer", 110, 1_000, process=2) - b.block("peer", 210, 800, release=790) - b.event("peer", "peer_progress", 300) - elif scenario == "checkpoint-settlement": - b.request("victim") - gate = b.block("victim", 180, 810, release=800) - b.data["regression"]["gateId"] = gate - b.event("victim", "checkpoint_call", 150, callId="checkpoint-1") - operation = b.op("victim", "checkpoint-loser") - pool = b.op("victim", "checkpoint-pool", kind="CONTEXT") - b.event( - "victim", - "checkpoint_ack", - 160, - callId="checkpoint-1", - source="service", - operationIds=[operation["id"]], - ) - b.event( - "victim", - "ack_held", - 170, - callId="checkpoint-1", - gateId=gate, - operationId=operation["id"], - ) - b.event("victim", "checkpoint_exit", 820, callId="checkpoint-1") - b.event("victim", "branch_join", 830, scopeId=pool["id"]) - elif scenario.startswith("early-completion-") or scenario in ( - "return-inflight", - "failure-inflight", - "late-operation", - ): - early = scenario.startswith("early-completion-") - b.request( - "victim", - end=400 if early else 1_000, - outcome="FAILED" if scenario == "failure-inflight" else "SUCCEEDED", - ) - gate = b.block( - "victim", 200, 350 if early else 810, release=None if early else 800 - ) - b.data["regression"]["gateId"] = gate - b.event("victim", "winner_ready", 250) - b.event("victim", "winner_selected", 260) - if early: - budgets["cleanupMs"] = 200 - names = ( - ("race-map",) - if scenario.endswith("map") - else ("outer-map", "nested-child", "race-parallel") - if scenario.endswith("nested") - else ("race-parallel",) - ) - for name in names: - b.op("victim", name, kind="CONTEXT") - elif scenario == "late-operation": - scope = b.op("victim", "losing-child", kind="CONTEXT")["id"] - b.event("victim", "scope_closed", 300, scopeId=scope) - b.event("victim", "late_attempt", 830, scopeId=scope) - b.event( - "victim", - "late_rejected", - 840, - scopeId=scope, - error={"type": "OrphanedChildException"}, - ) - else: - next(item for item in b.data["events"] if item["kind"] == "user_exit")[ - "atMs" - ] = 600 - if native == 2: - b.request("peer", 110, 1_100, process=2) - b.block("peer", 210, 900, release=890) - b.event("peer", "peer_progress", 700) - else: - b.request("victim") - b.block( - "victim", - 200, - 310 if scenario == "nested-progress" else 800, - release=300 if scenario == "nested-progress" else 790, - ) - if native == 2: - b.request("peer", 110, 1_010, process=2) - b.block( - "peer", - 210, - 320 if scenario == "nested-progress" else 810, - release=310 if scenario == "nested-progress" else 800, - ) - if scenario == "nested-progress": - budgets["progressMs"] = 500 - for role, request in b.requests.items(): - b.event(role, "progress_begin", 400) - b.event( - role, - "progress", - 700, - value=[ - [request["marker"] + ":0", request["marker"] + ":1"], - request["marker"], - ], - ) - for index, name in enumerate(("outer-parallel", "inner-map")): - operation = b.op(role, name, kind="CONTEXT") - b.event( - role, - "operation_config", - 410 + index, - operationId=operation["id"], - source="public-fixture", - maxConcurrency=1, - ) - b.op(role, "progress-child", kind="CONTEXT") - return b.finish() - - -def find(value: dict[str, Any], kind: str, role: str = "victim") -> dict[str, Any]: - return next( - item - for item in value["events"] - if item["kind"] == kind and item["requestId"] == value["selectedRequests"][role] - ) - - -@pytest.mark.parametrize("native", [1, 2]) -@pytest.mark.parametrize("scenario", sorted(SCENARIOS)) -def test_all_28_lifecycle_cells_have_positive_controls( - scenario: str, native: int -) -> None: - assert validate_lifecycle_regressions(build_evidence(scenario, native)) == [] - - -@pytest.mark.parametrize("scenario", sorted(SCENARIOS)) -def test_each_case_rejects_missing_wrapper_evidence(scenario: str) -> None: - value = build_evidence(scenario) - value["events"] = [ - item for item in value["events"] if item["kind"] != "wrapper_enter" - ] - assert validate_lifecycle_regressions(value) - - -@pytest.mark.parametrize( - "scenario", - ["early-completion-parallel", "early-completion-map", "early-completion-nested"], -) -def test_early_completion_cannot_be_supplied_by_release(scenario: str) -> None: - value = build_evidence(scenario) - value["retirement"]["faults"][0]["releasedAtMs"] = 300 - assert any( - "driver release" in item for item in validate_lifecycle_regressions(value) - ) - - -def test_loser_must_be_active_before_winner() -> None: - value = build_evidence("early-completion-parallel") - value["events"] = [item for item in value["events"] if item["kind"] != "effect"] - assert any( - "actually held" in item for item in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize( - "kind", ["body", "effect", "checkpoint_ack", "checkpoint_exit", "poll_exit"] -) -def test_late_activity_and_inflight_settlement_are_separate_assertions( - kind: str, -) -> None: - value = build_evidence("return-inflight") - event = deepcopy(find(value, "user_exit")) - event.update(kind=kind, sequence=100, atMs=1_001) - value["events"].append(event) - assert any( - "after wrapper exit" in item for item in validate_lifecycle_regressions(value) - ) - - -def test_held_checkpoint_requires_real_ack_and_ordered_exit() -> None: - value = build_evidence("checkpoint-settlement") - find(value, "checkpoint_ack")["source"] = "fixture" - assert validate_lifecycle_regressions(value) - value = build_evidence("checkpoint-settlement") - find(value, "checkpoint_exit")["atMs"] = 799 - find(value, "checkpoint_exit")["sequence"] = find(value, "io_exit")["sequence"] - 1 - assert any( - "held until" in item or "clock" in item - for item in validate_lifecycle_regressions(value) - ) - - -def test_real_timeout_is_not_step_retry_or_driver_replacement() -> None: - value = build_evidence("timeout-retry") - find(value, "service_retry", "retry")["source"] = "driver" - assert any( - "driver-created" in item for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("timeout-retry") - success = deepcopy(find(value, "body")) - retry = next(item for item in value["attempts"] if item["role"] == "retry") - success.update( - requestId=retry["requestId"], - ownerRequestId=retry["requestId"], - atMs=800, - sequence=2, - ) - value["events"].append(success) - assert validate_lifecycle_regressions(value) - - -def test_recovery_needs_original_environment_all_slots_and_bound() -> None: - value = build_evidence("deadline-recovery", 2) - next(item for item in value["attempts"] if item["role"] == "recovery-peer")[ - "environmentId" - ] = "replacement" - assert any( - "unrelated environment" in item - for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("deadline-recovery") - value["regression"]["budgets"]["recoveryMs"] = 10 - assert any( - "recover within" in item for item in validate_lifecycle_regressions(value) - ) - - -def test_pending_cannot_precede_finally() -> None: - value = build_evidence("pending-cleanup") - find(value, "cleanup_exit")["sequence"] = 100 - find(value, "cleanup_exit")["atMs"] = 501 - assert any("finally" in item for item in validate_lifecycle_regressions(value)) - - -def test_nested_progress_is_not_satisfied_by_unrelated_peer() -> None: - value = build_evidence("nested-progress", 2) - find(value, "progress", "peer")["value"] = find(value, "progress")["value"] - assert any( - "crossed invocation" in item for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("nested-progress") - find(value, "operation_config")["maxConcurrency"] = 2 - assert any("single-lane" in item for item in validate_lifecycle_regressions(value)) - - -@pytest.mark.parametrize("scenario", ["return-inflight", "failure-inflight"]) -def test_inflight_work_must_settle_before_actual_wrapper_exit(scenario: str) -> None: - value = build_evidence(scenario) - find(value, "io_exit").update(sequence=100, atMs=1_001) - assert any("unsettled" in item for item in validate_lifecycle_regressions(value)) - - -def test_failure_survives_scope_settlement() -> None: - value = build_evidence("failure-inflight") - value["histories"][0]["error"]["message"] = "cleanup replaced root failure" - assert any( - "original root failure" in item - for item in validate_lifecycle_regressions(value) - ) - - -def test_abandoned_child_is_checked_before_invocation_closure() -> None: - value = build_evidence("late-operation") - find(value, "late_rejected")["error"] = {"type": "UnrelatedError"} - assert any( - "completed child scope" in item - for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("late-operation") - value["histories"][0]["operations"].append( - { - "id": "late", - "name": "late-work", - "type": "STEP", - "parentId": "child", - "status": "SUCCEEDED", - } - ) - assert any( - "reached the durable service" in item - for item in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize("counter", ["fds", "rssBytes", "threads"]) -def test_warm_resource_growth_uses_post_cache_baseline(counter: str) -> None: - value = build_evidence("warm-lifecycle") - last = sorted( - [item for item in value["events"] if item["kind"] == "wrapper_return"], - key=lambda item: item["atMs"], - )[-1] - if counter == "threads": - last["resources"][counter] += [ - {"id": str(index), "name": "retained-cache-thread"} for index in range(10) - ] - else: - last["resources"][counter] += 100_000_000 - assert any( - "post-cache allowance" in item for item in validate_lifecycle_regressions(value) - ) - - -def test_replay_requires_real_callback_wait_context_and_stored_failure() -> None: - value = build_evidence("replay-callback-retry") - value["histories"][0]["operations"] = [ - item - for item in value["histories"][0]["operations"] - if item["type"] != "CALLBACK" - ] - assert any("CALLBACK" in item for item in validate_lifecycle_regressions(value)) - value = build_evidence("replay-callback-retry") - find(value, "stored_failure", "resume")["error"]["type"] = "ChangedError" - assert any("rethrow" in item for item in validate_lifecycle_regressions(value)) - - -def test_common_lifecycle_and_per_execution_replay_are_reusable_for_otel() -> None: - value = build_evidence("replay-callback-retry") - assert validate_lifecycle_events(value) == [] - assert validate_replay_evidence(value, value["attempts"][0]["executionArn"]) == [] - - -@pytest.mark.parametrize("invalid", [None, [], False, 1, float("nan"), "bad"]) -def test_malformed_regression_data_returns_errors(invalid: Any) -> None: - for key in ("regression", "attempts", "events", "profile", "histories"): - value = build_evidence("native-concurrency") - value[key] = invalid - assert validate_lifecycle_regressions(value) - - -@pytest.mark.parametrize( - "scenario,requirement", - [ - ("runtime-carrier-sampled", 7), - ("runtime-carrier-unsampled", 7), - ("otel-warm-lifecycle", 7), - ("otel-concurrent", 7), - ("export-error-recovery", 8), - ("held-export", 8), - ], -) -def test_otel_only_cells_keep_common_lifetimes_for_extension_validation( - scenario: str, requirement: int -) -> None: - value = build_evidence("native-concurrency") - value.update(scenario=scenario, requirementId=f"python-lmi-{requirement}") - assert validate_lifecycle_regressions(value) == [] - - -def test_unrelated_ack_and_missing_branch_join_cannot_prove_checkpoint_settlement() -> ( - None -): - value = build_evidence("checkpoint-settlement") - find(value, "checkpoint_ack")["operationIds"] = ["unrelated-operation"] - assert any( - "selected real checkpoint" in item - for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("checkpoint-settlement") - value["events"] = [ - item for item in value["events"] if item["kind"] != "branch_join" - ] - assert any("branch_join" in item for item in validate_lifecycle_regressions(value)) - - -def test_static_body_names_may_repeat_in_distinct_service_contexts() -> None: - value = build_evidence("native-concurrency") - body = deepcopy(find(value, "body")) - value["histories"][0]["operations"].append( - { - "id": "child-success", - "name": "success", - "parentId": "child", - "status": "SUCCEEDED", - "type": "STEP", - } - ) - body.update(operationId="child-success", sequence=50, atMs=900) - value["events"].append(body) - for sequence, event in enumerate( - sorted(value["events"], key=lambda item: item["atMs"]) - ): - event["sequence"] = sequence - assert validate_lifecycle_events(value) == [] - - -@pytest.mark.parametrize("policy", [None, "undefined", False]) -def test_recovery_cannot_bypass_original_capacity_with_an_unknown_policy( - policy: Any, -) -> None: - value = build_evidence("deadline-recovery") - if policy is None: - value["profile"].pop("recoveryPolicy") - else: - value["profile"]["recoveryPolicy"] = policy - recovery = next(item for item in value["attempts"] if item["role"] == "recovery") - recovery.update(processId="replacement", workerId="replacement", pid=999) - assert any( - "supported recoveryPolicy" in item - for item in validate_lifecycle_regressions(value) - ) - - -def test_boolean_retry_attempt_and_lane_count_are_not_integers() -> None: - value = build_evidence("replay-callback-retry") - next( - item - for item in value["events"] - if item["kind"] == "body" and item["operation"] == "retry" - )["attempt"] = True - assert any( - "two configured attempts" in item - for item in validate_lifecycle_regressions(value) - ) - value = build_evidence("nested-progress") - find(value, "operation_config")["maxConcurrency"] = True - assert any("single-lane" in item for item in validate_lifecycle_regressions(value)) - - -def test_runtime_remaining_time_accounts_for_dispatch_time_before_wrapper_entry() -> ( - None -): - value = build_evidence("timeout-retry") - value["profile"]["timeoutMs"] = 1_000 - # The configured timer began 500ms before the observed SDK wrapper entry. - assert value["attempts"][0]["deadlineAtMs"] == 600 - assert find(value, "runtime_deadline")["remainingTimeMs"] == 500 - assert validate_lifecycle_regressions(value) == [] - - -@pytest.mark.parametrize("missing", ["deadlineAtMs", "runtime_deadline"]) -def test_actual_runtime_deadline_evidence_is_mandatory(missing: str) -> None: - value = build_evidence("deadline-recovery") - if missing == "deadlineAtMs": - value["attempts"][0].pop(missing) - else: - value["events"] = [item for item in value["events"] if item["kind"] != missing] - assert validate_lifecycle_regressions(value) - - -@pytest.mark.parametrize("change", ["source", "deadline", "remaining", "boolean"]) -def test_counterfeit_or_inconsistent_runtime_deadline_is_rejected(change: str) -> None: - value = build_evidence("timeout-retry") - observation = find(value, "runtime_deadline") - if change == "source": - observation["source"] = "input" - elif change == "deadline": - value["attempts"][0]["deadlineAtMs"] += 5 - elif change == "remaining": - observation["remainingTimeMs"] = 501 - else: - observation["remainingTimeMs"] = True - assert validate_lifecycle_regressions(value) - - -def _resequence(value: dict[str, Any]) -> None: - for request in value["attempts"]: - events = sorted( - ( - item - for item in value["events"] - if item["requestId"] == request["requestId"] - ), - key=lambda item: item["atMs"], - ) - for sequence, item in enumerate(events): - item["sequence"] = sequence - - -def _timeout_wrapper( - outcome: str = "FAILED", kind: str = "wrapper_raise" -) -> dict[str, Any]: - value = build_evidence("timeout-retry") - owner = { - key: value["events"][0][key] - for key in ("requestId", "ownerRequestId", "executionArn") - } - value["events"].extend( - [ - {**owner, "kind": "task_start", "atMs": 101, "taskId": "root"}, - {**owner, "kind": "user_exit", "atMs": 645}, - { - **owner, - "kind": kind, - "atMs": 660, - "outcome": outcome, - "resources": resources(), - }, - ] - ) - _resequence(value) - return value - - -@pytest.mark.parametrize( - "outcome,kind", - [ - ("FAILED", "wrapper_raise"), - ("SUCCEEDED", "wrapper_return"), - ("PENDING", "wrapper_return"), - ], -) -def test_actual_sdk_wrapper_outcome_remains_independent_after_platform_timeout( - outcome: str, kind: str -) -> None: - value = _timeout_wrapper(outcome, kind) - assert value["attempts"][0]["outcome"] == "TIMED_OUT" - assert validate_lifecycle_regressions(value) == [] - assert find(value, kind)["outcome"] == outcome - - -def test_timeout_wrapper_exception_does_not_erase_task_settlement_checks() -> None: - value = _timeout_wrapper() - find(value, "io_exit")["atMs"] = 670 - _resequence(value) - assert any("unsettled" in error for error in validate_lifecycle_regressions(value)) - - -def test_timeout_wrapper_does_not_erase_platform_or_late_activity_evidence() -> None: - value = _timeout_wrapper() - value["events"] = [ - item for item in value["events"] if item["kind"] != "platform_timeout" - ] - assert any( - "platform_timeout" in error for error in validate_lifecycle_regressions(value) - ) - value = _timeout_wrapper() - find(value, "body")["atMs"] = 680 - _resequence(value) - assert any( - "after wrapper exit" in error for error in validate_lifecycle_regressions(value) - ) - - -def test_wrapper_outcomes_remain_required_and_exact_without_platform_timeout() -> None: - value = _timeout_wrapper() - find(value, "wrapper_raise")["outcome"] = "invented" - assert any( - "unrecognized" in error for error in validate_lifecycle_regressions(value) - ) - value = build_evidence("native-concurrency") - find(value, "wrapper_return")["outcome"] = "FAILED" - assert any("differs" in error for error in validate_lifecycle_regressions(value)) - - -def test_public_operation_end_can_close_parent_before_winner_result_returns() -> None: - value = build_evidence("late-operation") - find(value, "scope_closed")["atMs"] = 255 - assert find(value, "winner_selected")["atMs"] == 260 - _resequence(value) - assert validate_lifecycle_regressions(value) == [] - - -@pytest.mark.parametrize("boundary", ["scope_closed", "winner_selected"]) -def test_both_parent_closure_and_winner_selection_must_precede_late_attempt( - boundary: str, -) -> None: - value = build_evidence("late-operation") - find(value, boundary)["atMs"] = 835 - _resequence(value) - assert any( - "completed child scope" in error - for error in validate_lifecycle_regressions(value) - ) - - -def _add_handshake_gate( - value: dict[str, Any], role: str = "victim", *, before: bool = False -) -> None: - """Real fixtures observe losing I/O and a separate winner-admission gate.""" - blocked = find(value, "blocked", role) - gate = blocked["gateId"] + "-handshake" - start = blocked["atMs"] + (-10 if before else 10) - for offset, kind in ( - (-1, "io_start"), - (0, "blocked"), - (1, "alive"), - (20, "fault_released"), - (21, "io_exit"), - ): - value["events"].append( - { - **blocked, - "kind": kind, - "atMs": start + offset, - "gateId": gate, - "taskId": gate, - "mode": "normal", - } - ) - value["faultsCreated"].append({"id": gate, "ownerCaseId": value["caseId"]}) - value["retirement"]["faults"].append( - { - "id": gate, - "ownerCaseId": value["caseId"], - "releasedAtMs": start + 20, - "mode": "normal", - } - ) - _resequence(value) - - -@pytest.mark.parametrize("scenario", ["return-inflight", "failure-inflight"]) -@pytest.mark.parametrize("handshake_first", [False, True]) -def test_peer_overlap_selects_declared_losing_work_gate( - scenario: str, handshake_first: bool -) -> None: - value = build_evidence(scenario, 2) - _add_handshake_gate(value, before=handshake_first) - assert validate_lifecycle_regressions(value) == [] - - -@pytest.mark.parametrize( - "damage", ["missing", "duplicate", "wrong-gate-liveness", "early-exit"] -) -def test_handshake_cannot_replace_losing_work_overlap(damage: str) -> None: - value = build_evidence("return-inflight", 2) - gate = value["regression"]["gateId"] - _add_handshake_gate(value) - if damage == "missing": - value["events"] = [ - event - for event in value["events"] - if not (event["kind"] == "blocked" and event.get("gateId") == gate) - ] - expected = "exactly one blocked" - elif damage == "duplicate": - value["events"].append(deepcopy(find(value, "blocked"))) - expected = "exactly one blocked" - elif damage == "wrong-gate-liveness": - pivot = find(value, "blocked", "peer")["atMs"] - for event in value["events"]: - if event.get("gateId") == gate and event["kind"] in {"alive", "effect"}: - event["atMs"] = pivot - 1 - expected = "native overlap lacks positive continued liveness" - else: - find(value, "io_exit")["atMs"] = find(value, "blocked", "peer")["atMs"] - 1 - expected = "native capacity stopped before overlap" - _resequence(value) - assert any(expected in error for error in validate_lifecycle_regressions(value)) - - -@pytest.mark.parametrize( - "scenario,role", - [ - ("return-inflight", "peer"), - ("native-concurrency", "victim"), - ("nested-progress", "peer"), - ("deadline-recovery", "recovery"), - ], -) -def test_undeclared_cohort_gate_ambiguity_stays_incomplete( - scenario: str, role: str -) -> None: - value = build_evidence(scenario, 2) - _add_handshake_gate(value, role) - assert any( - "exactly one blocked" in error - for error in validate_lifecycle_regressions(value) - ) - - -def _measured_timeout() -> dict[str, Any]: - value = _timeout_wrapper() - find(value, "wrapper_raise")["atMs"] = 900 - find(value, "user_exit")["atMs"] = 895 - find(value, "task_exit")["atMs"] = 898 - owner = { - key: value["events"][0][key] - for key in ("requestId", "ownerRequestId", "executionArn") - } - for offset, kind in enumerate( - ("checkpoint_call", "checkpoint_ack", "checkpoint_exit") - ): - value["events"].append( - { - **owner, - "kind": kind, - "atMs": 690 if offset == 0 else 800 + offset, - "callId": "cleanup-checkpoint", - } - ) - fault = value["faultsCreated"][0] - fault.update( - releaseRequestedAtMs=750, releaseAcknowledgedAtMs=751, releasedAtMs=752 - ) - value["retirement"]["faults"][0]["releasedAtMs"] = 752 - uri = "s3://evidence/run/case/measurements/deadline.json" - value["collection"]["evidenceUris"].append(uri) - value["regression"]["deadlineMeasurement"] = { - "selectedRequests": deepcopy(value["selectedRequests"]), - "deadlineAtMs": 600, - "cleanupCutoffAtMs": 700, - "observationCutoffAtMs": 705, - "observedAtMs": 720, - "sourceEvidenceUri": uri, - } - _resequence(value) - return value - - -def test_normal_postmeasurement_cleanup_is_not_deadline_measurement_activity() -> None: - value = _measured_timeout() - assert validate_lifecycle_regressions(value) == [] - assert any( - event["kind"] == "checkpoint_exit" and event["atMs"] == 802 - for event in value["events"] - ) - - -def test_frozen_measurement_keeps_real_prerelease_late_effect_failures() -> None: - value = _measured_timeout() - effect = deepcopy(find(value, "effect")) - effect["atMs"] = 710 - value["events"].append(effect) - _resequence(value) - assert any( - "continued side effects" in error - for error in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize( - "damage", - [ - "owner", - "deadline", - "cleanup", - "grace", - "short-window", - "after-release", - "unknown-source", - "missing-release", - "malformed", - ], -) -def test_deadline_measurement_cannot_shorten_or_substitute_the_original_window( - damage: str, -) -> None: - value = _measured_timeout() - measurement = value["regression"]["deadlineMeasurement"] - if damage == "owner": - measurement["selectedRequests"]["victim"] = value["selectedRequests"]["retry"] - elif damage == "deadline": - measurement["deadlineAtMs"] = 650 - elif damage == "cleanup": - measurement["cleanupCutoffAtMs"] = 750 - elif damage == "grace": - measurement["observationCutoffAtMs"] = 700 - elif damage == "short-window": - measurement["observedAtMs"] = 704 - elif damage == "after-release": - measurement["observedAtMs"] = 760 - elif damage == "unknown-source": - measurement["sourceEvidenceUri"] = "s3://uncollected/measurement" - elif damage == "missing-release": - value["faultsCreated"][0].pop("releaseRequestedAtMs") - else: - value["regression"]["deadlineMeasurement"] = [] - assert any( - "deadline measurement" in error - for error in validate_lifecycle_regressions(value) - ) - - -def test_legacy_timeout_evidence_keeps_full_observation_activity_checks() -> None: - value = _measured_timeout() - value["regression"].pop("deadlineMeasurement") - assert any( - "continued side effects" in error - for error in validate_lifecycle_regressions(value) - ) - - -def test_deadline_measurement_does_not_hide_work_after_wrapper_exit() -> None: - value = _measured_timeout() - find(value, "checkpoint_exit")["atMs"] = 901 - _resequence(value) - assert any( - "after wrapper exit" in error or "unsettled" in error - for error in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize("when", [710, 730, 750]) -def test_effects_until_controller_release_remain_deadline_failures(when: int) -> None: - value = _measured_timeout() - effect = deepcopy(find(value, "effect")) - effect["atMs"] = when - value["events"].append(effect) - _resequence(value) - assert any( - "continued side effects" in error - for error in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize("field", ["receipt", "acknowledgement"]) -def test_deadline_measurement_rejects_release_evidence_before_request( - field: str, -) -> None: - value = _measured_timeout() - if field == "receipt": - value["faultsCreated"][0]["releasedAtMs"] = 710 - value["retirement"]["faults"][0]["releasedAtMs"] = 710 - else: - value["faultsCreated"][0]["releaseAcknowledgedAtMs"] = 710 - assert any( - "deadline measurement" in error - for error in validate_lifecycle_regressions(value) - ) - - -@pytest.mark.parametrize( - "kind", ["body", "effect", "checkpoint_call", "poll_call", "unpaired-response"] -) -def test_postrelease_cleanup_cannot_admit_new_user_work(kind: str) -> None: - value = _measured_timeout() - if kind in {"body", "effect"}: - body = deepcopy(find(value, kind)) - body["atMs"] = 810 - value["events"].append(body) - elif kind == "unpaired-response": - value["events"] = [ - event for event in value["events"] if event["kind"] != "checkpoint_call" - ] - else: - call = find(value, "checkpoint_call") - call["atMs"] = 800 - if kind == "poll_call": - call["kind"] = kind - value["events"].remove(find(value, "checkpoint_ack")) - find(value, "checkpoint_exit")["kind"] = "poll_exit" - _resequence(value) - assert any( - "continued side effects" in error - for error in validate_lifecycle_regressions(value) - ) diff --git a/lmi_tests/tests/unit/test_python_matrix.py b/lmi_tests/tests/unit/test_python_matrix.py deleted file mode 100644 index 4a561dd4f..000000000 --- a/lmi_tests/tests/unit/test_python_matrix.py +++ /dev/null @@ -1,248 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""The Python regression profile cannot pass with missing configuration slots.""" - -from dataclasses import replace -from pathlib import Path -from runpy import run_path - -import pytest - -from lmi_tests.python_catalog import requirements, required_cases -from aws_durable_execution_conformance_tests.lmi.matrix import ( - describe_case, - duplicate_coverage, - missing_cases, -) - -BASE_SCENARIOS = { - "native-concurrency", - "replay-callback-retry", - "warm-lifecycle", - "early-completion-parallel", - "early-completion-map", - "early-completion-nested", - "checkpoint-settlement", - "deadline-recovery", - "timeout-retry", - "pending-cleanup", - "nested-progress", - "return-inflight", - "failure-inflight", - "late-operation", -} -OTEL_SCENARIOS = { - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - "otel-warm-lifecycle", - "otel-concurrent", - "export-error-recovery", - "held-export", - "deadline-recovery", - "timeout-retry", - "pending-cleanup", - "late-operation", - "return-inflight", - "failure-inflight", -} - - -def test_python_lifecycle_has_fourteen_variants_on_both_native_configurations(): - cases = required_cases("lifecycle") - assert len(cases) == 28 - assert {case.scenario for case in cases} == BASE_SCENARIOS - assert { - (case.scenario, case.native_concurrency, case.otel_view) for case in cases - } == { - (scenario, concurrency, None) - for scenario in BASE_SCENARIOS - for concurrency in (1, 2) - } - assert not {"python-lmi-2", "python-lmi-3", "python-lmi-7", "python-lmi-8"} & { - case.requirement_id for case in cases - } - - -def test_python_otel_adds_twelve_variants_in_both_views_without_replacing_baseline(): - cases = required_cases("lifecycle-otel") - baseline = required_cases("lifecycle") - instrumented = [case for case in cases if case.otel_view is not None] - assert len(cases) == 76 - assert len(instrumented) == 48 - assert {case.key for case in baseline} <= {case.key for case in cases} - assert { - (case.scenario, case.native_concurrency, case.otel_view) - for case in instrumented - } == { - (scenario, concurrency, view) - for scenario in OTEL_SCENARIOS - for concurrency in (1, 2) - for view in ("invocation", "execution") - } - assert missing_cases(cases, [case.key for case in cases]) == [] - assert duplicate_coverage(case.key for case in cases) == set() - - -def test_declared_but_unexercised_c1_cannot_be_filled_with_more_c2_results(): - cases = required_cases("lifecycle") - c2 = [case.key for case in cases if case.native_concurrency == 2] - missing = missing_cases(cases, c2 + c2) - assert len(missing) == 14 - assert {case.native_concurrency for case in missing} == {1} - assert duplicate_coverage(c2 + c2) == set(c2) - - -def test_relabeling_native_concurrency_or_view_does_not_cover_the_original_slot(): - cases = required_cases("lifecycle-otel") - selected = next( - case - for case in cases - if case.scenario == "runtime-carrier-sampled" - and case.native_concurrency == 1 - and case.otel_view == "invocation" - ) - observed = [case.key for case in cases if case != selected] - for invalid in ( - replace(selected, native_concurrency=2), - replace(selected, native_concurrency=8), - replace(selected, otel_view="execution"), - replace(selected, otel_view=None), - replace(selected, scenario="runtime-carrier-unsampled"), - replace(selected, requirement_id="python-lmi-8"), - ): - assert missing_cases(cases, [*observed, invalid.key]) == [selected] - - -def test_otel_coverage_cannot_replace_uninstrumented_lifecycle_or_other_views(): - cases = required_cases("lifecycle-otel") - observed = [case.key for case in cases if case.otel_view == "invocation"] - missing = missing_cases(cases, observed) - assert len(missing) == 52 - assert {case.otel_view for case in missing} == {None, "execution"} - - -def test_peer_roles_follow_real_python_worker_configuration(): - cases = required_cases("lifecycle") - indexed = {(case.scenario, case.native_concurrency): case for case in cases} - for name in ("native-concurrency", "nested-progress"): - assert indexed[name, 1].roles == ("victim",) - assert indexed[name, 2].roles == ("victim", "peer") - assert indexed[name, 1].minimum_native_concurrency == 1 - assert indexed[name, 2].minimum_native_concurrency == 2 - assert indexed["deadline-recovery", 1].roles == ("victim", "recovery") - assert set(indexed["deadline-recovery", 2].roles) == { - "victim", - "peer", - "recovery", - "recovery-peer", - } - assert "peer" not in indexed["pending-cleanup", 1].roles - assert "peer" in indexed["pending-cleanup", 2].roles - assert indexed["replay-callback-retry", 2].minimum_native_concurrency == 1 - - -def test_blocked_faults_and_resource_baselines_require_exclusive_windows(): - cases = required_cases("lifecycle-otel") - shared = { - "replay-callback-retry", - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - } - assert all(case.exclusive == (case.scenario not in shared) for case in cases) - for case in cases: - assert case.native_concurrency is not None - assert case.minimum_native_concurrency <= case.native_concurrency - assert case.source_tests - assert all( - test.startswith("lmi_tests/tests/e2e/") for test in case.source_tests - ) - deadline = next(case for case in cases if case.scenario == "deadline-recovery") - assert deadline.budget_names == ("observationGraceMs", "cleanupMs", "recoveryMs") - assert describe_case(deadline) == "python-lmi-4/deadline-recovery/native-1/no-otel" - - -def test_expanded_catalog_exposes_selector_union_and_python_owned_ids(): - catalog = requirements("lifecycle-otel") - assert len(catalog) == 13 - assert set(catalog["python-lmi-7"]["requiredScenarios"]) == { - "runtime-carrier-sampled", - "runtime-carrier-unsampled", - "otel-warm-lifecycle", - "otel-concurrent", - } - for definition in catalog.values(): - assert definition["runtimes"] == ["python"] - assert definition["id"].startswith("python-lmi-") - - -def test_every_expanded_slot_declares_the_validator_behavior_budget_names(): - cases = required_cases("lifecycle-otel") - for case in cases: - assert "observationGraceMs" in case.budget_names - assert not { - "cleanupGraceMs", - "placementTimeoutMs", - "retryTimeoutMs", - "progressTimeoutMs", - } & set(case.budget_names) - if case.scenario == "warm-lifecycle": - assert set(case.budget_names) == { - "observationGraceMs", - "maxThreadGrowth", - "maxFdGrowth", - "maxRssGrowthBytes", - } - if case.scenario.startswith("early-completion-"): - assert set(case.budget_names) == {"observationGraceMs", "cleanupMs"} - if case.scenario == "timeout-retry": - assert set(case.budget_names) - {"exportMs"} == { - "observationGraceMs", - "cleanupMs", - "retryMs", - } - assert ("exportMs" in case.budget_names) == (case.otel_view is not None) - - -@pytest.mark.parametrize("native", [1, 2]) -def test_warm_lifecycle_positive_control_fits_the_complete_default_role_plan(native): - build = run_path(str(Path(__file__).with_name("test_python_lifecycle.py")))[ - "build_evidence" - ] - evidence = build("warm-lifecycle", native) - required = next( - case - for case in required_cases("lifecycle") - if case.scenario == "warm-lifecycle" and case.native_concurrency == native - ) - assert len(required.roles) == 12 - assert set(evidence["selectedRequests"]) == set(required.roles) - assert {attempt["role"] for attempt in evidence["attempts"]} == set(required.roles) - - -def test_otel_warm_defaults_plan_all_six_executions_and_two_native_resumes(): - expected = { - "victim", - "warm-1", - "warm-2", - "warm-2-resume", - "warm-3", - "warm-4", - "warm", - "warm-resume", - } - cells = [ - case - for case in required_cases("lifecycle-otel") - if case.scenario == "otel-warm-lifecycle" - ] - assert len(cells) == 4 - assert all(set(case.roles) == expected for case in cells) - - -@pytest.mark.parametrize("capability", ["unknown", "", None]) -def test_unknown_python_capability_is_rejected(capability): - with pytest.raises(ValueError): - requirements(capability) - with pytest.raises(ValueError): - required_cases(capability) diff --git a/lmi_tests/tests/unit/test_python_otel.py b/lmi_tests/tests/unit/test_python_otel.py deleted file mode 100644 index 33e234fb9..000000000 --- a/lmi_tests/tests/unit/test_python_otel.py +++ /dev/null @@ -1,1157 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""Native-profile OTel cells and deliberate regressions in observed evidence.""" - -from __future__ import annotations - -from copy import deepcopy -from datetime import UTC, datetime -from typing import Any - -import pytest - -from lmi_tests.python_otel import SCENARIOS, validate_lifecycle_telemetry -from aws_durable_execution_conformance_tests_otel.model import Span, span_to_dict - - -def build_evidence( - scenario: str, native: int = 2, view: str = "invocation" -) -> dict[str, Any]: - """Synthetic observations, never a claim of deployment or cloud acceptance.""" - if scenario in SCENARIOS[6:]: - from lmi_tests.tests.unit.test_python_lifecycle import ( - build_evidence as lifecycle_evidence, - ) - - baseline = lifecycle_evidence(scenario, native) - return add_telemetry(baseline, view) - attempts: list[dict[str, Any]] = [] - sequence = [] - - def add( - role: str, - start: int, - end: int, - *, - fixture: str = "success", - process: str = "process-1", - arn: str | None = None, - outcome: str = "SUCCEEDED", - ordinal: int = 1, - ) -> dict[str, Any]: - execution = arn or f"arn:execution:{role}" - attempt = { - "requestId": f"request-{role}", - "executionArn": execution, - "role": role, - "attempt": ordinal, - "fixture": fixture, - "marker": execution, - "environmentId": "environment-1", - "processId": process, - "workerId": process, - "startedAtMs": start, - "endedAtMs": end, - "outcome": outcome, - } - attempts.append(attempt) - return attempt - - if scenario == "otel-warm-lifecycle": - for index, fixture in enumerate(["success", "failure", "replay"] * 2): - role = "victim" if index == 0 else "warm" if index == 5 else f"warm-{index}" - start = 100 + index * 200 - original = add( - role, - start, - start + (60 if fixture == "replay" else 100), - fixture=fixture, - outcome="FAILED" - if fixture == "failure" - else "PENDING" - if fixture == "replay" - else "SUCCEEDED", - ) - sequence.append( - {"executionArn": original["executionArn"], "fixture": fixture} - ) - if fixture == "replay": - add( - f"{role}-resume", - start + 80, - start + 140, - fixture=fixture, - arn=original["executionArn"], - ordinal=1, - ) - else: - add("victim", 100, 200) - if scenario == "otel-concurrent" and native == 2: - add("peer", 105, 230, process="process-2") - if scenario == "export-error-recovery": - add("warm", 300, 400) - selected = { - attempt["role"]: attempt["requestId"] - for attempt in attempts - if attempt["attempt"] == 1 - } - end = max(attempt["endedAtMs"] for attempt in attempts) + 100 - events: list[dict[str, Any]] = [] - - def emit(owner: dict[str, Any], kind: str, at: int, **details: Any) -> None: - events.append( - { - "kind": kind, - "atMs": at, - "requestId": owner["requestId"], - "ownerRequestId": owner["requestId"], - "executionArn": owner["executionArn"], - **details, - } - ) - - sampled = scenario != "runtime-carrier-unsampled" - carriers = [] - spans = [] - export_outcomes = [] - histories = [] - expected_histories = {} - executions = list(dict.fromkeys(attempt["executionArn"] for attempt in attempts)) - for arn in executions: - members = [attempt for attempt in attempts if attempt["executionArn"] == arn] - terminal = members[-1] - expectation: dict[str, Any] = {"status": terminal["outcome"]} - if terminal["outcome"] == "FAILED": - expectation["error"] = {"type": "ValueError", "message": f"expected:{arn}"} - else: - expectation["result"] = arn - operations = [] - if members[0]["fixture"] == "replay": - for name, op_type in ( - ("success", "STEP"), - ("failure", "STEP"), - ("retry", "STEP"), - ("pause", "WAIT"), - ("completion", "CALLBACK"), - ("nested", "CONTEXT"), - ): - operation: dict[str, Any] = { - "id": name, - "name": name, - "type": op_type, - "parentId": None, - "status": "FAILED" if name == "failure" else "SUCCEEDED", - } - operation["error" if name == "failure" else "result"] = ( - {"type": "ValueError", "message": f"expected:{arn}"} - if name == "failure" - else arn - ) - operations.append(operation) - expected_histories[arn] = expectation - histories.append( - { - "executionArn": arn, - "ownerCaseId": "case-1", - "operations": operations, - **expectation, - } - ) - faults = [] - for attempt in attempts: - request_id, start, finish = ( - attempt["requestId"], - attempt["startedAtMs"], - attempt["endedAtMs"], - ) - trace_id = f"{executions.index(attempt['executionArn']) + 1:032x}" - parent_id = f"{attempts.index(attempt) + 100:016x}" - carrier = f"Root=1-{trace_id[:8]}-{trace_id[8:]};Parent={parent_id};Sampled={int(sampled)}" - carriers.append( - { - "requestId": request_id, - "environmentId": attempt["environmentId"], - "processId": attempt["processId"], - "workerId": attempt["workerId"], - "source": "process-environment", - "header": carrier, - "callerHeader": carrier, - "expectedSampled": sampled, - } - ) - snapshot = { - "threads": [{"id": "main", "name": "main"}], - "fds": 3, - "rssBytes": 1000, - } - emit(attempt, "wrapper_enter", start, resources=deepcopy(snapshot)) - emit( - attempt, - "otel_factory", - start + 2, - pluginInstanceId=f"plugin-{request_id}", - pluginExecutionArn=attempt["executionArn"], - ) - emit( - attempt, - "otel_enter", - start + 1, - providerId=f"provider-{attempt['processId']}", - exporterId=f"exporter-{attempt['processId']}", - contextBefore={"traceId": None, "spanId": None}, - ) - emit(attempt, "user_exit", finish - 30) - if attempt["fixture"] == "replay": - if attempt is next( - item - for item in attempts - if item["executionArn"] == attempt["executionArn"] - ): - emit( - attempt, - "body", - start + 4, - operation="success", - operationId="success", - attempt=1, - value=attempt["marker"], - ) - emit( - attempt, - "body", - start + 5, - operation="failure", - operationId="failure", - attempt=1, - value=attempt["marker"], - ) - emit( - attempt, - "body", - start + 6, - operation="retry", - operationId="retry", - attempt=1, - value=attempt["marker"], - ) - else: - emit( - attempt, - "body", - start + 6, - operation="retry", - operationId="retry", - attempt=2, - value=attempt["marker"], - ) - emit( - attempt, - "stored_failure", - start + 7, - error={ - "type": "ValueError", - "message": f"expected:{attempt['marker']}", - }, - ) - emit( - attempt, - "otel_exit", - finish + 1, - contextAfter={"traceId": None, "spanId": None}, - exporterShutdowns=0, - ) - emit( - attempt, - "wrapper_raise" if attempt["outcome"] == "FAILED" else "wrapper_return", - finish, - outcome=attempt["outcome"], - resources=deepcopy(snapshot), - ) - failure = scenario == "export-error-recovery" and attempt["role"] == "victim" - status = ( - "unsampled" if not sampled else "export-error" if failure else "delivered" - ) - outcome: dict[str, Any] = {"requestId": request_id, "status": status} - if failure: - outcome.update( - error="observed exporter failure", fixture="bounded-export-error" - ) - export_outcomes.append(outcome) - if sampled: - members = [ - item - for item in attempts - if item["executionArn"] == attempt["executionArn"] - ] - names = ["Invocation"] + (["Workflow"] if attempt is members[-1] else []) - for index, name in enumerate(names): - span_id = f"{attempts.index(attempt) * 2 + index + 1:016x}" - span_start = ( - members[0]["startedAtMs"] + 2 if name == "Workflow" else start + 2 - ) - parent = ( - next( - item["header"].split("Parent=")[1].split(";")[0] - for item in carriers - if item["requestId"] == members[0]["requestId"] - ) - if name == "Workflow" - else parent_id - ) - span = Span( - trace_id=trace_id, - span_id=span_id, - name=name, - start_time=datetime.fromtimestamp(span_start / 1000, UTC), - end_time=datetime.fromtimestamp((finish - 25) / 1000, UTC), - parent_span_id=parent, - status="ERROR" if attempt["outcome"] == "FAILED" else "OK", - attributes={ - "durable.execution.arn": attempt["executionArn"], - "durable.invocation.first": attempt is members[0], - }, - ) - spans.append( - { - "requestId": request_id, - "sampled": True, - "span": span_to_dict(span), - } - ) - emit( - attempt, - "otel_export_started", - finish - 24, - traceId=trace_id, - spanId=span_id, - ) - emit( - attempt, - "otel_export", - finish - 9, - traceId=trace_id, - spanId=span_id, - outcome="failure" - if failure - else "held" - if scenario == "held-export" - else "success", - ) - if scenario == "held-export" and index == 0: - emit( - attempt, - "export_held", - finish - 20, - gateId="export-gate", - traceId=trace_id, - spanId=span_id, - ) - emit( - attempt, - "export_released", - finish - 10, - gateId="export-gate", - mode="normal", - ) - faults.append( - { - "id": "export-gate", - "ownerCaseId": "case-1", - "releasedAtMs": finish - 10, - } - ) - if scenario == "otel-concurrent": - emit(attempt, "blocked", 120, gateId=f"gate-{request_id}") - emit(attempt, "control_released", finish - 10, gateId=f"gate-{request_id}") - if attempt["role"] == "peer": - emit(attempt, "alive", 205) - for attempt in attempts: - owned = sorted( - (event for event in events if event["requestId"] == attempt["requestId"]), - key=lambda event: event["atMs"], - ) - for index, event in enumerate(owned, start=1): - event["sequence"] = index - resources = [ - {"id": attempt["requestId"], "kind": "invocation", "ownerCaseId": "case-1"} - for attempt in attempts - ] - resources.extend( - {"id": arn, "kind": "execution", "ownerCaseId": "case-1"} for arn in executions - ) - return { - "schemaVersion": 1, - "coverageProfile": "python-lifecycle", - "runtime": "python", - "runId": "run-1", - "caseId": "case-1", - "cohortId": "cohort-1", - "requirementId": "python-lmi-8" - if scenario in {"export-error-recovery", "held-export"} - else "python-lmi-7", - "scenario": scenario, - "otelView": view, - "artifact": { - "sdk": "sdk-commit", - "conformance": "conformance-commit", - "plugin": "plugin-commit", - }, - "profile": { - "functionArn": f"arn:function:c{native}", - "configurationDigest": f"config-{native}", - "nativeConcurrency": native, - "recoveryPolicy": "reuse-worker", - "timeoutMs": 100, - }, - "observationEndedAtMs": end, - "attempts": attempts, - "selectedRequests": selected, - "events": events, - "sdkRequests": [], - "histories": histories, - "expectedHistories": expected_histories, - "collection": { - "watermarks": {"lifecycle": end, "sdkRequests": end, "history": end}, - "evidenceUris": ["s3://example/run-1/case-1/raw"], - }, - "resourcesCreated": resources, - "faultsCreated": faults, - "regression": { - "warmSequence": sequence, - "budgets": {"observationGraceMs": 10, "exportMs": 30}, - }, - "retirement": { - "retiredAtMs": end + 10, - "resources": [ - { - **item, - "settledAtMs": end + 1, - "evidenceUri": "s3://example/run-1/case-1/retired", - "outcome": "QUIESCENT" - if item["kind"] == "invocation" - else "SUCCEEDED", - } - for item in resources - ], - "faults": [{**item, "mode": "normal"} for item in faults], - }, - "telemetry": { - "views": { - view: { - "collection": { - "status": "complete", - "scope": "cohort", - "requestIds": [attempt["requestId"] for attempt in attempts], - }, - "carriers": carriers, - "spans": spans, - "exportOutcomes": export_outcomes, - } - } - }, - } - - -@pytest.mark.parametrize("scenario", SCENARIOS) -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("native", [1, 2]) -def test_all_48_otel_cells_require_real_observations( - scenario: str, view: str, native: int -) -> None: - assert validate_lifecycle_telemetry(build_evidence(scenario, native, view)) == [] - - -@pytest.mark.parametrize( - "scenario", ["runtime-carrier-sampled", "runtime-carrier-unsampled"] -) -def test_sampled_and_unsampled_are_distinct_mandatory_cases(scenario: str) -> None: - value = build_evidence(scenario) - carrier = value["telemetry"]["views"]["invocation"]["carriers"][0] - carrier["expectedSampled"] = not carrier["expectedSampled"] - assert any( - "requires explicit Sampled" in error - for error in validate_lifecycle_telemetry(value) - ) - - -@pytest.mark.parametrize( - "header", - [ - None, - "", - "malformed", - "Root=1-11111111-111111111111111111111111;Parent=1111111111111111;Sampled=1;Sampled=0", - ], -) -def test_injected_bad_carriers_do_not_satisfy_runtime_cases(header: Any) -> None: - value = build_evidence("runtime-carrier-sampled") - value["telemetry"]["views"]["invocation"]["carriers"][0]["header"] = header - assert validate_lifecycle_telemetry(value) - - -def test_runtime_root_must_match_actual_caller_propagation() -> None: - value = build_evidence("runtime-carrier-sampled") - carrier = value["telemetry"]["views"]["invocation"]["carriers"][0] - carrier["callerHeader"] = carrier["callerHeader"].replace( - "000000000000000000000001", "000000000000000000000002" - ) - assert any("caller Root" in error for error in validate_lifecycle_telemetry(value)) - - -@pytest.mark.parametrize( - "mutation", ["missing-exit", "context-leak", "exporter-shutdown", "shared-instance"] -) -def test_actual_plugin_lifetime_is_not_inferred_from_final_trace(mutation: str) -> None: - value = build_evidence("export-error-recovery") - exits = [event for event in value["events"] if event["kind"] == "otel_exit"] - if mutation == "missing-exit": - value["events"].remove(exits[0]) - elif mutation == "context-leak": - exits[0]["contextAfter"] = {"traceId": "a" * 32, "spanId": "b" * 16} - elif mutation == "exporter-shutdown": - exits[0]["exporterShutdowns"] = 1 - else: - factories = [ - event for event in value["events"] if event["kind"] == "otel_factory" - ] - factories[1]["pluginInstanceId"] = factories[0]["pluginInstanceId"] - assert validate_lifecycle_telemetry(value) - - -def test_failure_fixture_cannot_use_all_successful_exports() -> None: - value = build_evidence("export-error-recovery") - value["telemetry"]["views"]["invocation"]["exportOutcomes"][0]["status"] = ( - "delivered" - ) - assert any( - "requires export outcome export-error" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_export_recovery_cannot_use_a_replacement_process() -> None: - value = build_evidence("export-error-recovery") - value["attempts"][1]["workerId"] = "replacement" - assert any( - "original worker/process" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_held_export_prevents_otel_and_wrapper_completion() -> None: - value = build_evidence("held-export") - next(event for event in value["events"] if event["kind"] == "export_released")[ - "atMs" - ] = 200.5 - value["faultsCreated"][0]["releasedAtMs"] = 200.5 - assert any( - "prevent SDK wrapper completion" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_emergency_release_cannot_establish_held_export_contract() -> None: - value = build_evidence("held-export") - next(event for event in value["events"] if event["kind"] == "export_released")[ - "mode" - ] = "emergency" - assert any( - "normal measurement" in error for error in validate_lifecycle_telemetry(value) - ) - - -def test_ending_victim_must_leave_live_peer_observed() -> None: - value = build_evidence("otel-concurrent", 2) - value["events"] = [event for event in value["events"] if event["kind"] != "alive"] - assert any( - "no observed progress" in error for error in validate_lifecycle_telemetry(value) - ) - - -def test_weak_supplied_assertions_cannot_skip_workflow_status() -> None: - value = build_evidence("runtime-carrier-sampled") - view = value["telemetry"]["views"]["invocation"] - view["assertions"] = {"arn:execution:victim": {"minimum_spans": 0}} - next(record for record in view["spans"] if record["span"]["name"] == "Workflow")[ - "span" - ]["status"] = "ERROR" - assert any("status" in error for error in validate_lifecycle_telemetry(value)) - - -def test_every_boundary_case_runs_plugin_validation() -> None: - value = build_evidence("pending-cleanup") - from lmi_tests.python_catalog import validate - - assert value["requirementId"] == "python-lmi-14" - assert validate(value) == [] - value["events"] = [ - event for event in value["events"] if event["kind"] != "otel_exit" - ] - assert validate(value) - - -def add_telemetry(baseline: dict[str, Any], view: str) -> dict[str, Any]: - """Attach real-shape plugin observations to a complete core positive control.""" - value = deepcopy(baseline) - value["otelView"] = view - value["artifact"]["plugin"] = "plugin-commit" - value["regression"]["budgets"]["exportMs"] = 30 - attempts = value["attempts"] - events = value["events"] - arns = list(dict.fromkeys(attempt["executionArn"] for attempt in attempts)) - carriers, spans, exports = [], [], [] - - def emit(owner: dict[str, Any], kind: str, when: float, **extra: Any) -> None: - events.append( - { - "requestId": owner["requestId"], - "ownerRequestId": owner["requestId"], - "executionArn": owner["executionArn"], - "kind": kind, - "atMs": when, - **extra, - } - ) - - for index, attempt in enumerate(attempts): - request_id, arn = attempt["requestId"], attempt["executionArn"] - starts = [ - event - for event in events - if event["requestId"] == request_id and event["kind"] == "wrapper_enter" - ] - endings = [ - event - for event in events - if event["requestId"] == request_id - and event["kind"] in {"wrapper_return", "wrapper_raise"} - ] - start = starts[0]["atMs"] - if not endings: - # Positive cooperative-timeout observation. The platform result and - # later actual wrapper cleanup are separate retained boundaries. - root_exit = next( - event - for event in events - if event["requestId"] == request_id - and event["kind"] == "task_exit" - and event.get("taskId") == "root" - ) - finish = root_exit["atMs"] - emit(attempt, "user_exit", finish - 3) - emit( - attempt, - "wrapper_raise", - finish, - outcome=attempt["outcome"], - resources=deepcopy(starts[0]["resources"]), - ) - else: - finish = endings[0]["atMs"] - trace_id, parent_id = f"{arns.index(arn) + 1:032x}", f"{index + 100:016x}" - header = f"Root=1-{trace_id[:8]}-{trace_id[8:]};Parent={parent_id};Sampled=1" - carriers.append( - { - "requestId": request_id, - "environmentId": attempt["environmentId"], - "workerId": attempt["workerId"], - "processId": attempt["processId"], - "source": "process-environment", - "header": header, - "expectedSampled": True, - } - ) - emit( - attempt, - "otel_factory", - start + 0.2, - pluginInstanceId=f"plugin-{request_id}", - pluginExecutionArn=arn, - ) - emit( - attempt, - "otel_enter", - start + 0.1, - providerId=f"provider-{attempt['processId']}", - exporterId=f"exporter-{attempt['processId']}", - contextBefore={"traceId": None, "spanId": None}, - ) - emit( - attempt, - "otel_exit", - finish + 0.1, - exporterShutdowns=0, - contextAfter={"traceId": None, "spanId": None}, - ) - members = [member for member in attempts if member["executionArn"] == arn] - for offset, name in enumerate( - ["Invocation"] - + ( - ["Workflow"] - if attempt is members[-1] - and attempt["outcome"] in {"SUCCEEDED", "FAILED"} - else [] - ) - ): - span_id = f"{2 * index + offset + 1:016x}" - original = members[0] - span_start = ( - original["startedAtMs"] + 0.2 if name == "Workflow" else start + 0.2 - ) - parent = ( - next( - carrier["header"].split("Parent=")[1].split(";")[0] - for carrier in carriers - if carrier["requestId"] == original["requestId"] - ) - if name == "Workflow" - else parent_id - ) - span = Span( - trace_id, - span_id, - name, - datetime.fromtimestamp(span_start / 1000, UTC), - datetime.fromtimestamp((finish - 0.9) / 1000, UTC), - parent_span_id=parent, - status="ERROR" - if attempt["outcome"] in {"FAILED", "TIMED_OUT"} - else "OK", - attributes={ - "durable.execution.arn": arn, - "durable.invocation.first": attempt is original, - }, - ) - spans.append( - {"requestId": request_id, "sampled": True, "span": span_to_dict(span)} - ) - emit( - attempt, - "otel_export_started", - finish - 0.8, - traceId=trace_id, - spanId=span_id, - ) - emit( - attempt, - "otel_export", - finish - 0.2, - traceId=trace_id, - spanId=span_id, - outcome="success", - ) - exports.append({"requestId": request_id, "status": "delivered"}) - for attempt in attempts: - local = sorted( - (event for event in events if event["requestId"] == attempt["requestId"]), - key=lambda event: event["atMs"], - ) - for index, event in enumerate(local, start=1): - event["sequence"] = index - value["telemetry"] = { - "views": { - view: { - "collection": { - "status": "complete", - "scope": "cohort", - "requestIds": [attempt["requestId"] for attempt in attempts], - }, - "carriers": carriers, - "spans": spans, - "exportOutcomes": exports, - } - } - } - return value - - -@pytest.mark.parametrize("scenario", SCENARIOS[6:]) -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("native", [1, 2]) -def test_boundary_overlay_passes_both_actual_lifecycle_and_otel_validators( - scenario: str, view: str, native: int -) -> None: - from lmi_tests.python_lifecycle import validate_lifecycle_regressions - - value = build_evidence(scenario, native, view) - assert validate_lifecycle_regressions(value) == [] - assert validate_lifecycle_telemetry(value) == [] - - -@pytest.mark.parametrize("scenario", SCENARIOS[:6]) -def test_otel_only_cells_preserve_actual_wrapper_lifecycle_checks( - scenario: str, -) -> None: - from lmi_tests.python_lifecycle import validate_lifecycle_events - - assert validate_lifecycle_events(build_evidence(scenario)) == [] - - -def test_export_budget_cannot_be_chosen_by_the_export_observation() -> None: - value = build_evidence("held-export") - value["regression"]["budgets"]["exportMs"] = 5 - assert any( - "immutable exportMs" in error for error in validate_lifecycle_telemetry(value) - ) - - -def test_export_start_is_required_even_when_final_trace_is_complete() -> None: - value = build_evidence("runtime-carrier-sampled") - value["events"] = [ - event for event in value["events"] if event["kind"] != "otel_export_started" - ] - assert any( - "export start" in error or "exporter start" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_factory_runs_inside_sdk_call_after_observation_entry() -> None: - value = build_evidence("runtime-carrier-sampled") - next(event for event in value["events"] if event["kind"] == "otel_factory")[ - "atMs" - ] = 100 - assert any( - "actual SDK wrapper boundary" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_post_call_diagnostic_does_not_hide_export_after_sdk_return() -> None: - value = build_evidence("runtime-carrier-sampled") - event = next(event for event in value["events"] if event["kind"] == "otel_export") - event["atMs"] = 200.5 # Actual SDK return is 200; post-call OTEL_EXIT is 201. - assert any( - "escaped its OTel lifetime" in error - for error in validate_lifecycle_telemetry(value) - ) - - -@pytest.mark.parametrize("missing", ["failure", "replay"]) -def test_warm_coverage_cannot_replace_failure_or_replay_with_success( - missing: str, -) -> None: - value = build_evidence("otel-warm-lifecycle") - next( - item - for item in value["regression"]["warmSequence"] - if item["fixture"] == missing - )["fixture"] = "success" - assert any( - "success/failure/replay" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_workflow_is_not_exported_from_an_earlier_pending_invocation() -> None: - value = build_evidence("otel-warm-lifecycle") - spans = value["telemetry"]["views"]["invocation"]["spans"] - workflow = next( - record - for record in spans - if record["span"]["name"] == "Workflow" - and record["requestId"] == "request-warm-2-resume" - ) - workflow["requestId"] = "request-warm-2" - assert any( - "terminal invocation" in error for error in validate_lifecycle_telemetry(value) - ) - - -def test_same_logical_span_reexport_does_not_claim_exactly_once_transport() -> None: - value = build_evidence("runtime-carrier-sampled") - spans = value["telemetry"]["views"]["invocation"]["spans"] - spans.append(deepcopy(spans[0])) - assert validate_lifecycle_telemetry(value) == [] - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -def test_replayed_workflow_uses_its_original_service_execution_start(view: str) -> None: - value = build_evidence("otel-warm-lifecycle", view=view) - workflow = next( - record - for record in value["telemetry"]["views"][view]["spans"] - if record["span"]["name"] == "Workflow" - and record["requestId"] == "request-warm-2-resume" - ) - original = next( - attempt - for attempt in value["attempts"] - if attempt["requestId"] == "request-warm-2" - ) - service_start = original["startedAtMs"] - 25 - next( - history - for history in value["histories"] - if history["executionArn"] == original["executionArn"] - )["startedAtMs"] = service_start - workflow["span"]["start_time"] = datetime.fromtimestamp( - service_start / 1000, UTC - ).isoformat() - assert validate_lifecycle_telemetry(value) == [] - workflow["span"]["start_time"] = datetime.fromtimestamp( - (service_start - 1) / 1000, UTC - ).isoformat() - assert any( - "outside its lifecycle observation window" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def resumed_operation_evidence(*, duplicate: bool = False) -> dict[str, Any]: - """Minimal, sanitized versions of run 43331709913's execution-view spans. - - The wait finishes before resume but is exported on resume. The timeout case - re-exports one logical step identity from two native requests, changing its - start and invocation link. Neither case re-runs the completed step body. - """ - value = build_evidence( - "timeout-retry" if duplicate else "pending-cleanup", 2, "execution" - ) - owner = value["selectedRequests"]["retry" if duplicate else "resume"] - original = value["selectedRequests"]["victim"] - attempt = next(item for item in value["attempts"] if item["requestId"] == owner) - history = next( - item - for item in value["histories"] - if item["executionArn"] == attempt["executionArn"] - ) - operation = next( - item - for item in history["operations"] - if item["name"] == ("success" if duplicate else "pause") - ) - operation.update( - startedAtMs=110 if duplicate else 250, endedAtMs=130 if duplicate else 550 - ) - view = value["telemetry"]["views"]["execution"] - workflow = next( - item - for item in view["spans"] - if item["requestId"] == owner and item["span"]["name"] == "Workflow" - ) - - def append(request_id: str, start: float, exported: float) -> None: - record = deepcopy(workflow) - record["requestId"] = request_id - invocation = next( - item["span"] - for item in view["spans"] - if item["requestId"] == request_id and item["span"]["name"] == "Invocation" - ) - record["span"].update( - name=operation["name"], - span_id="f" * 16, - parent_span_id=workflow["span"]["span_id"], - start_time=datetime.fromtimestamp(start / 1000, UTC).isoformat(), - end_time=datetime.fromtimestamp( - operation["endedAtMs"] / 1000, UTC - ).isoformat(), - attributes={ - "durable.execution.arn": attempt["executionArn"], - "durable.operation.id": operation["id"], - "durable.operation.name": operation["name"], - "durable.operation.type": operation["type"], - "durable.operation.status": "SUCCEEDED", - }, - links=[ - {"trace_id": invocation["trace_id"], "span_id": invocation["span_id"]} - ], - ) - if duplicate: - record["span"]["attributes"]["durable.attempt.number"] = ( - 1 # A logical STEP may carry this attribute. - ) - view["spans"].append(record) - for kind, when in ( - ("otel_export_started", exported), - ("otel_export", exported + 1), - ): - value["events"].append( - { - "requestId": request_id, - "ownerRequestId": request_id, - "executionArn": attempt["executionArn"], - "kind": kind, - "atMs": when, - "traceId": invocation["trace_id"], - "spanId": "f" * 16, - "outcome": "success", - } - ) - - if duplicate: - append(original, 105, 350) - append(owner, operation["startedAtMs"], 800) - for request in value["attempts"]: - events = sorted( - ( - item - for item in value["events"] - if item["requestId"] == request["requestId"] - ), - key=lambda item: item["atMs"], - ) - for sequence, event in enumerate(events, 1): - event["sequence"] = sequence - return value - - -def test_resumed_logical_wait_uses_its_actual_service_operation_interval() -> None: - assert validate_lifecycle_telemetry(resumed_operation_evidence()) == [] - - -@pytest.mark.parametrize( - "change", - [ - "missing-start", - "missing-end", - "wrong-id", - "wrong-type", - "wrong-name", - "wrong-status", - "open", - "reversed", - "nonfinite", - "duplicate", - ], -) -def test_early_logical_span_requires_matching_complete_operation_history( - change: str, -) -> None: - value = resumed_operation_evidence() - history = next( - item - for item in value["histories"] - if any(op["name"] == "pause" for op in item["operations"]) - ) - operation = next(item for item in history["operations"] if item["name"] == "pause") - if change == "missing-start": - del operation["startedAtMs"] - elif change == "missing-end": - del operation["endedAtMs"] - elif change == "wrong-id": - operation["id"] = "different-operation" - elif change == "wrong-type": - operation["type"] = "STEP" - elif change == "wrong-name": - operation["name"] = "other-wait" - elif change == "wrong-status": - operation["status"] = "FAILED" - elif change == "open": - operation["status"] = "STARTED" - value["telemetry"]["views"]["execution"]["spans"][-1]["span"]["attributes"][ - "durable.operation.status" - ] = "STARTED" - elif change == "reversed": - operation["endedAtMs"] = 200 - elif change == "nonfinite": - operation["startedAtMs"] = float("nan") - else: - history["operations"].append(deepcopy(operation)) - assert validate_lifecycle_telemetry(value) - - -@pytest.mark.parametrize( - "change", - [ - "invocation-view", - "attempt", - "user-span", - "too-early", - "too-late", - "wrong-execution", - ], -) -def test_operation_history_does_not_relax_unrelated_span_windows(change: str) -> None: - value = resumed_operation_evidence() - view = value["telemetry"]["views"]["execution"] - span = view["spans"][-1]["span"] - if change == "invocation-view": - value["otelView"] = "invocation" - value["telemetry"]["views"] = {"invocation": view} - elif change == "attempt": - span["attributes"]["durable.attempt.outcome"] = "SUCCEEDED" - elif change == "user-span": - del span["attributes"]["durable.operation.id"] - elif change == "too-early": - span["start_time"] = datetime.fromtimestamp(0.249, UTC).isoformat() - elif change == "too-late": - span["end_time"] = datetime.fromtimestamp(0.551, UTC).isoformat() - else: - history = next( - item - for item in value["histories"] - if any(op["name"] == "pause" for op in item["operations"]) - ) - history["executionArn"] = "arn:execution:unrelated" - assert any( - "observation window" in error for error in validate_lifecycle_telemetry(value) - ) - - -def test_inconsistent_cross_request_span_does_not_corrupt_either_export_ledger() -> ( - None -): - value = resumed_operation_evidence(duplicate=True) - before = deepcopy(value) - errors = validate_lifecycle_telemetry(value) - assert any("duplicate span identity" in error for error in errors) - assert not any("omit or add canonical spans" in error for error in errors) - assert not any("observation window" in error for error in errors) - assert value == before - - -def test_per_request_ledger_still_rejects_an_actually_missing_canonical_span() -> None: - value = resumed_operation_evidence(duplicate=True) - view = value["telemetry"]["views"]["execution"] - original = value["selectedRequests"]["victim"] - view["spans"] = [ - item - for item in view["spans"] - if not (item["requestId"] == original and item["span"]["span_id"] == "f" * 16) - ] - assert any( - "omit or add canonical spans" in error - for error in validate_lifecycle_telemetry(value) - ) - - -def test_identical_same_request_reexport_may_have_another_transport_receipt() -> None: - value = build_evidence("runtime-carrier-sampled", 1, "execution") - spans = value["telemetry"]["views"]["execution"]["spans"] - repeated = {**deepcopy(spans[0]), "exportId": "another-observed-export-receipt"} - spans.append(repeated) - assert validate_lifecycle_telemetry(value) == [] - - -def test_historical_span_proof_does_not_hide_sdk_exporter_shutdown_failure() -> None: - value = resumed_operation_evidence() - next(event for event in value["events"] if event["kind"] == "otel_exit")[ - "exporterShutdowns" - ] = 1 - assert any("shut down" in error for error in validate_lifecycle_telemetry(value)) - - -@pytest.mark.parametrize("name_missing", [False, True]) -def test_historical_unnamed_operation_uses_its_actual_id_without_inventing_a_name( - name_missing: bool, -) -> None: - value = resumed_operation_evidence() - history = next( - item - for item in value["histories"] - if any(op["name"] == "pause" for op in item["operations"]) - ) - operation = next(item for item in history["operations"] if item["name"] == "pause") - operation.update(name=None, type="CONTEXT") - span = value["telemetry"]["views"]["execution"]["spans"][-1]["span"] - span["name"] = operation["id"] - span["attributes"].pop("durable.operation.name") - span["attributes"]["durable.operation.type"] = "CONTEXT" - if name_missing: - del operation["name"] - assert validate_lifecycle_telemetry(value) - else: - assert validate_lifecycle_telemetry(value) == [] diff --git a/lmi_tests/tests/unit/test_python_profile.py b/lmi_tests/tests/unit/test_python_profile.py deleted file mode 100644 index 5d376e586..000000000 --- a/lmi_tests/tests/unit/test_python_profile.py +++ /dev/null @@ -1,319 +0,0 @@ -# SPDX-FileCopyrightText: 2026-present Amazon.com, Inc. or its affiliates. -# -# SPDX-License-Identifier: Apache-2.0 -"""The Python profile must exercise every declared native/view/behavior slot.""" - -import json -from functools import lru_cache -from pathlib import Path -from runpy import run_path -from copy import deepcopy -from unittest.mock import patch - -import pytest - -from aws_durable_execution_conformance_tests.lmi.app import ( - _identity_errors as shared_identity_errors, - build_report as shared_build_report, - coverage_inventory as shared_coverage_inventory, -) -from lmi_tests.python_catalog import ( - load_manifest, - required_cases, - suite_definition, - validate, -) -from aws_durable_execution_conformance_tests.report import ReportStatus - - -def build_report(plan, rows): - return shared_build_report( - plan, rows, suite=suite_definition(plan["capabilityProfile"]) - ) - - -def coverage_inventory(plan): - return shared_coverage_inventory( - plan, suite=suite_definition(plan["capabilityProfile"]) - ) - - -def _identity_errors(plan, case, evidence): - return shared_identity_errors( - plan, case, evidence, suite=suite_definition(plan["capabilityProfile"]) - ) - - -def manifest(capability="lifecycle-otel"): - plan = { - "schemaVersion": 1, - "runId": "run-1", - "runtime": "python", - "profiles": [ - { - "profile_id": f"native-{n}", - "resource_id": f"arn:function:{n}", - "capacity": n, - "native_concurrency": n, - "artifact_id": "sdk-commit", - "configuration_id": f"config-{n}", - } - for n in (1, 2) - ], - } - plan.update(coverageProfile="python-lifecycle", capabilityProfile=capability) - for profile in plan["profiles"]: - profile["recovery_policy"] = "reuse-worker" - plan["scenarios"] = [ - { - "case_id": f"case-{index}", - "requirement_id": case.requirement_id, - "scenario": case.scenario, - "profile_id": f"native-{case.native_concurrency}", - "behavior_budgets": { - name: (0 if name.startswith("max") else 1000) - for name in case.budget_names - }, - **({"otel_view": case.otel_view} if case.otel_view else {}), - } - for index, case in enumerate(required_cases(capability)) - ] - return plan - - -def results(plan): - return [ - { - "case_id": case["case_id"], - "requirement_id": case["requirement_id"], - "profile_id": case["profile_id"], - "status": "PASSED", - "errors": [], - } - for case in plan["scenarios"] - ] - - -@pytest.mark.parametrize( - "capability,count", [("lifecycle", 28), ("lifecycle-otel", 76)] -) -def test_full_python_matrix_is_explicit_and_reported(tmp_path, capability, count): - plan = manifest(capability) - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - assert load_manifest(path) == plan - assert len(coverage_inventory(plan)["required"]) == count - assert coverage_inventory(plan)["missing"] == [] - report = build_report(plan, results(plan)) - assert report.exit_code() == 0 - assert report.run.suites == ["python-lmi"] - assert all(entry.id.startswith("python-lmi-") for entry in report.entries) - - -@pytest.mark.parametrize("slot", range(76)) -def test_removing_any_required_matrix_slot_blocks_acceptance(slot): - plan = manifest() - removed = plan["scenarios"].pop(slot) - report = build_report(plan, results(plan)) - assert report.exit_code() == 1 - entry = next( - entry for entry in report.entries if entry.id == removed["requirement_id"] - ) - assert entry.status == ReportStatus.UNCOVERED - assert any("Missing mandatory matrix slot" in error for error in entry.errors) - - -@pytest.mark.parametrize( - "predicate", - [ - lambda case: case["profile_id"] == "native-1", - lambda case: case.get("otel_view") == "execution", - lambda case: case["scenario"] == "runtime-carrier-unsampled", - lambda case: case["scenario"] == "export-error-recovery", - lambda case: case["scenario"] == "held-export", - lambda case: not case.get("otel_view"), - ], -) -def test_declared_but_unexercised_profiles_or_paths_never_pass(predicate): - plan = manifest() - plan["scenarios"] = [case for case in plan["scenarios"] if not predicate(case)] - assert build_report(plan, results(plan)).exit_code() == 1 - - -def test_duplicate_c2_cases_cannot_fill_c1_coverage(): - plan = manifest() - plan["scenarios"] = [ - case for case in plan["scenarios"] if case["profile_id"] == "native-2" - ] - copies = deepcopy(plan["scenarios"]) - for case in copies: - case["case_id"] += "-duplicate" - plan["scenarios"].extend(copies) - assert len(plan["scenarios"]) == 76 - assert build_report(plan, results(plan)).exit_code() == 1 - - -def test_every_gap_and_failed_variant_stays_blocking(): - plan = manifest() - passed = results(plan) - for status in ( - "FAILED", - "NOT_IMPLEMENTED", - "INCOMPLETE_EVIDENCE", - "INFRASTRUCTURE_ERROR", - ): - current = deepcopy(passed) - current[0].update(status=status, errors=["original fixture did not pass"]) - assert build_report(plan, current).exit_code() == 1 - assert build_report(plan, passed[1:]).exit_code() == 1 - - -@pytest.mark.parametrize( - "mutation", - [ - lambda plan: plan.update(runtime="java"), - lambda plan: plan.update(coverageProfile="python-ish"), - lambda plan: plan["scenarios"][0].update(scenario="made-up-selector"), - lambda plan: plan["scenarios"][0].update(otel_view="wrong-view"), - lambda plan: plan["scenarios"][0].update(coverage_profile="standard"), - ], -) -def test_invalid_profile_is_rejected_before_admission(tmp_path, mutation): - plan = manifest() - mutation(plan) - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError): - load_manifest(path) - - -def test_evidence_identity_includes_behavior_and_view(): - plan = manifest() - case = next(case for case in plan["scenarios"] if case.get("otel_view")) - evidence = { - "coverageProfile": "standard", - "scenario": "wrong-behavior", - "otelView": "wrong-view", - } - errors = _identity_errors(plan, case, evidence) - assert any("coverageProfile" in error for error in errors) - assert any("scenario" in error for error in errors) - assert any("OTel view" in error for error in errors) - - -def test_otel_validates_lifecycle_requirements_too(): - evidence = { - "coverageProfile": "python-lifecycle", - "runtime": "python", - "requirementId": "python-lmi-11", - "scenario": "timeout-retry", - "otelView": "execution", - "profile": {"recoveryPolicy": "reuse-worker"}, - } - with ( - patch( - "aws_durable_execution_conformance_tests.lmi.evidence.validate_common_evidence", - return_value=[], - ), - patch( - "lmi_tests.python_lifecycle.validate_lifecycle_regressions", - return_value=[], - ), - patch( - "lmi_tests.python_otel.validate_lifecycle_telemetry", - return_value=["ASSERTION: missing replay Workflow"], - ) as telemetry, - ): - assert validate(evidence) == ["ASSERTION: missing replay Workflow"] - telemetry.assert_called_once_with(evidence, phase="final") - - -def test_observed_budgets_cannot_be_inflated_after_the_run(): - plan = manifest() - case = plan["scenarios"][0] - evidence = { - "coverageProfile": "python-lifecycle", - "scenario": case["scenario"], - "regression": {"budgets": {"observationGraceMs": 999999}}, - } - assert any("budgets" in error for error in _identity_errors(plan, case, evidence)) - - -@pytest.mark.parametrize( - "value", - [ - None, - {}, - {"observationGraceMs": True}, - {"observationGraceMs": float("inf")}, - {"observationGraceMs": 0}, - {"observationGraceMs": -1}, - ], -) -def test_behavior_budget_configuration_is_required_and_finite(tmp_path, value): - plan = manifest() - plan["scenarios"][0]["behavior_budgets"] = value - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError): - load_manifest(path) - - -@pytest.mark.parametrize( - "mutation", - [ - lambda rows: rows[0].update(profile_id="wrong-profile"), - lambda rows: rows[0].update(errors=["unresolved assertion"]), - lambda rows: rows.append(deepcopy(rows[0])), - lambda rows: rows[0].update(case_id="unadmitted-case"), - ], -) -def test_strict_report_rejects_misbound_or_contradictory_results(mutation): - plan = manifest() - rows = results(plan) - mutation(rows) - assert build_report(plan, rows).exit_code() == 1 - - -@pytest.mark.parametrize("policy", [None, "unknown", True]) -def test_recovery_policy_must_be_declared_before_execution(tmp_path, policy): - plan = manifest() - plan["profiles"][0]["recovery_policy"] = policy - path = tmp_path / "plan.json" - path.write_text(json.dumps(plan)) - with pytest.raises(ValueError): - load_manifest(path) - - -@lru_cache(maxsize=2) -def _fixture(telemetry): - filename = "test_python_otel.py" if telemetry else "test_python_lifecycle.py" - return run_path(str(Path(__file__).with_name(filename)))["build_evidence"] - - -@pytest.mark.parametrize("cell", required_cases()) -def test_each_python_matrix_cell_composes_shared_evidence_with_local_contracts(cell): - build = _fixture(cell.otel_view is not None) - evidence = ( - build(cell.scenario, native=cell.native_concurrency, view=cell.otel_view) - if cell.otel_view - else build(cell.scenario, native=cell.native_concurrency) - ) - assert evidence["requirementId"] == cell.requirement_id - assert set(cell.roles) == set(evidence["selectedRequests"]) - assert validate(evidence, phase="observations") == [] - assert validate(evidence, phase="final") == [] - - -@pytest.mark.parametrize("runtime", ["java", "javascript", None]) -def test_local_validator_does_not_apply_python_contracts_to_another_runtime(runtime): - evidence = _fixture(False)("native-concurrency") - evidence["runtime"] = runtime - assert any("only to the Python SDK" in error for error in validate(evidence)) - - -@pytest.mark.parametrize("policy", [None, "unknown", []]) -def test_local_validator_requires_an_explicit_supported_recovery_policy(policy): - evidence = _fixture(False)("native-concurrency") - evidence["profile"]["recoveryPolicy"] = policy - assert any("recoveryPolicy" in error for error in validate(evidence)) diff --git a/lmi_tests/tests/unit/test_shared_adapter.py b/lmi_tests/tests/unit/test_shared_adapter.py index 0a2e54306..1de8bbb18 100644 --- a/lmi_tests/tests/unit/test_shared_adapter.py +++ b/lmi_tests/tests/unit/test_shared_adapter.py @@ -17,7 +17,7 @@ ) from lmi_tests import cloud as cloud_module from lmi_tests import shared_adapter as adapter_module -from lmi_tests.evidence import CollectionError, PlacementError, ProvisioningError +from lmi_tests.evidence import CollectionError, ProvisioningError from lmi_tests.shared_scenarios import ScenarioDriver @@ -28,12 +28,10 @@ def context( case_id="case", - scenario="replay-callback-retry", + scenario="suspension-retry", roles=("victim", "resume"), *, native=1, - view=None, - mode=None, ): profile = Profile( f"native-{native}", FUNCTION, native, native, "sdk-sha", "configuration-sha" @@ -42,13 +40,11 @@ def context( "run", Scenario( case_id, - "lmi-lifecycle-16" if mode else "python-lmi-5", + "lmi-lifecycle-5", profile.profile_id, scenario, roles=roles, - otel_view=view, - export_mode=mode, - coverage_profile="export-lifecycle" if mode else "python-lifecycle", + coverage_profile="standard", ), profile, ) @@ -133,7 +129,7 @@ def invoke(**kwargs): payload = json.loads(observed[0]["Payload"]) assert observed[0]["FunctionName"] == FUNCTION assert payload["scenario"] == "replay" - assert payload["sharedScenario"] == "replay-callback-retry" + assert payload["sharedScenario"] == "suspension-retry" assert payload["caseId"] == "case" assert payload["controlPrefix"] == cloud.control_prefix assert "/cases/" in cloud.control_prefix @@ -407,11 +403,6 @@ def for_item(self, item): }, {"phase": "WRAPPER_ENTER", "request": "first"}, {"phase": "WRAPPER_ENTER", "request": "second"}, - { - "phase": "CONFORMANCE", - "observation": {"kind": "flush_wait_completed", "outcome": "timeout"}, - }, - {"phase": "CONFORMANCE", "observation": {"kind": "export_completed"}}, {"phase": "LATE_REJECTED"}, {"phase": "ALIVE", "time": 2}, ] @@ -443,173 +434,6 @@ def test_callback_uses_same_execution_and_actual_pending_before_response(): assert cloud.callback_records[0]["executionArn"] == "execution:victim" -@pytest.mark.parametrize( - "scenario,fixture", - [ - ("early-completion-parallel", "parallel"), - ("early-completion-map", "map"), - ("early-completion-nested", "nested"), - ], -) -def test_early_completion_observations_do_not_release_losing_fault(scenario, fixture): - cloud = ScriptCloud() - driver = ScenarioDriver(context(scenario=scenario, roles=("victim",)), cloud) - driver.start() - driver.place() - driver.execute() - assert cloud.actions[0][1] == fixture - assert not any(action[0] == "release" for action in cloud.actions) - - -@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) -def test_held_async_export_observes_real_timeout_and_wrapper_before_release(mode): - cloud = ScriptCloud() - driver = ScenarioDriver( - context( - scenario="held-export", roles=("victim",), view="invocation", mode=mode - ), - cloud, - ) - driver.start() - driver.place() - driver.execute() - released = next( - index for index, action in enumerate(cloud.actions) if action[0] == "release" - ) - preceding = cloud.actions[:released] - assert ("phase", "victim", "OTEL_EXPORT_HELD") in preceding - assert (("phase", "victim", "WRAPPER_RETURN") in preceding) == ( - mode == "async-bounded-flush" - ) - assert cloud.actions[0][2]["otel_export"] == "held" - - -def test_missed_original_placement_is_not_retried_as_another_victim(): - cloud = ScriptCloud(2) - cloud.poll = lambda *args, **kwargs: [{"phase": "PLACEMENT_MISS"}] - driver = ScenarioDriver( - context(scenario="return-inflight", roles=("victim", "peer"), native=2), cloud - ) - driver.start() - with pytest.raises(PlacementError, match="Original victim"): - driver.place() - assert len(cloud.invocations) == 2 - - -@pytest.mark.parametrize( - "scenario,count", [("warm-lifecycle", 9), ("otel-warm-lifecycle", 6)] -) -def test_warm_sequence_uses_real_callback_resumes_not_extra_driver_invocations( - scenario, count -): - from lmi_tests.python_catalog import required_cases - - cell = next( - case - for case in required_cases("lifecycle-otel") - if case.scenario == scenario and case.native_concurrency == 1 - ) - cloud = ScriptCloud() - driver = ScenarioDriver( - context(scenario=scenario, roles=cell.roles, view=cell.otel_view), cloud - ) - driver.start() - driver.place() - driver.execute() - launches = [action for action in cloud.actions if action[0] == "start"] - assert len(launches) == count - assert [action[1] for action in launches] == ["success", "failure", "replay"] * ( - count // 3 - ) - assert len(cloud.callback_records) == count // 3 - assert len({item["arn"] for item in cloud.invocations}) == count - - -@pytest.mark.parametrize("native", [1, 2]) -def test_deadline_recovery_demands_original_environment_before_fault_release(native): - roles = ( - ("victim", "recovery") - if native == 1 - else ("victim", "peer", "recovery", "recovery-peer") - ) - cloud = ScriptCloud(native) - driver = ScenarioDriver( - context(scenario="deadline-recovery", roles=roles, native=native), cloud - ) - driver.wait_overlap = Mock(return_value=[]) - for role in roles: - driver.context.record_attempt( - role, role=role, requestId="request" if role == "victim" else role - ) - driver.start() - driver.place() - driver.execute() - expected = ("recovery",) if native == 1 else ("recovery", "recovery-peer") - driver.wait_overlap.assert_called_with(expected, "environment") - measured = next( - index - for index, action in enumerate(cloud.actions) - if action[0] == "measurement" - ) - released = next( - index - for index, action in enumerate(cloud.actions) - if action == ("release", driver.gate("fault")) - ) - assert measured < released - assert ("platform-timeout", "execution:victim") in cloud.actions - - -def test_timeout_retry_never_launches_a_replacement_execution(): - cloud = ScriptCloud() - driver = ScenarioDriver( - context(scenario="timeout-retry", roles=("victim", "retry")), cloud - ) - driver.context.record_attempt("original", role="victim", requestId="request") - driver.context.record_attempt("retry", role="retry", requestId="second") - driver.start() - driver.place() - driver.execute() - assert len(cloud.invocations) == 1 - assert cloud.actions[0][2]["resume_roles"] == ("retry",) - assert [action[1] for action in cloud.actions if action[0] == "release"] == [ - driver.gate("fault") - ] - - -@pytest.mark.parametrize( - "scenario,gate_suffix,expected_status", - [ - ("checkpoint-settlement", "checkpoint", "SUCCEEDED"), - ("pending-cleanup", "cleanup", "SUCCEEDED"), - ("return-inflight", "loser", "SUCCEEDED"), - ("failure-inflight", "loser", "FAILED"), - ("late-operation", "loser", "SUCCEEDED"), - ], -) -def test_scope_drivers_observe_real_boundary_before_releasing_its_control( - scenario, gate_suffix, expected_status -): - cloud = ScriptCloud() - roles = ("victim", "resume") if scenario == "pending-cleanup" else ("victim",) - driver = ScenarioDriver(context(scenario=scenario, roles=roles), cloud) - driver.start() - driver.place() - driver.execute() - release = next( - index - for index, action in enumerate(cloud.actions) - if action == ("release", f"victim-{gate_suffix}") - ) - finished = next( - index - for index, action in enumerate(cloud.actions) - if action == ("finish", "victim", expected_status) - ) - assert release < finished - assert any(action[0] == "phase" for action in cloud.actions[:release]) - - def test_each_case_has_independent_artifact_namespace(monkeypatch, tmp_path): class Driver: def __init__(self, context, cloud, **kwargs): @@ -658,7 +482,7 @@ def settled_cloud(*, view=None): }, ] cloud.refresh = lambda **kwargs: events - driver = ScenarioDriver(context(view=view), cloud) + driver = ScenarioDriver(context(), cloud) return driver, events @@ -669,13 +493,9 @@ def settled_cloud(*, view=None): ("io_start", "io_exit", "taskId"), ("checkpoint_call", "checkpoint_exit", "callId"), ("poll_call", "poll_exit", "callId"), - ("export_started", "export_completed", "exportId"), - ("flush_started", "flush_completed", "flushId"), ], ) -def test_retirement_requires_every_real_task_call_and_export_to_exit( - start, end, identity -): +def test_retirement_requires_every_real_task_and_call_to_exit(start, end, identity): driver, events = settled_cloud() assert driver.quiescent() event = {"requestId": "native", identity: "outstanding"} @@ -685,22 +505,6 @@ def test_retirement_requires_every_real_task_call_and_export_to_exit( assert driver.quiescent() -def test_export_quiescence_needs_positive_publication_evidence_even_when_unsampled(): - driver, events = settled_cloud(view="invocation") - assert not driver.quiescent() - observation = { - "kind": "export_quiescence", - "requestId": "native", - "pendingExportIds": [], - "pendingFlushIds": [], - "publicationCount": 1, - } - events.append({"phase": "CONFORMANCE", "observation": observation}) - assert not driver.quiescent() - observation["publicationCount"] = 0 - assert driver.quiescent() - - def test_unidentified_launch_or_missing_root_hook_cannot_claim_retirement(): driver, events = settled_cloud() driver.cloud.invocations[0]["native_requests"] = [] diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py index 4026bc337..1ca538698 100644 --- a/lmi_tests/tests/unit/test_shared_cleanup.py +++ b/lmi_tests/tests/unit/test_shared_cleanup.py @@ -55,7 +55,7 @@ def test_case_ownership_is_checked_before_control_release( "profiles": [{"profile_id": "c1", "resource_id": target}], "scenarios": [case], } - output = tmp_path / "python-lifecycle" + output = tmp_path / "standard" output.mkdir() (output / "manifest.json").write_text(json.dumps(plan)) identity = hashlib.sha256(b"case:cohort").hexdigest()[:20] diff --git a/lmi_tests/tests/unit/test_shared_deadline_measurement.py b/lmi_tests/tests/unit/test_shared_deadline_measurement.py deleted file mode 100644 index 02a028593..000000000 --- a/lmi_tests/tests/unit/test_shared_deadline_measurement.py +++ /dev/null @@ -1,263 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Normal victim release must follow the immutable deadline measurement.""" - -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - Scenario, - ScenarioContext, -) -from lmi_tests import shared_scenarios as module -from lmi_tests.evidence import CollectionError, PlacementError -from lmi_tests.tests.unit.test_cloud import driver as driver - - -class MeasurementCloud: - def __init__(self, context, clock, *, retry_observed=True, write_fails=False): - self.context, self.clock = context, clock - self.concurrency = 1 - self.manifest = {"invocationTimeout": 60, "driverTimeout": 1} - self.actions = [] - self.events = [ - { - "phase": "WRAPPER_ENTER", - "request": "original", - "execution": "execution", - "time": 0, - } - ] - if retry_observed: - self.events.append( - { - "phase": "WRAPPER_ENTER", - "request": "retry", - "execution": "execution", - "time": 52, - } - ) - self.raw_histories = {"execution": [{"EventType": "InvocationCompleted"}]} - self.source_uris = {"s3://case/measurement-source"} - self.write_fails = write_fails - self.snapshot = None - - def poll(self, predicate, **options): - observed = predicate() - if not observed: - raise CollectionError(options.get("message", "measurement wait incomplete")) - return observed - - def platform_timeout(self, item, request): - self.actions.append(("timeout", request)) - - def for_item(self, item): - return self.events - - def refresh(self, **options): - return self.events - - def history(self, item): - self.actions.append(("history", item["arn"])) - return self.raw_histories[item["arn"]] - - def record_deadline_measurement(self, snapshot): - if self.write_fails: - raise OSError("evidence storage failed") - self.snapshot = deepcopy(snapshot) - self.actions.append(("measurement", self.clock.now)) - - def release(self, gate): - self.actions.append(("release", gate, self.clock.now)) - - def wait_terminal(self, item): - self.actions.append(("terminal", item["arn"])) - self.events.append( - {"phase": "WRAPPER_RETURN", "request": "retry", "status": "FAILED"} - ) - self.raw_histories[item["arn"]].append({"EventType": "ExecutionFailed"}) - return {"Status": "FAILED"} - - def start(self, fixture, **options): - role = options["role"] - self.context.record_attempt(role, role=role, requestId=role) - self.actions.append(("start", role)) - return {"arn": role, "marker": role} - - def finish(self, item, status): - self.actions.append(("finish", item["arn"], status)) - - -def measured_driver( - monkeypatch, - *, - scenario="timeout-retry", - now=100, - retry_observed=True, - write_fails=False, -): - clock = SimpleNamespace(now=now) - monkeypatch.setattr(module.time, "time", lambda: clock.now) - roles = ( - ("victim", "retry") if scenario == "timeout-retry" else ("victim", "recovery") - ) - profile = Profile("native-1", "function", 1, 1, "sdk", "configuration") - context = ScenarioContext( - "run", - Scenario( - "case", - "python-lmi-11", - profile.profile_id, - scenario, - roles=roles, - behavior_budgets={ - "cleanupMs": 5000, - "observationGraceMs": 1000, - "retryMs": 2000, - }, - ), - profile, - ) - context.record_attempt("original", role="victim", requestId="original") - if scenario == "timeout-retry" and retry_observed: - context.record_attempt("retry", role="retry", requestId="retry") - cloud = MeasurementCloud( - context, clock, retry_observed=retry_observed, write_fails=write_fails - ) - result = module.ScenarioDriver(context, cloud) - result.items["victim"] = {"arn": "execution", "marker": "victim"} - result.boundary = { - "deadline": 50, - "request": "original", - "environment": "original-environment", - } - return result, cloud, clock - - -def test_completed_measurement_is_persisted_before_normal_release_and_terminal_wait( - monkeypatch, -): - scenario, cloud, clock = measured_driver(monkeypatch) - scenario._deadline() - kinds = [action[0] for action in cloud.actions] - assert kinds.index("measurement") < kinds.index("release") < kinds.index("terminal") - assert [action[1] for action in cloud.actions if action[0] == "release"] == [ - scenario.gate("fault") - ] - assert cloud.snapshot["selectedRequests"] == { - "victim": "original", - "retry": "retry", - } - assert cloud.snapshot["cleanupCutoffAtMs"] == 55_000 - assert cloud.snapshot["observationCutoffAtMs"] == 56_000 - assert cloud.snapshot["observedAtMs"] == clock.now * 1000 - assert cloud.snapshot["sourceUris"] == ["s3://case/measurement-source"] - assert not any( - event["phase"] == "WRAPPER_RETURN" for event in cloud.snapshot["events"] - ) - assert cloud.snapshot["histories"]["execution"] == [ - {"EventType": "InvocationCompleted"} - ] - assert not any(action[0] == "finish" for action in cloud.actions) - - -def test_victim_fault_cannot_be_released_before_cutoff_plus_observation_grace( - monkeypatch, -): - scenario, cloud, _ = measured_driver(monkeypatch, now=55.5) - with pytest.raises(CollectionError): - scenario._deadline() - assert cloud.snapshot is None - assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) - - -def test_victim_fault_cannot_be_released_before_same_execution_retry_is_observed( - monkeypatch, -): - scenario, cloud, _ = measured_driver(monkeypatch, retry_observed=False) - with pytest.raises(CollectionError, match="real service retry"): - scenario._deadline() - assert cloud.snapshot is None - assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) - - -def test_original_recovery_placement_failure_is_not_repaired_by_victim_release( - monkeypatch, -): - scenario, cloud, _ = measured_driver(monkeypatch, scenario="deadline-recovery") - scenario.wait_overlap = Mock( - side_effect=PlacementError("original environment unavailable") - ) - with pytest.raises(PlacementError, match="original environment"): - scenario._deadline() - assert cloud.snapshot is None - assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) - - -def test_measurement_storage_failure_keeps_normal_fault_release_unperformed( - monkeypatch, -): - scenario, cloud, _ = measured_driver(monkeypatch, write_fails=True) - with pytest.raises(OSError, match="evidence storage"): - scenario._deadline() - assert not any(action[0] in {"release", "terminal"} for action in cloud.actions) - - -@pytest.mark.parametrize("status", ["SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"]) -def test_terminal_observation_preserves_actual_outcome_and_waits_for_all_wrappers( - driver, status -): - driver.manifest["driverTimeout"] = 1 - driver.wait = Mock() - driver.lam.get_durable_execution = Mock( - side_effect=[{"Status": "RUNNING"}, {"Status": status}] - ) - opened = [ - {"phase": "WRAPPER_ENTER", "request": "original"}, - {"phase": "WRAPPER_ENTER", "request": "retry"}, - ] - closed = [ - *opened, - {"phase": "WRAPPER_RAISE", "request": "original"}, - {"phase": "WRAPPER_RETURN", "request": "retry"}, - ] - driver.for_item = Mock(side_effect=[opened, closed]) - driver.history = Mock(return_value=[]) - observed = driver.wait_terminal({"arn": "execution", "marker": "victim"}) - assert observed == {"Status": status} - assert driver.for_item.call_count == 2 - assert driver.execution_results["execution"] == observed - driver.history.assert_called_once() - - -def test_terminal_service_status_does_not_replace_missing_native_wrapper_exit(driver): - driver.manifest["driverTimeout"] = 1 - driver.lam.get_durable_execution = Mock(return_value={"Status": "FAILED"}) - driver.for_item = Mock( - return_value=[{"phase": "WRAPPER_ENTER", "request": "still-running"}] - ) - - def poll(predicate, **options): - result = predicate() - if not result: - raise CollectionError(options["message"]) - return result - - driver.poll = poll - with pytest.raises(CollectionError, match="wrapper exits"): - driver.wait_terminal({"arn": "execution", "marker": "victim"}) - - -def test_persisted_measurement_does_not_follow_mutated_later_observations(driver): - snapshot = { - "selectedRequests": {"victim": "original"}, - "events": [{"phase": "EFFECT", "time": 54}], - } - driver.record_deadline_measurement(snapshot) - snapshot["events"][0]["time"] = 99 - assert driver.deadline_measurement["events"][0]["time"] == 54 - assert (driver.artifacts_dir / "measurements/deadline.json").exists() diff --git a/lmi_tests/tests/unit/test_shared_evidence_fixes.py b/lmi_tests/tests/unit/test_shared_evidence_fixes.py deleted file mode 100644 index 88f6571a7..000000000 --- a/lmi_tests/tests/unit/test_shared_evidence_fixes.py +++ /dev/null @@ -1,245 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Service lifetimes and frozen measurements retain independently observed facts.""" - -from copy import deepcopy -import json -from types import SimpleNamespace -from unittest.mock import patch - -import pytest - -from lmi_tests.shared_evidence import CaseSources, _deadline_measurement, build_evidence - - -def sources(raw): - context = SimpleNamespace( - scenario=SimpleNamespace(case_id="case", scenario="native-concurrency") - ) - cloud = SimpleNamespace( - invocations=[ - {"arn": "execution", "marker": "marker", "payload": {"scenario": "success"}} - ], - raw_histories={"execution": raw}, - execution_results={"execution": {"Status": "SUCCEEDED", "Result": '"marker"'}}, - ) - return CaseSources(context, cloud, ()) - - -@pytest.mark.parametrize( - "family", ["Step", "Wait", "Callback", "Context", "ChainedInvoke"] -) -def test_operation_lifetime_comes_from_its_service_events(family): - raw = [ - { - "EventType": family + "Started", - "Id": "op", - "Name": "controlled", - "EventTimestamp": 1.25, - }, - {"EventType": family + "Succeeded", "Id": "op", "EventTimestamp": 2.75}, - ] - history, _, errors = sources(raw).histories([]) - assert errors == [] - assert history[0]["operations"][0]["startedAtMs"] == 1250 - assert history[0]["operations"][0]["endedAtMs"] == 2750 - - -def test_retry_events_cannot_move_the_original_operation_start(): - raw = [ - { - "EventType": "StepStarted", - "Id": "op", - "Name": "controlled", - "EventTimestamp": 1, - }, - {"EventType": "StepRetryStarted", "Id": "op", "EventTimestamp": 2}, - {"EventType": "StepFailed", "Id": "op", "EventTimestamp": 3}, - {"EventType": "StepStarted", "Id": "op", "EventTimestamp": 4}, - {"EventType": "StepSucceeded", "Id": "op", "EventTimestamp": 5}, - ] - history, _, _ = sources(raw).histories([]) - operation = history[0]["operations"][0] - assert (operation["startedAtMs"], operation["endedAtMs"], operation["status"]) == ( - 1000, - 5000, - "SUCCEEDED", - ) - - -def test_missing_original_service_timestamp_stays_missing(): - raw = [ - {"EventType": "StepStarted", "Id": "op", "Name": "controlled"}, - {"EventType": "StepRetryStarted", "Id": "op", "EventTimestamp": 2}, - {"EventType": "StepStopped", "Id": "op", "EventTimestamp": 3}, - ] - history, _, _ = sources(raw).histories([]) - operation = history[0]["operations"][0] - assert "startedAtMs" not in operation - assert operation["endedAtMs"] == 3000 - assert operation["status"] == "STOPPED" - - -@pytest.fixture -def measurement(tmp_path): - # Preserve the archived c2 baseline's observed deadline/measurement deltas. - selected = {"victim": "original", "retry": "retry"} - record = { - "runId": "run", - "caseId": "case", - "cohortId": "cohort-1", - "scenario": "timeout-retry", - "executionArn": "execution", - "selectedRequests": selected, - "deadlineAtMs": 1794527680692.2056, - "cleanupCutoffAtMs": 1794527685692.2056, - "observationCutoffAtMs": 1794527686692.2056, - "observedAtMs": 1794527688724.1567, - "events": [{"kind": "effect", "atMs": 1794527685892.097}], - "histories": {"execution": []}, - "sourceUris": ["memory://frozen-source"], - } - path = tmp_path / "measurements/deadline.json" - path.parent.mkdir() - path.write_text(json.dumps(record)) - context = SimpleNamespace( - run_id="run", - scenario=SimpleNamespace( - case_id="case", cohort_id="cohort-1", scenario="timeout-retry" - ), - ) - cloud = SimpleNamespace( - artifacts_dir=tmp_path, - deadline_measurement=deepcopy(record), - invocations=[{"role": "victim", "arn": "execution"}], - ) - return context, cloud, selected, record, path - - -def test_persisted_measurement_exports_only_identity_bounds_and_source_reference( - measurement, -): - context, cloud, selected, record, path = measurement - value, errors = _deadline_measurement(context, cloud, selected) - assert errors == [] - assert value == { - **{ - key: record[key] - for key in ( - "selectedRequests", - "deadlineAtMs", - "cleanupCutoffAtMs", - "observationCutoffAtMs", - "observedAtMs", - ) - }, - "sourceEvidenceUri": path.resolve().as_uri(), - } - assert "events" not in value - - -@pytest.mark.parametrize( - "field", - ["runId", "caseId", "cohortId", "scenario", "executionArn", "selectedRequests"], -) -def test_measurement_cannot_be_substituted_between_cases_or_original_requests( - measurement, field -): - context, cloud, selected, record, path = measurement - record[field] = ( - {"victim": "replacement"} if field == "selectedRequests" else "other" - ) - path.write_text(json.dumps(record)) - value, errors = _deadline_measurement(context, cloud, selected) - assert value is None and errors - - -def test_measurement_must_match_the_record_written_before_release(measurement): - context, cloud, selected, _, _ = measurement - cloud.deadline_measurement["observedAtMs"] += 1 - value, errors = _deadline_measurement(context, cloud, selected) - assert value is None and errors - - -def test_missing_measurement_is_legacy_only_if_no_snapshot_was_recorded(measurement): - context, cloud, selected, _, path = measurement - path.unlink() - assert _deadline_measurement(context, cloud, selected)[1] - cloud.deadline_measurement = None - assert _deadline_measurement(context, cloud, selected) == (None, []) - - -def test_reloading_archived_measurement_does_not_require_the_old_driver_process( - measurement, -): - context, cloud, selected, _, _ = measurement - cloud.deadline_measurement = None - value, errors = _deadline_measurement(context, cloud, selected) - assert not errors and value["selectedRequests"] == selected - - -def test_assembler_keeps_post_measurement_events_in_the_main_evidence(measurement): - context, cloud, selected, record, _ = measurement - context.scenario = SimpleNamespace( - case_id="case", - cohort_id="cohort-1", - scenario="timeout-retry", - requirement_id="python-lmi-11", - coverage_profile="python-lifecycle", - otel_view=None, - export_mode=None, - behavior_budgets={ - "cleanupMs": 5000, - "observationGraceMs": 1000, - "retryMs": 40000, - }, - ) - context.profile = SimpleNamespace( - artifact_id="sdk", - configuration_id="config", - resource_id="function", - native_concurrency=2, - recovery_policy="reuse-worker", - ) - cloud.manifest = {"conformanceRevision": "conformance", "invocationTimeout": 60} - cloud.observation_ended_at = 1794527693 - cloud.retired_at = cloud.observation_ended_at - cloud.source_uris = {"memory://source"} - cloud.control_records = [] - cloud._save = lambda *args: None - cloud.invocations[0]["marker"] = "marker" - cloud.invocations[0]["payload"] = {"scenario": "deadline"} - events = [ - { - "kind": "effect", - "requestId": "original", - "ownerRequestId": "original", - "executionArn": "execution", - "atMs": record["cleanupCutoffAtMs"] + 200, - }, - { - "kind": "checkpoint_exit", - "requestId": "original", - "ownerRequestId": "original", - "executionArn": "execution", - "atMs": record["observedAtMs"] + 1000, - }, - ] - captured = SimpleNamespace( - observations=lambda: (deepcopy(events), []), - histories=lambda _: ([], {}, []), - attempts=lambda _: ( - [{"requestId": "original", "executionArn": "execution"}], - selected, - [], - ), - ) - with patch("lmi_tests.shared_evidence.CaseSources.capture", return_value=captured): - value = build_evidence(context, cloud) - assert [event["atMs"] for event in value["events"]] == [ - event["atMs"] for event in events - ] - assert ( - value["regression"]["deadlineMeasurement"]["observedAtMs"] - == record["observedAtMs"] - ) diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py index 28cb816d6..958de2a55 100644 --- a/lmi_tests/tests/unit/test_shared_manifest.py +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -9,7 +9,6 @@ from aws_durable_execution_conformance_tests.lmi.app import load_manifest from lmi_tests.shared import make_manifest, verify_checkout -from lmi_tests.python_catalog import suite_definition from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity @@ -35,7 +34,6 @@ def deployment(): }, "wheels": { "aws_durable_execution_sdk_python-3.0.0.whl": "core-sha", - "aws_durable_execution_sdk_python_otel-2.0.0.whl": "otel-sha", }, "sharedLease": { "bucket": "private", @@ -47,27 +45,29 @@ def deployment(): } -@pytest.mark.parametrize( - "profile,count", [("python-lifecycle", 76), ("export-lifecycle", 24)] -) -def test_generated_manifest_covers_every_required_slot(tmp_path, profile, count): +def test_generated_manifest_covers_the_seven_shared_scenarios(tmp_path): + from aws_durable_execution_conformance_tests.lmi.app import coverage_inventory + source = deployment() - plan = make_manifest(source, profile, tmp_path) - assert len(plan["scenarios"]) == count - identifiers = {case["requirement_id"] for case in plan["scenarios"]} - if profile == "python-lifecycle": - assert all(identity.startswith("python-lmi-") for identity in identifiers) - else: - assert identifiers == {"lmi-lifecycle-16"} - assert {case["profile_id"] for case in plan["scenarios"]} == {"c1", "c2"} + plan = make_manifest(source, "standard", tmp_path) + assert len(plan["scenarios"]) == 7 + assert {c["requirement_id"] for c in plan["scenarios"]} == { + f"lmi-lifecycle-{i}" for i in range(1, 8) + } + assert {c["profile_id"] for c in plan["scenarios"]} == {"c1", "c2"} + assert all( + "otel_view" not in c and "export_mode" not in c for c in plan["scenarios"] + ) assert plan["sdk"]["deployment"]["conformanceRevision"] == CONFORMANCE_REVISION path = tmp_path / "plan.json" path.write_text(json.dumps(plan)) + assert load_manifest(path) == plan + assert coverage_inventory(plan)["missing"] == [] assert ( - load_manifest( - path, suite=suite_definition() if profile == "python-lifecycle" else None - ) - == plan + next(c for c in plan["scenarios"] if c["scenario"] == "normal-failure")[ + "requirement_id" + ] + == "lmi-lifecycle-7" ) plan["sdk"]["deployment"]["commit"] = "mutated" assert source["commit"] == "sdk-commit" @@ -77,11 +77,11 @@ def test_no_unowned_or_unidentified_artifact_can_be_planned(tmp_path): source = deployment() source.pop("sharedLease") with pytest.raises(ValueError, match="ownership"): - make_manifest(source, "python-lifecycle", tmp_path) + make_manifest(source, "standard", tmp_path) source = deployment() source["wheels"] = {} with pytest.raises(ValueError, match="wheel"): - make_manifest(source, "python-lifecycle", tmp_path) + make_manifest(source, "standard", tmp_path) def test_configuration_identity_changes_with_the_tested_settings(): diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py index f3fad0abb..bcdedfd3e 100644 --- a/lmi_tests/tests/unit/test_shared_observations.py +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -31,7 +31,7 @@ LifecycleObservations, OperationObservation, ) -from lmi_tests.tests.regressions.test_issue741 import lambda_context +from lmi_tests.tests.unit.support import lambda_context class MemoryLedger: @@ -96,7 +96,7 @@ def shared_payload(scenario: str = "success") -> dict[str, Any]: "caseId": "case", "cohortId": "cohort-1", "role": "victim", - "coverageProfile": "python-lifecycle", + "coverageProfile": "standard", "sharedScenario": scenario, "scenario": scenario, "marker": "marker", @@ -297,7 +297,6 @@ def test_trace_records_immutable_case_identity_and_actual_source_sequence( ) -> None: ledger = install_ledger(monkeypatch) payload = shared_payload() - payload.update(otelView="invocation", exportMode="async-bounded-flush") trace = fixture.Trace(payload, lambda_context(30_000), "actual-execution") observed_at = time.time() * 1000 - 10 record = trace.observe("boundary", atMs=observed_at) @@ -307,8 +306,8 @@ def test_trace_records_immutable_case_identity_and_actual_source_sequence( "sourceSequence": record["sequence"], "captureSequence": 1, } - assert record["caseId"] == "case" and record["exportMode"] == "async-bounded-flush" - assert trace.payload["otel_view"] == "invocation" and trace.payload["run"] == "unit" + assert record["caseId"] == "case" + assert trace.payload["run"] == "unit" assert ledger.observations() == [record] diff --git a/lmi_tests/tests/unit/test_shared_partial_evidence.py b/lmi_tests/tests/unit/test_shared_partial_evidence.py index a3267b74d..ad00416cd 100644 --- a/lmi_tests/tests/unit/test_shared_partial_evidence.py +++ b/lmi_tests/tests/unit/test_shared_partial_evidence.py @@ -23,11 +23,11 @@ def context(): "run", Scenario( "case", - "python-lmi-11", + "lmi-lifecycle-3", "c1", - "timeout-retry", + "deadline-during-step", roles=("victim", "retry"), - coverage_profile="python-lifecycle", + coverage_profile="standard", ), profile, ) @@ -113,11 +113,11 @@ def test_final_read_bound_alone_does_not_invent_quiescence(tmp_path): ctx = context() ctx.scenario = Scenario( "case", - "python-lmi-1", + "lmi-lifecycle-1", "c1", - "native-concurrency", + "native-overlap", roles=("victim",), - coverage_profile="python-lifecycle", + coverage_profile="standard", ) cloud = SimpleNamespace( manifest={"invocationTimeout": 60, "conformanceRevision": "conformance"}, diff --git a/lmi_tests/tests/unit/test_shared_telemetry.py b/lmi_tests/tests/unit/test_shared_telemetry.py deleted file mode 100644 index 4c08b524d..000000000 --- a/lmi_tests/tests/unit/test_shared_telemetry.py +++ /dev/null @@ -1,646 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Real public SDK/plugin calls with in-memory observation and gate transport.""" - -from concurrent.futures import ThreadPoolExecutor -import json -import os -import threading -import time -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_sdk_python import durable_execution -from aws_durable_execution_sdk_python.execution import ( - DurableExecutionInvocationInput, - DurableExecutionInvocationInputWithClient, -) -from aws_durable_execution_sdk_python.plugin import DurableInstrumentationPlugin -from lmi_tests import telemetry -from lmi_tests.shared_exporter import CURRENT_OWNER, owned -from lmi_tests.shared_telemetry import collect_telemetry -from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context - -HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1" - - -class SharedTrace: - def __init__( - self, - marker="victim", - *, - view="invocation", - mode="sync", - behavior="success", - profile="export-lifecycle", - ): - self.payload = { - "marker": marker, - "scenario": "success", - "run": "unit", - "caseId": marker, - "coverageProfile": profile, - "otel_view": view, - "exportMode": mode, - "otel_export": behavior, - "otel_export_gate": marker + "-export", - "behaviorBudgets": { - "flushWaitMs": 40, - "exportCompletionMs": 5000, - "exportMs": 5000, - "observationGraceMs": 1, - }, - } - self.identity = { - "request": "request-" + marker, - "execution": "arn:aws:lambda:us-west-2:123456789012:function:lmi:1/durable-execution/" - + marker - + "/local", - "environment": "memory-environment", - "process": "memory-process", - "pid": os.getpid(), - "marker": marker, - } - self.lock = threading.Lock() - self.capture_lock = threading.Lock() - self.capture_sequence = 0 - self.events = [] - self.sequence = 0 - self.held = threading.Event() - self.release = threading.Event() - self.published = threading.Event() - self.publication_entered = threading.Event() - self.allow_publication = threading.Event() - self.allow_publication.set() - self.fail_publication = False - self.sdk_completed_at = None - - def emit(self, phase, **fields): - with self.lock: - self.sequence += 1 - record = { - **self.identity, - "phase": phase, - "time": time.time(), - "sequence": self.sequence, - **fields, - } - self.events.append(record) - return record - - def capture_observation(self, kind, **fields): - with self.capture_lock: - self.capture_sequence += 1 - return { - "kind": kind, - "atMs": time.time_ns() / 1_000_000, - "captureSequence": self.capture_sequence, - **fields, - } - - def publish_observation(self, observation): - self.publication_entered.set() - if not self.allow_publication.wait(5): - raise TimeoutError("local observer fixture was not released") - if self.fail_publication: - raise RuntimeError("observed ledger write failure") - record = self.emit("CONFORMANCE", observation=observation) - if observation["kind"] == "export_quiescence": - self.published.set() - return record - - def observe(self, kind, **fields): - return self.publish_observation(self.capture_observation(kind, **fields)) - - def gate_release(self, name, *, on_hold=None, on_release=None): - assert name == self.payload["otel_export_gate"] - held_at_ms = None if self.release.is_set() else time.time_ns() / 1_000_000 - self.held.set() - if held_at_ms is not None and on_hold is not None: - on_hold(held_at_ms) - if not self.release.wait(5): - raise TimeoutError("local export fixture was not released") - return SimpleNamespace( - gate_id=name, observed_at_ms=time.time_ns() / 1_000_000, mode="normal" - ) - - def gate(self, name): - self.gate_release(name) - - -def run_public(trace): - source = invocation() - event = source.to_json_dict() - event["DurableExecutionArn"] = trace.identity["execution"] - event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ - "InputPayload" - ] = json.dumps(trace.payload) - event["InitialExecutionState"]["Operations"][0]["StartTimestamp"] = int( - time.time() * 1000 - ) - typed = DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( - DurableExecutionInvocationInput.from_json_dict(event), source.service_client - ) - context = lambda_context(30000) - context.aws_request_id = trace.identity["request"] - trace.emit("WRAPPER_ENTER") - - def call(plugins): - handler = durable_execution( - lambda event, ctx: ctx.step(lambda _: event["marker"], name="saved"), - plugins=plugins, - ) - try: - return handler(typed, context) - finally: - trace.sdk_completed_at = time.time() - - result = telemetry.invoke(trace, call) - trace.emit("WRAPPER_RETURN", status=result["Status"], time=trace.sdk_completed_at) - return result - - -def observations(trace, kind): - return [ - record["observation"] - for record in trace.events - if record["phase"] == "CONFORMANCE" and record["observation"]["kind"] == kind - ] - - -def wait_quiescence(trace): - assert trace.published.wait(5), "actual publication never reached quiescence" - until = time.monotonic() + 2 - while time.monotonic() < until: - active = telemetry.active_work(trace) - if ( - not active["pendingExportIds"] - and not active["pendingFlushIds"] - and not active["publicationCount"] - ): - assert active["publicationErrors"] == [] - return - threading.Event().wait(0.001) - pytest.fail("pending runtime ownership survived publication") - - -def attempt(trace): - return { - "requestId": trace.identity["request"], - "executionArn": trace.identity["execution"], - "environmentId": trace.identity["environment"], - "processId": trace.identity["process"], - "workerId": trace.identity["process"], - } - - -@pytest.fixture(autouse=True) -def runtimes(monkeypatch): - monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) - monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) - telemetry._RUNTIMES.clear() - factory = telemetry.Runtime - monkeypatch.setattr( - telemetry, - "Runtime", - lambda view, export_mode="sync", **options: factory( - view, export_mode, **{"flush_timeout_ms": 20, **options} - ), - ) - yield - for runtime in telemetry._RUNTIMES.values(): - runtime.provider.shutdown() - telemetry._RUNTIMES.clear() - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) -@pytest.mark.parametrize("behavior", ["success", "failure"]) -def test_public_sdk_modes_preserve_handler_and_actual_export_outcome( - view, mode, behavior -): - trace = SharedTrace(view=view, mode=mode, behavior=behavior) - assert run_public(trace)["Status"] == "SUCCEEDED" - wait_quiescence(trace) - completed = observations(trace, "export_completed") - assert completed and {item["outcome"] for item in completed} == { - "failed" if behavior == "failure" else "succeeded" - } - if mode == "sync": - assert max(item["atMs"] for item in completed) <= trace.sdk_completed_at * 1000 - else: - wait = observations(trace, "flush_wait_completed") - assert len(wait) == 1 and wait[0]["outcome"] == ( - "error" if behavior == "failure" else "completed" - ) - assert observations(trace, "flush_completed")[0]["atMs"] <= wait[0]["atMs"] - normalized = collect_telemetry( - trace.events, - [attempt(trace)], - view=view, - expected_sampled=True, - caller_headers={trace.identity["request"]: HEADER}, - observed_until_ms=time.time_ns() / 1_000_000, - ) - assert normalized["views"][view]["collection"]["status"] == "complete" - names = [record["span"]["name"] for record in normalized["views"][view]["spans"]] - assert names.count("Invocation") == names.count("Workflow") == 1 - assert normalized["views"][view]["carriers"][0]["header"] == HEADER - - -@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) -def test_held_export_has_mode_specific_real_sdk_boundary(mode): - trace = SharedTrace(mode=mode, behavior="held") - with ThreadPoolExecutor(max_workers=1) as worker: - result = worker.submit(run_public, trace) - try: - assert trace.held.wait(3) - if mode == "sync": - assert trace.sdk_completed_at is None - assert not result.done() - else: - assert result.result(timeout=3)["Status"] == "SUCCEEDED" - assert ( - observations(trace, "flush_wait_completed")[0]["outcome"] - == "timeout" - ) - assert telemetry.active_work(trace)["pendingExportIds"] - assert not trace.published.is_set() - assert not observations(trace, "flush_completed") - finally: - trace.release.set() - assert result.result(timeout=3)["Status"] == "SUCCEEDED" - wait_quiescence(trace) - if mode == "async-bounded-flush": - assert ( - min(item["atMs"] for item in observations(trace, "export_completed")) - > trace.sdk_completed_at * 1000 - ) - assert observations(trace, "export_quiescence")[-1]["pendingFlushIds"] == [] - - -def test_late_async_export_keeps_original_owner_during_warm_invocation(): - first = SharedTrace("first", mode="async-bounded-flush", behavior="held") - warm = SharedTrace("warm", mode="async-bounded-flush") - try: - assert run_public(first)["Status"] == "SUCCEEDED" - assert first.held.is_set() - assert run_public(warm)["Status"] == "SUCCEEDED" - assert telemetry.active_work(first)["pendingExportIds"] - assert telemetry.active_work(warm)["pendingExportIds"] - finally: - first.release.set() - wait_quiescence(first) - wait_quiescence(warm) - for trace in (first, warm): - assert all( - record["span"]["attributes"]["durable.execution.arn"] - == trace.identity["execution"] - for record in trace.events - if record["phase"] == "OTEL_EXPORT" - ) - assert ( - observations(first, "provider_observed")[0]["providerId"] - == observations(warm, "provider_observed")[0]["providerId"] - ) - - -def test_delayed_evidence_publication_does_not_extend_sdk_flush_wait(): - trace = SharedTrace(mode="async-bounded-flush") - trace.allow_publication.clear() - try: - assert run_public(trace)["Status"] == "SUCCEEDED" - assert trace.publication_entered.is_set() - assert trace.sdk_completed_at is not None - assert telemetry.active_work(trace)["publicationCount"] > 0 - assert not trace.published.is_set() - finally: - trace.allow_publication.set() - wait_quiescence(trace) - waited = observations(trace, "flush_wait_completed")[0] - started = observations(trace, "flush_started")[0] - assert ( - waited["atMs"] - started["atMs"] - < trace.payload["behaviorBudgets"]["flushWaitMs"] - ) - assert waited["atMs"] <= trace.sdk_completed_at * 1000 - - -def test_failed_publication_stays_owned_and_never_claims_quiescence(): - trace = SharedTrace(mode="async-bounded-flush") - trace.fail_publication = True - assert run_public(trace)["Status"] == "SUCCEEDED" - assert trace.publication_entered.wait(2) - active = telemetry.active_work(trace) - assert active["publicationCount"] > 0 - assert active["publicationErrors"] - assert not trace.published.is_set() - - -@pytest.mark.parametrize( - "hook", - [ - "on_invocation_start", - "on_invocation_end", - "on_operation_start", - "on_operation_end", - "on_operation_change", - "on_user_function_start", - "on_user_function_end", - "provide_propagation_metadata", - ], -) -def test_proxy_forwards_every_public_hook_and_restores_calling_owner(hook): - runtime = telemetry.Runtime("invocation") - first = runtime.tracker.owner(SharedTrace("first")) - newer = runtime.tracker.owner(SharedTrace("newer")) - delegate = Mock(spec=DurableInstrumentationPlugin) - info, answer = object(), object() - - def called(value): - assert value is info and CURRENT_OWNER.get() is first - return answer - - getattr(delegate, hook).side_effect = called - proxy = telemetry.ObservedPlugin(delegate, first) - try: - with owned(newer): - result = getattr(proxy, hook)(info) - assert CURRENT_OWNER.get() is newer - if hook == "provide_propagation_metadata": - assert result is answer - finally: - runtime.provider.shutdown() - getattr(delegate, hook).assert_called_once_with(info) - - -def test_missing_public_owner_cannot_borrow_current_runtime_state(): - runtime = telemetry.Runtime("invocation") - try: - with pytest.raises(RuntimeError, match="original public-plugin"): - runtime.tracker.accept(()) - finally: - runtime.provider.shutdown() - - -@pytest.mark.parametrize("sampled", [False, True]) -def test_collection_requires_quiescence_and_source_sequence_completeness( - monkeypatch, sampled -): - monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER[:-1] + str(int(sampled))) - trace = SharedTrace() - run_public(trace) - wait_quiescence(trace) - options = dict( - view="invocation", - expected_sampled=sampled, - caller_headers={}, - observed_until_ms=time.time_ns() / 1_000_000, - ) - assert ( - collect_telemetry(trace.events, [attempt(trace)], **options)["views"][ - "invocation" - ]["collection"]["status"] - == "complete" - ) - missing = [ - record - for record in trace.events - if record["phase"] != "CONFORMANCE" - or record["observation"]["kind"] != "export_quiescence" - ] - assert ( - collect_telemetry(missing, [attempt(trace)], **options)["views"]["invocation"][ - "collection" - ]["status"] - == "incomplete" - ) - assert ( - collect_telemetry(trace.events[1:], [attempt(trace)], **options)["views"][ - "invocation" - ]["collection"]["status"] - == "incomplete" - ) - - -def test_real_gate_reports_first_hold_and_does_not_rehold_an_open_gate(monkeypatch): - from lmi_tests import fixture - from lmi_tests.tests.unit.test_shared_observations import ( - install_ledger, - shared_payload, - ) - - ledger = install_ledger(monkeypatch) - payload = shared_payload() - trace = fixture.Trace(payload, lambda_context(30000), "execution") - prefix = payload["controlPrefix"] - ledger.objects[prefix + "gate"] = b"hold" - ledger.objects[prefix + "release-all"] = b"hold" - held, released = [], [] - with ThreadPoolExecutor(max_workers=1) as caller: - work = caller.submit( - trace.gate_release, "gate", on_hold=held.append, on_release=released.append - ) - try: - observed = ledger.wait_for("blocked") - finally: - ledger.put_object(Bucket=trace.bucket, Key=prefix + "gate", Body=b"release") - result = work.result(timeout=2) - assert held == [result.held_at_ms] == [observed["observation"]["atMs"]] - assert released == [result] - assert result.held_at_ms < result.observed_at_ms and result.mode == "normal" - held.clear() - opened = trace.gate_release("gate", on_hold=held.append) - assert opened.held_at_ms is None and held == [] - - -def test_actual_capture_order_survives_reversed_publication(monkeypatch): - from lmi_tests import fixture - from lmi_tests.tests.unit.test_shared_observations import ( - install_ledger, - shared_payload, - ) - - ledger = install_ledger(monkeypatch) - trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") - at_ms = time.time_ns() / 1_000_000 - exported = trace.capture_observation("export_completed", atMs=at_ms) - wrapper = trace.observe("wrapper_return", atMs=at_ms) - published = trace.publish_observation(exported) - assert exported["captureSequence"] < wrapper["observation"]["captureSequence"] - assert published["sequence"] > wrapper["sequence"] - assert ledger.observations()[0]["observation"]["kind"] == "wrapper_return" - - -def test_capture_does_not_wait_for_s3_publication_lock(monkeypatch): - from lmi_tests import fixture - from lmi_tests.tests.unit.test_shared_observations import ( - install_ledger, - shared_payload, - ) - - ledger = install_ledger(monkeypatch) - trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") - entered, release = threading.Event(), threading.Event() - original = ledger.put_object - - def blocked_put(**fields): - entered.set() - assert release.wait(3) - return original(**fields) - - monkeypatch.setattr(ledger, "put_object", blocked_put) - with ThreadPoolExecutor(max_workers=2) as workers: - publication = workers.submit(trace.observe, "first") - try: - assert entered.wait(2) - captured = workers.submit(trace.capture_observation, "second").result( - timeout=1 - ) - assert captured["captureSequence"] == 2 - finally: - release.set() - publication.result(timeout=2) - - -def test_proxy_restores_owner_when_real_delegate_hook_raises(): - runtime = telemetry.Runtime("invocation") - first = runtime.tracker.owner(SharedTrace("first")) - newer = runtime.tracker.owner(SharedTrace("newer")) - delegate = Mock(spec=DurableInstrumentationPlugin) - delegate.on_operation_end.side_effect = RuntimeError("hook failure") - try: - with owned(newer): - with pytest.raises(RuntimeError, match="hook failure"): - telemetry.ObservedPlugin(delegate, first).on_operation_end(object()) - assert CURRENT_OWNER.get() is newer - finally: - runtime.provider.shutdown() - - -def test_async_processor_respects_smaller_provider_wait_budget(): - runtime = telemetry.Runtime( - "invocation", "async-bounded-flush", flush_timeout_ms=2000 - ) - trace = SharedTrace(mode="async-bounded-flush", behavior="held") - owner = runtime.tracker.owner(trace) - try: - with owned(owner): - span = runtime.provider.get_tracer("public-fixture").start_span("held") - span.end() - assert trace.held.wait(2) - before = time.monotonic() - assert runtime.provider.force_flush(timeout_millis=20) is False - assert time.monotonic() - before < 0.5 - finally: - trace.release.set() - runtime.tracker.close_invocation(owner) - runtime.provider.shutdown() - assert observations(trace, "flush_wait_completed")[0]["outcome"] == "timeout" - assert observations(trace, "flush_completed")[0]["outcome"] == "succeeded" - - -def test_converter_rejects_mode_view_or_unjournaled_owner_substitution(): - trace = SharedTrace() - run_public(trace) - wait_quiescence(trace) - options = dict( - expected_sampled=True, - caller_headers={}, - observed_until_ms=time.time_ns() / 1_000_000, - ) - with pytest.raises(ValueError, match="view differs"): - collect_telemetry(trace.events, [attempt(trace)], view="execution", **options) - with pytest.raises(ValueError, match="exporter mode differs"): - collect_telemetry( - trace.events, - [attempt(trace)], - view="invocation", - export_mode="async-bounded-flush", - **options, - ) - with pytest.raises(ValueError, match="unjournaled"): - collect_telemetry(trace.events, [], view="invocation", **options) - - -def test_held_fixture_records_only_calls_that_actually_waited(): - trace = SharedTrace(mode="sync", behavior="held") - with ThreadPoolExecutor(max_workers=1) as caller: - work = caller.submit(run_public, trace) - try: - assert trace.held.wait(2) - finally: - trace.release.set() - assert work.result(timeout=3)["Status"] == "SUCCEEDED" - wait_quiescence(trace) - started = observations(trace, "export_started") - held = observations(trace, "export_held") - assert len(started) > len(held) == 1 - assert len(observations(trace, "export_completed")) == len(started) - - -@pytest.mark.parametrize("mode", ["sync", "async-bounded-flush"]) -def test_process_provider_does_not_retain_completed_invocations(mode): - import gc - import weakref - - references = [] - for index in range(12): - trace = SharedTrace("retired-" + str(index), mode=mode) - assert run_public(trace)["Status"] == "SUCCEEDED" - wait_quiescence(trace) - references.append(weakref.ref(trace)) - del trace - gc.collect() - assert all(reference() is None for reference in references) - assert len(telemetry._RUNTIMES) == 1 - runtime = next(iter(telemetry._RUNTIMES.values())) - assert ( - runtime.tracker.owners - == runtime.tracker.exports - == runtime.tracker.flushes - == {} - ) - - -def test_runtime_variants_keep_view_and_processor_configuration_immutable(): - for view, mode in ( - ("invocation", "sync"), - ("execution", "sync"), - ("invocation", "async-bounded-flush"), - ): - trace = SharedTrace(view + mode, view=view, mode=mode) - assert run_public(trace)["Status"] == "SUCCEEDED" - wait_quiescence(trace) - assert set(telemetry._RUNTIMES) == { - ("invocation", "sync"), - ("execution", "sync"), - ("invocation", "async-bounded-flush"), - } - assert len({runtime.identity for runtime in telemetry._RUNTIMES.values()}) == 3 - - -@pytest.mark.parametrize("shared,expected", [(True, "emergency"), (False, "normal")]) -def test_shared_gate_abort_has_priority_without_changing_legacy_release_order( - monkeypatch, shared, expected -): - from lmi_tests import fixture - from lmi_tests.tests.unit.test_shared_observations import ( - install_ledger, - shared_payload, - ) - - ledger = install_ledger(monkeypatch) - payload = ( - shared_payload() - if shared - else {"run": "unit", "marker": "legacy", "scenario": "success"} - ) - trace = fixture.Trace(payload, lambda_context(30000), "execution") - ledger.objects[trace.control_prefix + "gate"] = b"release" - ledger.objects[trace.control_prefix + "release-all"] = b"release" - result = trace.gate_release("gate") - assert result.mode == expected - assert result.held_at_ms is None diff --git a/lmi_tests/tests/unit/test_telemetry.py b/lmi_tests/tests/unit/test_telemetry.py deleted file mode 100644 index 6ad509c1c..000000000 --- a/lmi_tests/tests/unit/test_telemetry.py +++ /dev/null @@ -1,271 +0,0 @@ -"""Public-wrapper OTel controls and negative evidence fixtures; no AWS calls.""" - -import copy -import json -import os -import time - -import pytest - -from aws_durable_execution_sdk_python import durable_execution -from aws_durable_execution_sdk_python.config import Duration -from aws_durable_execution_sdk_python.execution import DurableExecutionInvocationInput -from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner -from lmi_tests import fixture, telemetry, telemetry_evidence -from lmi_tests.tests.regressions.test_issue741 import invocation, lambda_context - - -HEADER = "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=1" - - -class MemoryTrace: - def __init__(self, payload, context, execution): - self.payload = payload - self.sequence = 0 - self.events = [] - self.identity = { - "request": context.aws_request_id, - "execution": execution, - "environment": "local", - "process": "local-worker", - "pid": os.getpid(), - "marker": payload["marker"], - } - - def emit(self, phase, **fields): - self.sequence += 1 - event = { - **self.identity, - "phase": phase, - "sequence": self.sequence, - "time": time.time(), - **fields, - } - self.events.append(event) - return event - - def body(self, name, value, step): - self.emit("BODY", operation=name, value=value, attempt=step.attempt) - return value - - -@pytest.fixture(autouse=True) -def isolated_runtimes(monkeypatch): - monkeypatch.setenv("LMI_RUN_ID", "unit") - monkeypatch.delenv("DURABLE_EXECUTION_PLUGINS", raising=False) - monkeypatch.setattr( - fixture, "resources", lambda: {"threads": [], "fds": 0, "rss": 0} - ) - telemetry._RUNTIMES.clear() - yield - for runtime in telemetry._RUNTIMES.values(): - runtime.provider.shutdown() - telemetry._RUNTIMES.clear() - - -def run( - monkeypatch, view, marker, *, header=HEADER, scenario="success", mode="success" -): - if header is None: - monkeypatch.delenv("_X_AMZN_TRACE_ID", raising=False) - else: - monkeypatch.setenv("_X_AMZN_TRACE_ID", header) - original = os.environ.get("_X_AMZN_TRACE_ID") - source = invocation() - event = source.to_json_dict() - event["DurableExecutionArn"] = ( - "arn:aws:lambda:us-west-2:123456789012:function:lmi:1/durable-execution/" - + marker - + "/local" - ) - event["InitialExecutionState"]["Operations"][0]["StartTimestamp"] = int( - time.time() * 1000 - ) - payload = { - "run": "unit", - "marker": marker, - "scenario": scenario, - "otel_view": view, - "otel_export": mode, - } - event["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ - "InputPayload" - ] = json.dumps(payload) - context = lambda_context(30000) - context.aws_request_id = "request-" + marker - trace = MemoryTrace(payload, context, event["DurableExecutionArn"]) - monkeypatch.setattr(fixture, "Trace", lambda *_: trace) - monkeypatch.setattr( - fixture.LambdaClient, "initialize_client", lambda: source.service_client - ) - result = fixture.handler(event, context) - assert os.environ.get("_X_AMZN_TRACE_ID") == original - assert result["Status"] == ("FAILED" if scenario == "failure" else "SUCCEEDED") - telemetry_evidence.invocation( - trace.events, native=header == HEADER, expected_outcome=mode - ) - return trace.events - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("sampled", ["0", "1"]) -@pytest.mark.parametrize("scenario", ["success", "failure"]) -def test_real_public_wrapper_exports_and_respects_sampling( - monkeypatch, view, sampled, scenario -): - events = run( - monkeypatch, view, "first", header=HEADER[:-1] + sampled, scenario=scenario - ) - telemetry_evidence.terminal(events, failed=scenario == "failure") - exports = [e for e in events if e["phase"] == "OTEL_EXPORT"] - assert bool(exports) == (sampled == "1") - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("header", [None, "", "malformed", "Root=invalid;Sampled=0"]) -def test_warm_reuse_does_not_borrow_previous_header(monkeypatch, view, header): - first = run(monkeypatch, view, "first") - second = run(monkeypatch, view, "second", header=header) - telemetry_evidence.warm_reuse(first + second) - first_root = next( - e["span"]["trace_id"] for e in first if e["phase"] == "OTEL_EXPORT" - ) - second_roots = { - e["span"]["trace_id"] for e in second if e["phase"] == "OTEL_EXPORT" - } - assert first_root not in second_roots - assert bool(second_roots) == ( - telemetry_evidence.explicit_sampling(header) is not False - ) - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -def test_failed_export_does_not_close_provider_or_corrupt_next_invocation( - monkeypatch, view -): - first = run(monkeypatch, view, "failed-export", mode="failure") - second = run(monkeypatch, view, "healthy") - telemetry_evidence.warm_reuse(first + second) - assert all(e["outcome"] == "success" for e in second if e["phase"] == "OTEL_EXPORT") - - -@pytest.mark.parametrize( - "mutation", ["execution", "trace", "parent", "cleanup", "missing", "duplicate"] -) -def test_evidence_rejects_corrupt_or_incomplete_telemetry(monkeypatch, mutation): - events = copy.deepcopy(run(monkeypatch, "invocation", "control")) - spans = [e for e in events if e["phase"] == "OTEL_EXPORT"] - root = next(e for e in spans if e["span"]["name"] == "Invocation") - if mutation == "execution": - root["span"]["attributes"]["durable.execution.arn"] = "other" - elif mutation == "trace": - root["span"]["trace_id"] = "0" * 32 - elif mutation == "parent": - root["span"]["parent_id"] = "0" * 16 - elif mutation == "cleanup": - next(e for e in events if e["phase"] == "OTEL_EXIT")["context_restored"] = False - elif mutation == "missing": - events.remove(root) - else: - events.append(copy.deepcopy(root)) - with pytest.raises((AssertionError, telemetry_evidence.CollectionError)): - telemetry_evidence.invocation(events) - - -@pytest.mark.parametrize("view", ["invocation", "execution"]) -@pytest.mark.parametrize("failed", [False, True]) -def test_public_runner_suspends_replays_and_finalizes_once(monkeypatch, view, failed): - monkeypatch.setenv("_X_AMZN_TRACE_ID", HEADER) - traces = [] - bodies = [] - - def business(event, context): - value = context.step( - lambda _: bodies.append("saved") or event["marker"], name="saved" - ) - context.wait(Duration.from_seconds(1), name="pause") - if failed: - raise ValueError("expected failure") - return value - - def entry(event, context): - invocation_input = ( - event - if hasattr(event, "durable_execution_arn") - else DurableExecutionInvocationInput.from_json_dict(event) - ) - diagnostic = MemoryTrace( - {"marker": "replay", "otel_view": view}, - context, - invocation_input.durable_execution_arn, - ) - # The local runner uses a fixed Lambda-context test double. Give each - # controlled attempt a diagnostic ID; native tests use real request IDs. - diagnostic.identity["request"] = "controlled-" + str(len(traces)) - traces.append(diagnostic) - return telemetry.invoke( - diagnostic, - lambda plugins: durable_execution(business, plugins=plugins)( - event, context - ), - ) - - with DurableFunctionTestRunner(entry, poll_interval=0.01) as runner: - result = runner.run(input={"marker": "replay"}, execution_timeout=15) - assert result.status.value == ("FAILED" if failed else "SUCCEEDED") - assert bodies == ["saved"] - assert len(traces) >= 2 - events = [event for diagnostic in traces for event in diagnostic.events] - telemetry_evidence.invocation(events) - telemetry_evidence.terminal(events, failed=failed) - - -@pytest.mark.parametrize("raises", [False, True]) -def test_instrumented_invoke_owns_header_client_and_closes_it(monkeypatch, raises): - from types import SimpleNamespace - from unittest.mock import Mock - from lmi_tests import cloud as module - - isolated = Mock() - owner = SimpleNamespace(lam=Mock()) - monkeypatch.setattr(module, "client", lambda name: isolated) - expected = ( - "Root=1-6955b900-123456789012345678901234;Parent=1234567890123456;Sampled=0" - ) - - def send(**kwargs): - event, hook = isolated.meta.events.register.call_args.args - assert event == "before-call.lambda.Invoke" - params = {"headers": {}} - hook(params=params) - assert params["headers"]["X-Amzn-Trace-Id"] == expected - assert kwargs["Payload"] == b"payload" - if raises: - raise RuntimeError("transport failed") - return {"DurableExecutionArn": "execution"} - - isolated.invoke.side_effect = send - caller = module.RunnerClient(owner, expected) - if raises: - with pytest.raises(RuntimeError, match="transport failed"): - caller.invoke(Payload=b"payload") - else: - assert caller.invoke(Payload=b"payload")["DurableExecutionArn"] == "execution" - isolated.close.assert_called_once() - owner.lam.invoke.assert_not_called() - - -def test_post_call_evidence_does_not_move_the_sdk_boundary(monkeypatch): - original = MemoryTrace.emit - - def delayed(self, phase, **fields): - record = original(self, phase, **fields) - if phase == "OTEL_EXIT": - time.sleep(0.02) # Controlled diagnostic I/O latency, not a durable wait. - return record - - monkeypatch.setattr(MemoryTrace, "emit", delayed) - events = run(monkeypatch, "invocation", "boundary") - boundary = next(event for event in events if event["phase"] == "WRAPPER_RETURN") - evidence_exit = next(event for event in events if event["phase"] == "OTEL_EXIT") - assert boundary["time"] <= evidence_exit["time"] diff --git a/pyproject.toml b/pyproject.toml index 8f6cd41a3..a9ce7947b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,23 +33,17 @@ python = "3.14" workspace.members = [ "packages/aws-durable-execution-sdk-python", "packages/aws-durable-execution-sdk-python-testing", - "packages/aws-durable-execution-sdk-python-otel", -] -dependencies = [ - "boto3>=1.43.0", "pytest", "ruff", "build", - "opentelemetry-sdk>=1.20.0", "opentelemetry-propagator-aws-xray", ] +dependencies = ["boto3>=1.43.0", "pytest", "ruff", "build", "mypy"] [tool.hatch.envs.lmi.scripts] unit = "pytest lmi_tests/tests/unit {args}" -cloud = "pytest lmi_tests/tests/e2e --lmi-cloud {args}" -regressions = "pytest lmi_tests/tests/regressions --run-lmi-regressions {args}" lint = "ruff check lmi_tests && ruff format --check lmi_tests" build = "python -m lmi_tests.deploy build {args}" shared-setup = "python -m lmi_tests.shared setup {args}" shared = "python -m lmi_tests.shared run {args}" shared-cleanup = "python -m lmi_tests.shared cleanup {args}" -shared-tests = "python -m pytest lmi_tests/tests/unit lmi_tests/tests/e2e --ignore=lmi_tests/tests/e2e/test_lifecycle.py --ignore=lmi_tests/tests/e2e/test_observability.py {args}" +shared-tests = "python -m pytest lmi_tests/tests/unit lmi_tests/tests/e2e {args}" [tool.pytest.ini_options] # Declare custom markers to avoid warnings with --strict-markers From ecb0cb3499f248093a51e9e268720c566ce9da0b Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:16:17 -0700 Subject: [PATCH 21/32] fix: reuse LMI completion timestamp --- lmi_tests/fixture.py | 7 ++-- .../e2e/test_shared_observations_local.py | 11 +++++++ .../tests/unit/test_shared_observations.py | 32 ++++++++++++++++++- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index bb3a215aa..ff083634a 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -677,9 +677,12 @@ def invoke(plugins=None): # Preserve the actual SDK boundary before post-call evidence I/O. completed_at_ns = time.time_ns() trace.sdk_completed_at = completed_at_ns / 1_000_000_000 + # Both semantic records describe this same boundary. Converting + # seconds back to milliseconds can round to a different float. + trace.sdk_completed_at_ms = completed_at_ns / 1_000_000 if payload.get("coverageProfile"): trace.sdk_observation = trace.capture_observation( - "sdk_completion", atMs=completed_at_ns / 1_000_000 + "sdk_completion", atMs=trace.sdk_completed_at_ms ) trace.sdk_resources = resources() @@ -716,7 +719,7 @@ def invoke(plugins=None): observe( trace, "pending", - atMs=trace.sdk_completed_at * 1000, + atMs=trace.sdk_completed_at_ms, source="sdk-wrapper", ) if ( diff --git a/lmi_tests/tests/e2e/test_shared_observations_local.py b/lmi_tests/tests/e2e/test_shared_observations_local.py index 49fa61e08..956aafc60 100644 --- a/lmi_tests/tests/e2e/test_shared_observations_local.py +++ b/lmi_tests/tests/e2e/test_shared_observations_local.py @@ -81,6 +81,17 @@ def test_actual_service_replay_keeps_operation_identity_and_skips_completed_bodi for item in replay["operations"] ) assert "PENDING" in {item["outcome"] for item in values(ledger, "wrapper_return")} + observations = ledger.observations() + for record in observations: + if record["observation"]["kind"] != "pending": + continue + wrapper = next( + item["observation"] + for item in observations + if item["request"] == record["request"] + and item["observation"]["kind"] == "wrapper_return" + ) + assert record["observation"]["atMs"] == wrapper["atMs"] def test_warm_invocation_observes_late_work_under_its_original_request(monkeypatch): diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py index bcdedfd3e..5a78892fb 100644 --- a/lmi_tests/tests/unit/test_shared_observations.py +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -11,6 +11,7 @@ import threading import time from typing import Any +from types import SimpleNamespace import pytest @@ -31,7 +32,7 @@ LifecycleObservations, OperationObservation, ) -from lmi_tests.tests.unit.support import lambda_context +from lmi_tests.tests.unit.support import invocation, lambda_context class MemoryLedger: @@ -393,3 +394,32 @@ def test_failed_real_client_call_keeps_its_dispatch_and_exit_identity( exited = next(item for item in observations if item["kind"] == prefix + "_exit") assert entered["callId"] == exited["callId"] and entered["atMs"] <= exited["atMs"] assert not any(item["kind"] == "checkpoint_ack" for item in observations) + + +@pytest.mark.parametrize("completed_ns", [1794607984811568141, 1794607984811569459]) +def test_pending_and_wrapper_return_share_one_captured_timestamp( + monkeypatch, completed_ns +): + ledger = install_ledger(monkeypatch) + original = invocation() + payload = original.to_json_dict() + payload["InitialExecutionState"]["Operations"][0]["ExecutionDetails"][ + "InputPayload" + ] = json.dumps(shared_payload()) + injected = fixture.DurableExecutionInvocationInputWithClient.from_durable_execution_invocation_input( + fixture.DurableExecutionInvocationInput.from_json_dict(payload), + original.service_client, + ) + monkeypatch.setattr( + fixture, "time", SimpleNamespace(time=time.time, time_ns=lambda: completed_ns) + ) + monkeypatch.setattr( + fixture, + "durable_execution", + lambda *args, **kwargs: lambda *args: {"Status": "PENDING"}, + ) + assert fixture.handler(injected, lambda_context(30000)) == {"Status": "PENDING"} + observations = [record["observation"] for record in ledger.observations()] + wrapper = next(item for item in observations if item["kind"] == "wrapper_return") + pending = next(item for item in observations if item["kind"] == "pending") + assert pending["atMs"] == wrapper["atMs"] == completed_ns / 1_000_000 From 2c8162b0f8097f461ed1ca894c5f8be707b3dc29 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:25:43 -0700 Subject: [PATCH 22/32] fix: handle unacquired LMI run leases --- lmi_tests/shared.py | 18 ++++++++++ lmi_tests/shared_lock.py | 6 +++- lmi_tests/tests/unit/test_shared_cleanup.py | 37 +++++++++++++++++++++ lmi_tests/tests/unit/test_shared_lock.py | 13 ++++++++ 4 files changed, 73 insertions(+), 1 deletion(-) diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 12db36cb9..7c867642a 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -224,6 +224,24 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: This is the final CI recovery path after the runner process exits. It never rewrites assertion results or clears quarantine based on service status alone. """ + if "sharedLease" not in deployment: + if ( + deployment.get("owned") + or deployment.get("functions") + or any(directory.glob("*/manifest.json")) + ): + raise OwnershipError( + "Deployment or case records exist without a run lease; cleanup cannot verify ownership" + ) + save( + directory / "cleanup.json", + { + "runId": deployment["run"], + "status": "SKIPPED_NO_LEASE", + "reason": "Deployment did not acquire target ownership; no resource cleanup was attempted", + }, + ) + return 0 receipt = directory / "ownership-released.json" lease = RunLease.from_manifest( {"deployment": deployment, "lease": deployment["sharedLease"]} diff --git a/lmi_tests/shared_lock.py b/lmi_tests/shared_lock.py index 663549d61..e60735154 100644 --- a/lmi_tests/shared_lock.py +++ b/lmi_tests/shared_lock.py @@ -66,8 +66,12 @@ def acquire( or not previous["profiles"] or any(value != "quiescent" for value in previous["profiles"].values()) ): + owner = { + name: previous.get(name) for name in ("runId", "state", "profiles") + } raise OwnershipError( - "Persistent LMI targets are owned or quarantined by another run" + "Persistent LMI targets are owned or quarantined by another run: " + + json.dumps(owner, sort_keys=True) ) record = { "schemaVersion": 1, diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py index 1ca538698..1a5658ca2 100644 --- a/lmi_tests/tests/unit/test_shared_cleanup.py +++ b/lmi_tests/tests/unit/test_shared_cleanup.py @@ -92,3 +92,40 @@ def test_case_ownership_is_checked_before_control_release( ) lease.finish_profile.assert_called_once_with(target, False) lease.release.assert_not_called() + + +def test_failed_lease_acquisition_skips_cleanup_without_touching_another_run( + monkeypatch, tmp_path +): + from_manifest = Mock(side_effect=AssertionError("No lease was acquired")) + client = Mock(side_effect=AssertionError("Unowned resources must not be touched")) + monkeypatch.setattr(shared.RunLease, "from_manifest", from_manifest) + monkeypatch.setattr(shared, "client", client) + assert shared.cleanup({"run": "new-run", "functions": {}}, tmp_path) == 0 + assert ( + json.loads((tmp_path / "cleanup.json").read_text())["status"] + == "SKIPPED_NO_LEASE" + ) + assert not (tmp_path / "ownership-released.json").exists() + from_manifest.assert_not_called() + client.assert_not_called() + + +@pytest.mark.parametrize("evidence", ["owned", "functions", "case-plan"]) +def test_missing_lease_with_prior_activity_is_not_treated_as_unacquired( + monkeypatch, tmp_path, evidence +): + deployment = {"run": "run", "functions": {}} + if evidence == "case-plan": + output = tmp_path / "standard" + output.mkdir() + (output / "manifest.json").write_text("{}") + else: + deployment[evidence] = True if evidence == "owned" else {"c1": "arn:owned"} + client = Mock( + side_effect=AssertionError("Cannot mutate resources without ownership") + ) + monkeypatch.setattr(shared, "client", client) + with pytest.raises(OwnershipError, match="cannot verify ownership"): + shared.cleanup(deployment, tmp_path) + client.assert_not_called() diff --git a/lmi_tests/tests/unit/test_shared_lock.py b/lmi_tests/tests/unit/test_shared_lock.py index 1e8e3d6f2..c353117e1 100644 --- a/lmi_tests/tests/unit/test_shared_lock.py +++ b/lmi_tests/tests/unit/test_shared_lock.py @@ -108,3 +108,16 @@ def test_unknown_retirement_state_cannot_be_claimed_or_cleared(state): with pytest.raises(OwnershipError): lease.finish_profile("c1", True) assert store.record["profiles"]["c1"] == state + + +def test_blocked_acquisition_identifies_owner_without_exposing_fence_or_mutating_lock(): + store = Store() + owner = RunLease.acquire("private", "stack", "original-run", ("c1", "c2"), s3=store) + owner.finish_profile("c1", False) + before = json.dumps(store.record, sort_keys=True) + with pytest.raises(OwnershipError) as error: + RunLease.acquire("private", "stack", "new-run", ("c1", "c2"), s3=store) + assert "original-run" in str(error.value) + assert "quarantined" in str(error.value) + assert owner.token not in str(error.value) + assert json.dumps(store.record, sort_keys=True) == before From 0996b2eeadc2be6199d95b6a6599d0635b10f28c Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:33:51 -0700 Subject: [PATCH 23/32] refactor: use Actions for LMI run concurrency --- .github/workflows/lmi-e2e-tests.yml | 4 +- lmi_tests/README.md | 35 ++- lmi_tests/deploy.py | 11 - lmi_tests/shared-suite-design.md | 6 +- lmi_tests/shared.py | 162 ++++-------- lmi_tests/shared_adapter.py | 23 +- lmi_tests/shared_lock.py | 243 ------------------ lmi_tests/summary.py | 26 +- lmi_tests/tests/unit/test_cloud.py | 13 +- lmi_tests/tests/unit/test_deploy.py | 35 +-- lmi_tests/tests/unit/test_shared_adapter.py | 31 +-- lmi_tests/tests/unit/test_shared_cleanup.py | 152 ++++++----- lmi_tests/tests/unit/test_shared_lock.py | 123 --------- lmi_tests/tests/unit/test_shared_manifest.py | 46 +++- .../unit/test_shared_partial_evidence.py | 1 - 15 files changed, 253 insertions(+), 658 deletions(-) delete mode 100644 lmi_tests/shared_lock.py delete mode 100644 lmi_tests/tests/unit/test_shared_lock.py diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index f0fa10e93..2bca89579 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -71,8 +71,8 @@ jobs: permissions: contents: read id-token: write - # All runs use the same provider. Hold this slot through cleanup; do not - # include a PR/ref/run ID in the group. + # GitHub Actions is the cross-run concurrency guard. Hold this slot through + # cleanup and use the same group for every PR/ref targeting this provider. concurrency: group: lmi-e2e-shared-capacity-provider cancel-in-progress: false diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 921e3c93f..4c0561d0b 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -16,8 +16,8 @@ shared matrix directly rather than maintaining a second requirement catalog. | `lmi-lifecycle-6` | `warm-reuse` | c1 | victim, warm | | `lmi-lifecycle-7` | `normal-failure` | c1 | victim | -The shared runner owns assertions, strict reports, placement identity, leases, -launch journals and independent retirement auditing. This repository owns the +The shared runner owns assertions, strict reports, placement identity, per-case +scheduling, launch journals and independent retirement auditing. This repository owns the real public SDK handlers, AWS transport, source observations and deployment. OTel/exporter tests and the previous Python-specific matrix are deferred. There is no local conformance profile or validator that replaces a shared rule. @@ -56,16 +56,11 @@ persistent functions use Python 3.14, arm64, 2048 MiB and native concurrency 1 a 2, with one environment each. SDK artifact and configuration digests are retained in every manifest and checked before invoking either target. -```bash -export AWS_REGION=us-west-2 -export LMI_SHARED_RUN_ID=your-unique-run-id -# Set CAPACITY_PROVIDER_ARN, TEST_ACCOUNT_ID and TEST_LAMBDA_EXECUTION_ROLE_ARN. -hatch run lmi:build -hatch run lmi:python -m lmi_tests.deploy deploy \ - --stack-name python-lmi-e2e --run-id "$LMI_SHARED_RUN_ID" --runtime python3.14 -hatch run lmi:shared -hatch run lmi:shared-cleanup -``` +Run cloud tests through the workflow's `workflow_dispatch` action or a trusted +same-repository PR. Its fixed `lmi-e2e-shared-capacity-provider` concurrency group +serializes cloud jobs across PRs and refs from deployment through final cleanup, +with `cancel-in-progress: false`. GitHub Actions is the only cross-run concurrency +control. Direct CLI calls do not acquire a lock and are not serialized with CI. `shared plan` writes the seven-case manifest without invoking functions. `shared run` writes under `lmi_tests/artifacts/shared/standard/`: the manifest, coverage, @@ -75,8 +70,8 @@ the same durable execution; it is not another driver Invoke. Trusted same-repository pull requests and manual workflow runs exercise the cloud suite. Fork and Dependabot pull requests run only the unprivileged local harness. -Runs using the same capacity provider are serialized through cleanup. A conditional -S3 run lease also prevents overlapping deployments or reuse of quarantined work. +There is no S3 lease, fence token or persistent quarantine record. Existing S3 +ownership objects are not read and cannot block a later workflow run. ## Actual observations and failure handling @@ -121,8 +116,10 @@ Coordination barriers retain their complete `controlRecords` journal separately from injected victim `faultsCreated`; releasing a healthy peer is not victim fault recovery. -Cleanup releases only the current run's controls and stops only its identified -executions. It does not erase assertion errors or clear ownership simply because -the runner process ended. A failed case can release its target only after all -actual launches and tasks have independently settled. Unknown invocations, -missing evidence and unresolved late work retain quarantine for reconciliation. +Cleanup releases only the current run's controls and requests stops only for its +identified executions. It validates run, case and function identity before those +requests. It records `NO_CASES`, `CLEANUP_REQUESTED` or `FAILED` in `cleanup.json`; +completed cleanup requests do not claim that every task retired. Assertion and +retirement failures remain in the shared reports. The shared runner can prevent +reuse of an unretired target within the current run, but that state is not stored +as a lock for future workflow runs. diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 37648c796..2da90e2c5 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -20,7 +20,6 @@ from botocore.exceptions import ClientError from lmi_tests.evidence import ProvisioningError -from lmi_tests.shared_lock import RunLease ROOT = Path(__file__).resolve().parents[1] @@ -473,16 +472,6 @@ def deploy_stack(manifest, data): raise ProvisioningError( "Evidence bucket must be private before uploading the tested artifact" ) - partition = manifest["provider"].split(":")[1] - targets = tuple( - f"arn:{partition}:lambda:{manifest['region']}:{manifest['account']}:function:{manifest['stack']}-{key}:{QUALIFIER}" - for key in CONCURRENCIES - ) - lease = RunLease.acquire( - manifest["bucket"], manifest["stack"], manifest["run"], targets, s3=storage - ) - manifest["sharedLease"] = lease.to_manifest() - save(ARTIFACTS / "manifest.json", manifest) if stack is not None: wait_for_idle_functions(stack) manifest["owned"] = True diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index 71d9e3496..c4fe3f888 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -25,4 +25,8 @@ acceptance or fix the underlying SDK lifecycle behavior. The core-only Lambda bundle excludes driver code and conformance packages. CI retains the dedicated test-account boundary, trusted-PR condition, exact dependency -pin, conditional ownership lease and final reconciliation path. +pin and run-scoped cleanup. GitHub Actions is the sole cross-run concurrency +guard: the fixed cloud-job group covers deployment, assertions and cleanup, with +`cancel-in-progress: false`. The adapter has no shared-lock module, S3 lease, +fence token or persisted quarantine. The shared runner still schedules individual +cases and reports retirement failures within each run. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 7c867642a..5d5aed898 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -17,7 +17,7 @@ from lmi_tests.deploy import ARTIFACTS, ROOT, client, save from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity -from lmi_tests.shared_lock import OwnershipError, RunLease +from lmi_tests.evidence import CollectionError COVERAGE_PROFILES = ("standard",) @@ -72,10 +72,6 @@ def make_manifest( if coverage not in COVERAGE_PROFILES: raise ValueError("Unknown Python adapter coverage profile") - if "sharedLease" not in deployment: - raise ValueError( - "Deployment was not prepared with persistent shared-run ownership" - ) actual = deepcopy(dict(deployment)) actual["conformanceRevision"] = CONFORMANCE_REVISION core_wheels = { @@ -142,7 +138,6 @@ def make_manifest( "sdk": { "deployment": actual, "artifacts_dir": str(directory.resolve() / "source"), - "lease": actual["sharedLease"], }, } @@ -165,115 +160,73 @@ def run(deployment: Mapping[str, Any], directory: Path) -> int: raise ValueError( "Shared run output must be empty; preserve prior attempts and failure evidence" ) - lease = RunLease.from_manifest( - {"deployment": deployment, "lease": deployment["sharedLease"]} - ) exit_code = 0 - completed = False - try: - for coverage in COVERAGE_PROFILES: - output = directory / coverage - output.mkdir() - plan = make_manifest(deployment, coverage, output) - path = output / "manifest.json" - save(path, plan) - validated = load_manifest(path) - results = asyncio.run( - live_results( - validated, - "lmi_tests.shared_adapter:create_adapter", - audit_dir=output, - ) - ) - report = build_report(validated, results) - save(output / "evidence.json", results) - save(output / "coverage.json", coverage_inventory(validated)) - print(render_console(report)) - for format_name in ("json", "junit"): - write_report(report, format_name, str(output / "report")) - exit_code = max(exit_code, report.exit_code()) - completed = True - finally: - if completed: - try: - lease.release() - save( - directory / "ownership-released.json", - {"runId": deployment["run"], "lease": lease.to_manifest()}, - ) - except OwnershipError as error: - save( - directory / "ownership-retained.json", - {"error": str(error), "lease": lease.to_manifest()}, - ) - exit_code = 1 - else: - save( - directory / "ownership-retained.json", - { - "reason": "Interrupted or incomplete shared run", - "lease": lease.to_manifest(), - }, + for coverage in COVERAGE_PROFILES: + output = directory / coverage + output.mkdir() + plan = make_manifest(deployment, coverage, output) + path = output / "manifest.json" + save(path, plan) + validated = load_manifest(path) + results = asyncio.run( + live_results( + validated, + "lmi_tests.shared_adapter:create_adapter", + audit_dir=output, ) + ) + report = build_report(validated, results) + save(output / "evidence.json", results) + save(output / "coverage.json", coverage_inventory(validated)) + print(render_console(report)) + for format_name in ("json", "junit"): + write_report(report, format_name, str(output / "report")) + exit_code = max(exit_code, report.exit_code()) return exit_code def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: - """Release only persisted case controls; unknown retirement retains ownership. + """Release this run's controls and request stops without altering its reports. - This is the final CI recovery path after the runner process exits. It never - rewrites assertion results or clears quarantine based on service status alone. + GitHub Actions holds the cloud job's concurrency slot through this step. + Successful requests do not imply that every runtime task has retired. """ - if "sharedLease" not in deployment: - if ( - deployment.get("owned") - or deployment.get("functions") - or any(directory.glob("*/manifest.json")) - ): - raise OwnershipError( - "Deployment or case records exist without a run lease; cleanup cannot verify ownership" - ) + plans = [] + for coverage in COVERAGE_PROFILES: + path = directory / coverage / "manifest.json" + if not path.exists(): + continue + plan = json.loads(path.read_text()) + if plan.get("runId") != deployment["run"]: + raise CollectionError("Cleanup manifest belongs to another run") + profiles = {profile["profile_id"]: profile for profile in plan["profiles"]} + for case in plan["scenarios"]: + target = profiles[case["profile_id"]]["resource_id"] + if target not in deployment["functions"].values(): + raise CollectionError("Cleanup target is outside this deployment") + plans.append((coverage, plan)) + if not plans: save( directory / "cleanup.json", { "runId": deployment["run"], - "status": "SKIPPED_NO_LEASE", - "reason": "Deployment did not acquire target ownership; no resource cleanup was attempted", + "status": "NO_CASES", + "errors": [], + "reason": "No shared case manifests were recorded; no cleanup requests were sent", }, ) return 0 - receipt = directory / "ownership-released.json" - lease = RunLease.from_manifest( - {"deployment": deployment, "lease": deployment["sharedLease"]} - ) - if receipt.exists() and json.loads(receipt.read_text()) == { - "runId": deployment["run"], - "lease": lease.to_manifest(), - }: - return 0 - record, _ = lease._read() - record = lease._owned(record) - if all(value == "quiescent" for value in record["profiles"].values()): - lease.release() - save(receipt, {"runId": deployment["run"], "lease": lease.to_manifest()}) - return 0 storage, service = ( client("s3", deployment["region"]), client("lambda", deployment["region"]), ) errors: list[str] = [] - for coverage in COVERAGE_PROFILES: - plan_path = directory / coverage / "manifest.json" - if not plan_path.exists(): - continue - plan = json.loads(plan_path.read_text()) - if plan.get("runId") != deployment["run"]: - raise OwnershipError("Cleanup manifest belongs to another run") + for coverage, plan in plans: profiles = {profile["profile_id"]: profile for profile in plan["profiles"]} for case in plan["scenarios"]: target = profiles[case["profile_id"]]["resource_id"] if target not in deployment["functions"].values(): - raise OwnershipError("Cleanup target is outside the owned deployment") + raise CollectionError("Cleanup target is outside the owned deployment") owner = hashlib.sha256( f"{case['case_id']}:{case.get('cohort_id', 'cohort-1')}".encode() ).hexdigest()[:20] @@ -288,7 +241,7 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: for item in page.get("Contents", []) ] if any(not key.startswith(prefix) for key in objects): - raise OwnershipError( + raise CollectionError( "Control listing escaped the case-owned prefix" ) executions: set[str] = set() @@ -299,12 +252,13 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: payload.get("run") != deployment["run"] or payload.get("caseId") != case["case_id"] ): - raise OwnershipError( + raise CollectionError( "Invocation intent belongs to another case" ) if invocation.get("arn"): executions.add(invocation["arn"]) else: + recovered_executions: set[str] = set() marker = payload["marker"] event_prefix = f"runs/{deployment['run']}/events/{marker}/" for page in storage.get_paginator("list_objects_v2").paginate( @@ -319,11 +273,12 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: event.get("run") != deployment["run"] or event.get("caseId") != case["case_id"] ): - raise OwnershipError( + raise CollectionError( "Recovered invocation evidence belongs to another case" ) - executions.add(event["execution"]) - if not executions: + recovered_executions.add(event["execution"]) + executions.update(recovered_executions) + if not recovered_executions: errors.append( f"Unreconciled Invoke: {case['case_id']}/{marker}" ) @@ -331,7 +286,7 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: not arn.startswith(target + "/durable-execution/") for arn in executions ): - raise OwnershipError("Refusing to stop a foreign execution") + raise CollectionError("Refusing to stop a foreign execution") for key in sorted( objects, key=lambda value: value != prefix + "release-all" ): @@ -340,7 +295,7 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: ] as stream: state = stream.read() if state not in {b"hold", b"release"}: - raise OwnershipError( + raise CollectionError( "Unknown control content; refusing to overwrite it" ) if state == b"hold": @@ -364,19 +319,16 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: raise except Exception as error: errors.append(f"{case['case_id']}: {type(error).__name__}: {error}") - for target, status in record["profiles"].items(): - if status != "quiescent": - lease.finish_profile(target, False) save( directory / "cleanup.json", { "runId": deployment["run"], "errors": errors, - "status": "OWNERSHIP_RETAINED", - "reason": "Fault release and Stop do not prove task/request quiescence; retained run requires reconciliation", + "status": "FAILED" if errors else "CLEANUP_REQUESTED", + "reason": "Run-scoped control release and execution-stop requests; case reports retain task retirement findings", }, ) - return 1 + return int(bool(errors)) def main() -> None: @@ -399,7 +351,7 @@ def main() -> None: deployment = json.loads(args.manifest.read_text()) expected_run = os.environ.get("LMI_SHARED_RUN_ID") if expected_run and deployment.get("run") != expected_run: - raise OwnershipError("Deployment manifest belongs to a different CI run") + raise CollectionError("Deployment manifest belongs to a different CI run") if args.command == "cleanup": raise SystemExit(cleanup(deployment, args.output)) if args.command == "plan": diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py index 660cf434f..26aabeaae 100644 --- a/lmi_tests/shared_adapter.py +++ b/lmi_tests/shared_adapter.py @@ -23,11 +23,6 @@ from lmi_tests.shared_contract import configuration_identity -class RunOwnership(Protocol): - def assert_owned(self, resource_id: str) -> None: ... - def finish_profile(self, resource_id: str, quiescent: bool) -> None: ... - - class EvidenceBuilder(Protocol): def __call__( self, context: ScenarioContext, cloud: Cloud, *, phase: str @@ -74,7 +69,6 @@ def __init__( self, manifest: Mapping[str, Any], *, - ownership: RunOwnership, evidence_builder: EvidenceBuilder, cloud_factory: Callable[..., Cloud] = Cloud, ) -> None: @@ -82,14 +76,12 @@ def __init__( self.sdk = self.manifest["sdk"] self.deployment = self.sdk["deployment"] self.artifacts_dir = Path(self.sdk["artifacts_dir"]) - self.ownership = ownership self.evidence_builder = evidence_builder self.cloud_factory = cloud_factory self.cases: dict[str, _Case] = {} async def prepare(self, profile: Profile) -> None: def verify() -> None: - self.ownership.assert_owned(profile.resource_id) if profile.resource_id not in self.deployment["functions"].values(): raise ProvisioningError( "Shared profile is not part of the owned SDK deployment" @@ -125,9 +117,6 @@ def _case(self, context: ScenarioContext) -> _Case: async def start(self, context: ScenarioContext) -> None: if context.scenario.case_id in self.cases: raise CollectionError("A later scenario cannot replace an admitted case") - await asyncio.to_thread( - self.ownership.assert_owned, context.profile.resource_id - ) key = next( ( key @@ -244,15 +233,13 @@ def retire() -> bool: return await case.call(retire) async def finish(self, profile: Profile, *, quiescent: bool) -> None: - await asyncio.to_thread( - self.ownership.finish_profile, profile.resource_id, quiescent - ) + # The shared runner retains per-case retirement results. Cross-run + # serialization ends with the GitHub Actions cloud job. + return None def create_adapter(manifest: Mapping[str, Any]) -> SharedAdapter: - """Factory used by the shared CLI; persistent ownership is mandatory.""" + """Create an adapter for the cloud job serialized by GitHub Actions.""" from lmi_tests.shared_evidence import build_evidence - from lmi_tests.shared_lock import RunLease - ownership = RunLease.from_manifest(manifest["sdk"]) - return SharedAdapter(manifest, ownership=ownership, evidence_builder=build_evidence) + return SharedAdapter(manifest, evidence_builder=build_evidence) diff --git a/lmi_tests/shared_lock.py b/lmi_tests/shared_lock.py deleted file mode 100644 index e60735154..000000000 --- a/lmi_tests/shared_lock.py +++ /dev/null @@ -1,243 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Conditional S3 ownership for one persistent SDK deployment. - -The lease token fences concurrent runs; it is not an authentication credential. -S3 access continues to require the caller's normal SigV4/IAM authorization. -There is deliberately no expiry-based stealing of a run with residual work. -""" - -from __future__ import annotations - -from dataclasses import dataclass -import json -from typing import Any, Mapping -import uuid - -import boto3 -from botocore.config import Config -from botocore.exceptions import ClientError - - -class OwnershipError(RuntimeError): - """A persistent target cannot be safely acquired, updated, or released.""" - - -@dataclass(frozen=True) -class RunLease: - bucket: str - key: str - run_id: str - token: str - resources: tuple[str, ...] - s3: Any - - @classmethod - def acquire( - cls, - bucket: str, - stack: str, - run_id: str, - resources: tuple[str, ...], - *, - s3: Any = None, - ) -> RunLease: - lease = cls( - bucket, - f"ownership/{stack}/shared-run.json", - run_id, - uuid.uuid4().hex, - tuple(sorted(resources)), - s3 - if s3 is not None - else boto3.client( - "s3", - config=Config( - connect_timeout=3, - read_timeout=10, - retries={"total_max_attempts": 2}, - ), - ), - ) - previous, etag = lease._read() - if previous is not None and ( - previous.get("state") != "released" - or not isinstance(previous.get("profiles"), dict) - or not previous["profiles"] - or any(value != "quiescent" for value in previous["profiles"].values()) - ): - owner = { - name: previous.get(name) for name in ("runId", "state", "profiles") - } - raise OwnershipError( - "Persistent LMI targets are owned or quarantined by another run: " - + json.dumps(owner, sort_keys=True) - ) - record = { - "schemaVersion": 1, - "runId": run_id, - "token": lease.token, - "state": "owned", - "profiles": {resource: "active" for resource in lease.resources}, - } - try: - lease._write(record, etag) - except ClientError as error: - if error.response["Error"]["Code"] in { - "PreconditionFailed", - "ConditionalRequestConflict", - }: - raise OwnershipError( - "Another run acquired the persistent LMI targets" - ) from error - raise - return lease - - @classmethod - def from_manifest(cls, sdk: Mapping[str, Any]) -> RunLease: - record = sdk.get("lease") - if not isinstance(record, dict): - raise OwnershipError( - "Shared LMI execution requires an acquired persistent run lease" - ) - required = ("bucket", "key", "runId", "token", "resources") - if any(key not in record for key in required): - raise OwnershipError("Run lease manifest is incomplete") - deployment = sdk["deployment"] - if ( - record["bucket"] != deployment["bucket"] - or record["runId"] != deployment["run"] - or record["key"] != f"ownership/{deployment['stack']}/shared-run.json" - or set(record["resources"]) != set(deployment["functions"].values()) - ): - raise OwnershipError("Run lease does not belong to this SDK deployment") - return cls( - record["bucket"], - record["key"], - record["runId"], - record["token"], - tuple(record["resources"]), - boto3.client( - "s3", - region_name=sdk["deployment"]["region"], - config=Config( - connect_timeout=3, - read_timeout=10, - retries={"total_max_attempts": 2}, - ), - ), - ) - - def to_manifest(self) -> dict[str, Any]: - return { - "bucket": self.bucket, - "key": self.key, - "runId": self.run_id, - "token": self.token, - "resources": list(self.resources), - } - - def _read(self) -> tuple[dict[str, Any] | None, str | None]: - try: - response = self.s3.get_object(Bucket=self.bucket, Key=self.key) - except ClientError as error: - if error.response["Error"]["Code"] in {"NoSuchKey", "404", "NotFound"}: - return None, None - raise - with response["Body"] as body: - record = json.loads(body.read()) - if ( - not isinstance(record, dict) - or type(record.get("schemaVersion")) is not int - or record["schemaVersion"] != 1 - ): - raise OwnershipError("Stored run ownership is malformed") - return record, response["ETag"] - - def _write(self, record: Mapping[str, Any], etag: str | None) -> None: - condition = {"IfMatch": etag} if etag is not None else {"IfNoneMatch": "*"} - self.s3.put_object( - Bucket=self.bucket, - Key=self.key, - Body=json.dumps(record, sort_keys=True).encode(), - ContentType="application/json", - **condition, - ) - - def _owned(self, record: Mapping[str, Any] | None) -> dict[str, Any]: - if ( - not isinstance(record, dict) - or record.get("state") != "owned" - or record.get("runId") != self.run_id - or record.get("token") != self.token - or not isinstance(record.get("profiles"), dict) - or set(record["profiles"]) != set(self.resources) - or any( - not isinstance(value, str) - or value not in {"active", "quiescent", "quarantined"} - for value in record["profiles"].values() - ) - ): - raise OwnershipError( - "Persistent target ownership no longer matches this run" - ) - return record - - def assert_owned(self, resource_id: str) -> None: - """Verify the owner and mark a previously settled profile active again.""" - for _ in range(3): - record, etag = self._read() - record = self._owned(record) - if ( - resource_id not in self.resources - or record["profiles"][resource_id] == "quarantined" - ): - raise OwnershipError( - "The requested function is not owned or has residual work" - ) - if record["profiles"][resource_id] == "active": - return - record["profiles"][resource_id] = "active" - try: - self._write(record, etag) - return - except ClientError as error: - if error.response["Error"]["Code"] not in { - "PreconditionFailed", - "ConditionalRequestConflict", - }: - raise - raise OwnershipError("Could not reactivate the owned profile without a race") - - def finish_profile(self, resource_id: str, quiescent: bool) -> None: - if resource_id not in self.resources: - raise OwnershipError("Cannot retire another run's function") - for _ in range(3): - record, etag = self._read() - record = self._owned(record) - previous = record["profiles"][resource_id] - record["profiles"][resource_id] = ( - "quarantined" - if not quiescent or previous == "quarantined" - else "quiescent" - ) - try: - self._write(record, etag) - return - except ClientError as error: - if error.response["Error"]["Code"] not in { - "PreconditionFailed", - "ConditionalRequestConflict", - }: - raise - raise OwnershipError("Could not record retirement without racing another owner") - - def release(self) -> None: - record, etag = self._read() - record = self._owned(record) - if any(value != "quiescent" for value in record["profiles"].values()): - raise OwnershipError( - "Unretired LMI work retains persistent target ownership" - ) - record["state"] = "released" - self._write(record, etag) diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py index d01164263..79644f8f2 100644 --- a/lmi_tests/summary.py +++ b/lmi_tests/summary.py @@ -36,18 +36,20 @@ def summarize(directory): for path in sorted(directory.glob("*-error.json")): data = json.loads(path.read_text()) lines.append(f"- {path.stem}: {data['type']}") - receipt = shared / "ownership-released.json" - released = ( - config is not None - and receipt.exists() - and json.loads(receipt.read_text()) - == {"runId": config["run"], "lease": config.get("sharedLease")} - ) - lines.append( - "Run ownership released after retirement; persistent resources retained." - if released - else "Run cleanup not confirmed; ownership and persistent resources are retained." - ) + path = shared / "cleanup.json" + cleanup = json.loads(path.read_text()) if path.exists() else {} + if config is None or cleanup.get("runId") != config["run"]: + lines.append("Run cleanup not confirmed; persistent resources are retained.") + elif cleanup.get("status") == "NO_CASES": + lines.append("No scenario work was recorded for cleanup.") + elif cleanup.get("status") == "CLEANUP_REQUESTED" and cleanup.get("errors") == []: + lines.append( + "Run-scoped cleanup requests completed; see case reports for task retirement." + ) + else: + lines.append( + "Run cleanup failed or is incomplete; inspect cleanup diagnostics." + ) return "\n\n".join(lines) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py index 6ae0495c6..156779586 100644 --- a/lmi_tests/tests/unit/test_cloud.py +++ b/lmi_tests/tests/unit/test_cloud.py @@ -386,7 +386,7 @@ def test_case_collection_is_scoped_but_final_collection_is_complete(driver): driver.logs.get_paginator.assert_called_once_with("filter_log_events") -def test_summary_rejects_other_run_reports_and_requires_the_matching_release_receipt( +def test_summary_rejects_other_run_reports_and_requires_matching_cleanup( tmp_path, ): config = { @@ -394,7 +394,6 @@ def test_summary_rejects_other_run_reports_and_requires_the_matching_release_rec "runtime": "python3.14", "concurrencies": {"c1": 1, "c2": 2}, "run": "current", - "sharedLease": {"token": "fence"}, } (tmp_path / "manifest.json").write_text(json.dumps(config)) shared = tmp_path / "shared" @@ -407,15 +406,17 @@ def test_summary_rejects_other_run_reports_and_requires_the_matching_release_rec } ) ) - receipt = shared / "ownership-released.json" + receipt = shared / "cleanup.json" receipt.write_text( - json.dumps({"runId": "previous", "lease": config["sharedLease"]}) + json.dumps({"runId": "previous", "status": "CLEANUP_REQUESTED", "errors": []}) ) summary = summarize(tmp_path) assert "another run" in summary and "PASSED" not in summary assert "Run cleanup not confirmed" in summary - receipt.write_text(json.dumps({"runId": "current", "lease": config["sharedLease"]})) - assert "Run ownership released after retirement" in summarize(tmp_path) + receipt.write_text( + json.dumps({"runId": "current", "status": "CLEANUP_REQUESTED", "errors": []}) + ) + assert "Run-scoped cleanup requests completed" in summarize(tmp_path) def test_summary_does_not_accept_a_report_without_deployment_identity(tmp_path): diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index daeef0a65..16c693ebc 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -138,9 +138,6 @@ def apis(monkeypatch, tmp_path): clients = {name: Mock() for name in ("cloudformation", "s3", "lambda")} monkeypatch.setattr(deploy, "client", clients.__getitem__) monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) - from lmi_tests.tests.unit.test_shared_lock import Store - - lease_store = Store() clients["s3"].get_public_access_block.return_value = { "PublicAccessBlockConfiguration": { "BlockPublicAcls": True, @@ -149,14 +146,9 @@ def apis(monkeypatch, tmp_path): "RestrictPublicBuckets": True, } } - clients["s3"].get_object.side_effect = lease_store.get_object - - def store(**kwargs): - if kwargs["Key"].startswith("ownership/"): - return lease_store.put_object(**kwargs) - return {} - - clients["s3"].put_object.side_effect = store + clients["s3"].get_object.side_effect = AssertionError( + "Deployment must not read an S3 lock" + ) return clients @@ -173,22 +165,6 @@ def test_later_run_updates_fixed_functions_without_recreating_resources( verify = Mock() monkeypatch.setattr(deploy, "verify", verify) deploy.deploy_stack(manifest, b"first-code") - # No SDK work is launched by this deployment-only test. Retire its ownership - # window before testing a second revision against the same persistent targets. - from lmi_tests.shared_lock import RunLease - - record = manifest["sharedLease"] - lease = RunLease( - record["bucket"], - record["key"], - record["runId"], - record["token"], - tuple(record["resources"]), - apis["s3"], - ) - for resource in lease.resources: - lease.finish_profile(resource, True) - lease.release() second = { **manifest, "run": "second", @@ -198,6 +174,11 @@ def test_later_run_updates_fixed_functions_without_recreating_resources( deploy.deploy_stack(second, b"second-code") cfn.create_stack.assert_called_once() assert cfn.update_stack.call_count == 2 + apis["s3"].get_object.assert_not_called() + assert all( + not call.kwargs["Key"].startswith("ownership/") + for call in apis["s3"].put_object.call_args_list + ) templates = [ json.loads(call.kwargs["TemplateBody"]) for call in cfn.update_stack.call_args_list diff --git a/lmi_tests/tests/unit/test_shared_adapter.py b/lmi_tests/tests/unit/test_shared_adapter.py index 1de8bbb18..dcda6ca76 100644 --- a/lmi_tests/tests/unit/test_shared_adapter.py +++ b/lmi_tests/tests/unit/test_shared_adapter.py @@ -245,10 +245,9 @@ def release_all(self): self.actions.append("release") -def adapter(tmp_path, *, factory=FakeCloud, ownership=None): +def adapter(tmp_path, *, factory=FakeCloud): return adapter_module.SharedAdapter( {"sdk": {"deployment": deployment(), "artifacts_dir": str(tmp_path)}}, - ownership=ownership or Mock(spec=["assert_owned", "finish_profile", "release"]), evidence_builder=lambda context, cloud, *, phase: { "retirement": {"owner": context.scenario.case_id, "phase": phase} }, @@ -256,9 +255,8 @@ def adapter(tmp_path, *, factory=FakeCloud, ownership=None): ) -def test_prepare_checks_persistent_ownership_and_actual_profile(tmp_path): - owner = Mock(spec=["assert_owned", "finish_profile", "release"]) - value = adapter(tmp_path, ownership=owner) +def test_prepare_checks_the_actual_deployment_profile(tmp_path): + value = adapter(tmp_path) from lmi_tests.shared_contract import configuration_identity valid = Profile( @@ -272,19 +270,20 @@ def test_prepare_checks_persistent_ownership_and_actual_profile(tmp_path): asyncio.run(value.prepare(valid)) with pytest.raises(ProvisioningError, match="configuration identity"): asyncio.run(value.prepare(PROFILE)) - assert all(call.args == (FUNCTION,) for call in owner.assert_owned.call_args_list) assert FakeCloud.instances[-1].actions == ["verify"] bad = Profile("bad", FUNCTION, 1, 2, "sdk-sha", "configuration-sha") with pytest.raises(ProvisioningError, match="Native concurrency"): asyncio.run(value.prepare(bad)) -def test_lost_ownership_prevents_any_driver_launch(tmp_path): - owner = Mock(spec=["assert_owned", "finish_profile", "release"]) - owner.assert_owned.side_effect = RuntimeError("lease lost") - value = adapter(tmp_path, ownership=owner) - with pytest.raises(RuntimeError, match="lease lost"): - asyncio.run(value.start(context())) +def test_foreign_target_prevents_any_driver_launch(tmp_path): + value = adapter(tmp_path) + ctx = context() + ctx.profile = Profile( + "foreign", "arn:foreign", 1, 1, "sdk-sha", "configuration-sha" + ) + with pytest.raises(ProvisioningError, match="outside"): + asyncio.run(value.start(ctx)) assert value.cases == {} @@ -329,14 +328,6 @@ async def exercise(): asyncio.run(exercise()) -def test_finish_records_quarantine_without_releasing_the_whole_run(tmp_path): - owner = Mock(spec=["assert_owned", "finish_profile", "release"]) - value = adapter(tmp_path, ownership=owner) - asyncio.run(value.finish(PROFILE, quiescent=False)) - owner.finish_profile.assert_called_once_with(FUNCTION, False) - assert not owner.release.called - - class ScriptCloud: def __init__(self, native=1): self.concurrency = native diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py index 1a5658ca2..0c3a3df23 100644 --- a/lmi_tests/tests/unit/test_shared_cleanup.py +++ b/lmi_tests/tests/unit/test_shared_cleanup.py @@ -1,6 +1,6 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Cleanup never uses a stale receipt or mutates work before ownership checks.""" +"""Run-scoped cleanup needs no distributed lock and never touches foreign work.""" import hashlib import io @@ -10,45 +10,29 @@ import pytest from lmi_tests import shared -from lmi_tests.shared_lock import OwnershipError +from lmi_tests.evidence import CollectionError -def test_stale_receipt_cannot_skip_current_ownership_verification( - monkeypatch, tmp_path -): - deployment = {"run": "same-run", "sharedLease": {"token": "current"}} - lease = Mock() - lease.to_manifest.return_value = deployment["sharedLease"] - lease._read.side_effect = OwnershipError("current ownership must be verified") - monkeypatch.setattr(shared.RunLease, "from_manifest", lambda value: lease) - receipt = tmp_path / "ownership-released.json" - receipt.write_text(json.dumps({"runId": "same-run", "lease": {"token": "old"}})) - with pytest.raises(OwnershipError): - shared.cleanup(deployment, tmp_path) - lease._read.assert_called_once() - lease._read.reset_mock() - receipt.write_text(json.dumps({"runId": "same-run", "lease": {"token": "current"}})) - assert shared.cleanup(deployment, tmp_path) == 0 - lease._read.assert_not_called() +def test_no_recorded_cases_skip_aws_cleanup(monkeypatch, tmp_path): + client = Mock(side_effect=AssertionError("No case work was recorded")) + monkeypatch.setattr(shared, "client", client) + assert shared.cleanup({"run": "run", "functions": {}}, tmp_path) == 0 + assert json.loads((tmp_path / "cleanup.json").read_text())["status"] == "NO_CASES" + client.assert_not_called() @pytest.mark.parametrize("foreign", [True, False]) -def test_case_ownership_is_checked_before_control_release( +def test_case_identity_is_checked_before_control_release( monkeypatch, tmp_path, foreign ): target = "arn:aws:lambda:us-west-2:123:function:owned:$LATEST.PUBLISHED" owned = target + "/durable-execution/name/id" deployment = { "run": "run", - "sharedLease": {}, "bucket": "private", "region": "us-west-2", "functions": {"c1": target}, } - lease = Mock() - lease._read.return_value = ({}, "etag") - lease._owned.return_value = {"profiles": {target: "active"}} - monkeypatch.setattr(shared.RunLease, "from_manifest", lambda value: lease) case = {"case_id": "case", "cohort_id": "cohort", "profile_id": "c1"} plan = { "runId": "run", @@ -59,7 +43,7 @@ def test_case_ownership_is_checked_before_control_release( output.mkdir() (output / "manifest.json").write_text(json.dumps(plan)) identity = hashlib.sha256(b"case:cohort").hexdigest()[:20] - invocations = output / "source" / "cases" / identity / "invocations" + invocations = output / "source/cases" / identity / "invocations" invocations.mkdir(parents=True) (invocations / "intent.json").write_text( json.dumps( @@ -74,12 +58,14 @@ def test_case_ownership_is_checked_before_control_release( storage.get_paginator.return_value.paginate.return_value = [ {"Contents": [{"Key": prefix + "release-all"}]} ] - storage.get_object.return_value = {"Body": io.BytesIO(b"hold")} + storage.get_object.side_effect = lambda **kwargs: {"Body": io.BytesIO(b"hold")} service.get_durable_execution.return_value = {"Status": "RUNNING"} monkeypatch.setattr( shared, "client", lambda name, region: storage if name == "s3" else service ) - assert shared.cleanup(deployment, tmp_path) == 1 + assert shared.cleanup(deployment, tmp_path) == int(foreign) + report = json.loads((tmp_path / "cleanup.json").read_text()) + assert report["status"] == ("FAILED" if foreign else "CLEANUP_REQUESTED") if foreign: storage.put_object.assert_not_called() service.stop_durable_execution.assert_not_called() @@ -90,42 +76,92 @@ def test_case_ownership_is_checked_before_control_release( service.stop_durable_execution.assert_called_once_with( DurableExecutionArn=owned ) - lease.finish_profile.assert_called_once_with(target, False) - lease.release.assert_not_called() + assert all( + call.kwargs["Key"].startswith(prefix) + for call in storage.get_object.call_args_list + ) -def test_failed_lease_acquisition_skips_cleanup_without_touching_another_run( - monkeypatch, tmp_path +@pytest.mark.parametrize("wrong", ["run", "target"]) +def test_cleanup_rejects_foreign_manifest_before_aws_calls( + monkeypatch, tmp_path, wrong ): - from_manifest = Mock(side_effect=AssertionError("No lease was acquired")) - client = Mock(side_effect=AssertionError("Unowned resources must not be touched")) - monkeypatch.setattr(shared.RunLease, "from_manifest", from_manifest) - monkeypatch.setattr(shared, "client", client) - assert shared.cleanup({"run": "new-run", "functions": {}}, tmp_path) == 0 - assert ( - json.loads((tmp_path / "cleanup.json").read_text())["status"] - == "SKIPPED_NO_LEASE" + deployment = {"run": "run", "functions": {"c1": "arn:owned"}} + output = tmp_path / "standard" + output.mkdir() + (output / "manifest.json").write_text( + json.dumps( + { + "runId": "other" if wrong == "run" else "run", + "profiles": [ + { + "profile_id": "c1", + "resource_id": "arn:foreign" + if wrong == "target" + else "arn:owned", + } + ], + "scenarios": [{"profile_id": "c1", "case_id": "case"}], + } + ) ) - assert not (tmp_path / "ownership-released.json").exists() - from_manifest.assert_not_called() + client = Mock(side_effect=AssertionError("Foreign targets must not be touched")) + monkeypatch.setattr(shared, "client", client) + with pytest.raises(CollectionError): + shared.cleanup(deployment, tmp_path) client.assert_not_called() -@pytest.mark.parametrize("evidence", ["owned", "functions", "case-plan"]) -def test_missing_lease_with_prior_activity_is_not_treated_as_unacquired( - monkeypatch, tmp_path, evidence +def test_one_identified_execution_does_not_hide_another_unreconciled_invoke( + monkeypatch, tmp_path ): - deployment = {"run": "run", "functions": {}} - if evidence == "case-plan": - output = tmp_path / "standard" - output.mkdir() - (output / "manifest.json").write_text("{}") - else: - deployment[evidence] = True if evidence == "owned" else {"c1": "arn:owned"} - client = Mock( - side_effect=AssertionError("Cannot mutate resources without ownership") + from pathlib import Path + + target = "arn:aws:lambda:us-west-2:123:function:owned:$LATEST.PUBLISHED" + execution = target + "/durable-execution/name/id" + deployment = { + "run": "run", + "bucket": "private", + "region": "us-west-2", + "functions": {"c1": target}, + } + case = {"case_id": "case", "profile_id": "c1"} + output = tmp_path / "standard" + output.mkdir() + (output / "manifest.json").write_text( + json.dumps( + { + "runId": "run", + "profiles": [{"profile_id": "c1", "resource_id": target}], + "scenarios": [case], + } + ) ) - monkeypatch.setattr(shared, "client", client) - with pytest.raises(OwnershipError, match="cannot verify ownership"): - shared.cleanup(deployment, tmp_path) - client.assert_not_called() + owner = hashlib.sha256(b"case:cohort-1").hexdigest()[:20] + intents = output / "source/cases" / owner / "invocations" + intents.mkdir(parents=True) + known, missing = intents / "known.json", intents / "missing.json" + payload = {"run": "run", "caseId": "case"} + known.write_text( + json.dumps({"payload": {**payload, "marker": "known"}, "arn": execution}) + ) + missing.write_text( + json.dumps({"payload": {**payload, "marker": "missing"}, "arn": None}) + ) + original_glob = Path.glob + monkeypatch.setattr( + Path, + "glob", + lambda path, pattern: ( + iter([known, missing]) if path == intents else original_glob(path, pattern) + ), + ) + storage, service = Mock(), Mock() + storage.get_paginator.return_value.paginate.return_value = [] + service.get_durable_execution.return_value = {"Status": "SUCCEEDED"} + monkeypatch.setattr( + shared, "client", lambda name, region: storage if name == "s3" else service + ) + assert shared.cleanup(deployment, tmp_path) == 1 + errors = json.loads((tmp_path / "cleanup.json").read_text())["errors"] + assert errors == ["Unreconciled Invoke: case/missing"] diff --git a/lmi_tests/tests/unit/test_shared_lock.py b/lmi_tests/tests/unit/test_shared_lock.py deleted file mode 100644 index c353117e1..000000000 --- a/lmi_tests/tests/unit/test_shared_lock.py +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Ownership must survive run failure and conditional-write races.""" - -import io -import json - -from botocore.exceptions import ClientError -import pytest - -from lmi_tests.shared_lock import OwnershipError, RunLease - - -class Store: - def __init__(self): - self.record = None - self.version = 0 - self.race = False - - def get_object(self, **kwargs): - if self.record is None: - raise ClientError({"Error": {"Code": "NoSuchKey"}}, "GetObject") - return { - "Body": io.BytesIO(json.dumps(self.record).encode()), - "ETag": str(self.version), - } - - def put_object(self, **kwargs): - if ( - self.race - or (kwargs.get("IfNoneMatch") == "*" and self.record is not None) - or ("IfMatch" in kwargs and kwargs["IfMatch"] != str(self.version)) - ): - raise ClientError({"Error": {"Code": "PreconditionFailed"}}, "PutObject") - self.record = json.loads(kwargs["Body"]) - self.version += 1 - - -def test_run_holds_all_targets_until_both_profiles_retire(): - store = Store() - lease = RunLease.acquire("private", "stack", "run", ("c1", "c2"), s3=store) - lease.assert_owned("c1") - with pytest.raises(OwnershipError): - RunLease.acquire("private", "stack", "other", ("c1", "c2"), s3=store) - lease.finish_profile("c1", quiescent=True) - with pytest.raises(OwnershipError): - lease.release() - lease.finish_profile("c2", quiescent=True) - lease.release() - replacement = RunLease.acquire("private", "stack", "next", ("c1", "c2"), s3=store) - replacement.assert_owned("c2") - with pytest.raises(OwnershipError): - lease.assert_owned("c1") - - -def test_quarantine_cannot_be_erased_by_later_success_or_clock_expiry(): - store = Store() - lease = RunLease.acquire("private", "stack", "run", ("c1", "c2"), s3=store) - lease.finish_profile("c1", quiescent=False) - lease.finish_profile("c1", quiescent=True) - lease.assert_owned("c2") - with pytest.raises(OwnershipError): - lease.assert_owned("c1") - with pytest.raises(OwnershipError): - lease.release() - with pytest.raises(OwnershipError): - RunLease.acquire("private", "stack", "new", ("c1", "c2"), s3=store) - - -def test_conditional_write_race_does_not_steal_ownership(): - store = Store() - store.race = True - with pytest.raises(OwnershipError): - RunLease.acquire("private", "stack", "run", ("c1",), s3=store) - assert store.record is None - - -def test_unknown_resources_and_malformed_records_fail_closed(): - store = Store() - lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) - with pytest.raises(OwnershipError): - lease.finish_profile("foreign", quiescent=True) - with pytest.raises(OwnershipError): - lease.assert_owned("foreign") - store.record = {"schemaVersion": 999} - with pytest.raises(OwnershipError): - lease.assert_owned("c1") - - -def test_reusing_a_settled_profile_retains_the_lease_for_the_new_group(): - store = Store() - lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) - lease.finish_profile("c1", quiescent=True) - lease.assert_owned("c1") - with pytest.raises(OwnershipError): - lease.release() - lease.finish_profile("c1", quiescent=True) - lease.release() - - -@pytest.mark.parametrize("state", ["unknown", None, 1, {"unverified": True}]) -def test_unknown_retirement_state_cannot_be_claimed_or_cleared(state): - store = Store() - lease = RunLease.acquire("private", "stack", "run", ("c1",), s3=store) - store.record["profiles"]["c1"] = state - with pytest.raises(OwnershipError): - lease.assert_owned("c1") - with pytest.raises(OwnershipError): - lease.finish_profile("c1", True) - assert store.record["profiles"]["c1"] == state - - -def test_blocked_acquisition_identifies_owner_without_exposing_fence_or_mutating_lock(): - store = Store() - owner = RunLease.acquire("private", "stack", "original-run", ("c1", "c2"), s3=store) - owner.finish_profile("c1", False) - before = json.dumps(store.record, sort_keys=True) - with pytest.raises(OwnershipError) as error: - RunLease.acquire("private", "stack", "new-run", ("c1", "c2"), s3=store) - assert "original-run" in str(error.value) - assert "quarantined" in str(error.value) - assert owner.token not in str(error.value) - assert json.dumps(store.record, sort_keys=True) == before diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py index 958de2a55..8f6c0ab47 100644 --- a/lmi_tests/tests/unit/test_shared_manifest.py +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -35,13 +35,6 @@ def deployment(): "wheels": { "aws_durable_execution_sdk_python-3.0.0.whl": "core-sha", }, - "sharedLease": { - "bucket": "private", - "key": "ownership/stack/shared-run.json", - "runId": "run-1", - "token": "fence", - "resources": ["c1", "c2"], - }, } @@ -51,6 +44,8 @@ def test_generated_manifest_covers_the_seven_shared_scenarios(tmp_path): source = deployment() plan = make_manifest(source, "standard", tmp_path) assert len(plan["scenarios"]) == 7 + assert "lease" not in plan["sdk"] + assert "sharedLease" not in plan["sdk"]["deployment"] assert {c["requirement_id"] for c in plan["scenarios"]} == { f"lmi-lifecycle-{i}" for i in range(1, 8) } @@ -73,11 +68,7 @@ def test_generated_manifest_covers_the_seven_shared_scenarios(tmp_path): assert source["commit"] == "sdk-commit" -def test_no_unowned_or_unidentified_artifact_can_be_planned(tmp_path): - source = deployment() - source.pop("sharedLease") - with pytest.raises(ValueError, match="ownership"): - make_manifest(source, "standard", tmp_path) +def test_no_unidentified_artifact_can_be_planned(tmp_path): source = deployment() source["wheels"] = {} with pytest.raises(ValueError, match="wheel"): @@ -107,3 +98,34 @@ def test_only_the_exact_clean_shared_revision_is_accepted(tmp_path): side_effect=[CONFORMANCE_REVISION + "\n", ""], ): verify_checkout(tmp_path) + + +def test_shared_run_writes_reports_without_an_external_lock(monkeypatch, tmp_path): + from unittest.mock import Mock + from lmi_tests import shared + from aws_durable_execution_conformance_tests.lmi import app + + async def results(plan, adapter_name, *, audit_dir): + assert adapter_name == "lmi_tests.shared_adapter:create_adapter" + assert "lease" not in plan["sdk"] + return [ + { + "case_id": case["case_id"], + "requirement_id": case["requirement_id"], + "profile_id": case["profile_id"], + "status": "PASSED", + "errors": [], + } + for case in plan["scenarios"] + ] + + client = Mock( + side_effect=AssertionError("Run orchestration must not acquire an AWS lock") + ) + monkeypatch.setattr(shared, "client", client) + monkeypatch.setattr(app, "live_results", results) + output = tmp_path / "run" + assert shared.run(deployment(), output) == 0 + report = json.loads((output / "standard/report.json").read_text()) + assert report["summary"]["passed"] == 7 + client.assert_not_called() diff --git a/lmi_tests/tests/unit/test_shared_partial_evidence.py b/lmi_tests/tests/unit/test_shared_partial_evidence.py index ad00416cd..3e76a4a04 100644 --- a/lmi_tests/tests/unit/test_shared_partial_evidence.py +++ b/lmi_tests/tests/unit/test_shared_partial_evidence.py @@ -65,7 +65,6 @@ def build(ctx, observed, *, phase): result = SharedAdapter( {"sdk": {"deployment": {}, "artifacts_dir": str(tmp_path)}}, - ownership=Mock(), evidence_builder=build, ) result.cases["case"] = _Case(driver) From ac35090637dd85909d6e21a1ac894f201f85111a Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:57:35 -0700 Subject: [PATCH 24/32] fix: preserve the original recovery peer --- lmi_tests/README.md | 11 +- lmi_tests/cloud.py | 4 + lmi_tests/fixture.py | 19 +- lmi_tests/shared_scenarios.py | 98 ++++++++- .../tests/e2e/test_peer_recovery_local.py | 44 ++++ .../tests/unit/test_lifecycle_mapping.py | 11 +- lmi_tests/tests/unit/test_peer_recovery.py | 206 ++++++++++++++++++ 7 files changed, 381 insertions(+), 12 deletions(-) create mode 100644 lmi_tests/tests/e2e/test_peer_recovery_local.py create mode 100644 lmi_tests/tests/unit/test_peer_recovery.py diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 4c0561d0b..83a4a0f17 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -93,9 +93,14 @@ quiescent. Current SDK gaps are failures, not expected passes or exclusions. The deadline fixture holds real I/O across the runtime deadline and cooperatively exits after it. A runtime clock alone never supplies a `platform_timeout` event: that event requires the original request's service `InvocationCompleted` history. -The recovery driver keeps the original healthy peer held while an independent -recovery request establishes capacity in the original environment. Releasing a -victim fault cannot establish the preceding cleanup or recovery assertion. +The recovery victim first waits in a checkpointed preparation step. The driver +starts the peer halfway through the victim's native timeout, verifies the original +cohort and the peer's remaining recovery window, then releases preparation to +admit the fault. Recovery overlap excludes expired original requests and their +retries. The original peer remains held until recovery completes. A still-running +durable execution is collected as such; terminal-state waiting and stops belong +to retirement after assertions. Releasing a victim fault cannot establish the +preceding cleanup or recovery assertion. The warm fixture creates invocation-owned late work inside a step. A later invocation must find it in the same process; it observes the old callback using diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index e1b2f735d..6efabcc8c 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -194,6 +194,7 @@ def start( gate=None, target_environment=None, *, + preparation_gate=None, role=None, resume_roles=(), dispatcher_payload=None, @@ -231,6 +232,9 @@ def start( if gate: payload["gate"] = gate self.hold(gate) + if preparation_gate: + payload["preparation_gate"] = preparation_gate + self.hold(preparation_gate) runner = DurableFunctionCloudTestRunner( self.manifest["functions"][self.configuration], region=self.manifest["region"], diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index ff083634a..fbda43858 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -207,9 +207,11 @@ def released(self, name): observe(self, "control_error", gateId=name, error=str(error)) raise - def gate(self, name, *, effects=False, attempt=1, ready=None): + def gate(self, name, *, effects=False, attempt=1, ready=None, preparation=False): # This return value is checkpointed by existing controlled-io steps. - self.gate_release(name, effects=effects, attempt=attempt, ready=ready) + self.gate_release( + name, effects=effects, attempt=attempt, ready=ready, preparation=preparation + ) def gate_release( self, @@ -220,6 +222,7 @@ def gate_release( ready=None, on_hold=None, on_release=None, + preparation=False, ): """Observe normal versus emergency release without changing step results.""" end = time.monotonic() + 150 @@ -266,6 +269,7 @@ def gate_release( return released if ( self.payload["scenario"] == "deadline" + and not preparation and time.time() >= self.deadline + 0.2 ): observe(self, "cooperative_io_cancelled", source="runtime-deadline") @@ -278,8 +282,9 @@ def gate_release( on_hold(held_at_ms) self.emit("BLOCKED", gate=name) observe(self, "blocked", gateId=name, atMs=held_at_ms) - observe(self, "barrier_arrived", gateId=name, atMs=held_at_ms) - if self.payload["scenario"] == "deadline": + if preparation or not self.payload.get("preparation_gate"): + observe(self, "barrier_arrived", gateId=name, atMs=held_at_ms) + if self.payload["scenario"] == "deadline" and not preparation: observe(self, "work_in_flight", gateId=name, atMs=held_at_ms) if self.payload.get("role") == "recovery": observe(self, "worker_recovered", atMs=held_at_ms) @@ -442,6 +447,12 @@ def run_workflow(event, context, trace): if scenario == "failure": raise ValueError("expected:" + marker) if scenario in {"barrier", "deadline"}: + if preparation_gate := event.get("preparation_gate"): + context.step( + lambda _: trace.gate(preparation_gate, preparation=True), + name="prepare-recovery", + config=NO_RETRY, + ) context.step( lambda step: trace.gate( event["gate"], effects=scenario == "deadline", attempt=step.attempt diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py index 2e33ef809..a9b97abb7 100644 --- a/lmi_tests/shared_scenarios.py +++ b/lmi_tests/shared_scenarios.py @@ -56,6 +56,7 @@ def __init__( reserve_capacity, ) self.boundary: dict[str, Any] | None = None + self.peer_boundary: dict[str, Any] | None = None @property def scenario(self) -> str: @@ -78,6 +79,7 @@ def launch( fixture: str, *, gate: str | None = None, + preparation_gate: str | None = None, resume_roles: tuple[str, ...] = (), target_environment: str | None = None, ) -> dict[str, Any]: @@ -86,6 +88,7 @@ def launch( item = self.cloud.start( fixture, gate=gate, + preparation_gate=preparation_gate, target_environment=target_environment, role=role, resume_roles=resume_roles, @@ -108,6 +111,9 @@ def start(self) -> None: gate=self.gate("fault") if self.scenario in {"deadline-during-step", "peer-recovery"} else None, + preparation_gate=self.gate("prepare") + if self.scenario == "peer-recovery" + else None, resume_roles=("resume",) if self.scenario == "suspension-retry" else (), ) @@ -117,9 +123,35 @@ def wait_overlap( markers = {self.items[role]["marker"] for role in roles} def established(): - events = [ + observed = [ event for event in self.cloud.refresh() if event["marker"] in markers ] + originals = {} + for role in roles: + entries = [ + event + for event in observed + if event["phase"] == "WRAPPER_ENTER" + and event["marker"] == self.items[role]["marker"] + ] + if not entries: + return None + entry = min( + entries, key=lambda event: (event["time"], event["sequence"]) + ) + originals[role] = entry["request"] + if time.time() >= entry["deadline"]: + raise PlacementError( + f"Original {role} invocation expired before overlap was established" + ) + events = [ + event for event in observed if event["request"] in originals.values() + ] + if any( + event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + for event in events + ): + raise PlacementError("An original cohort invocation already ended") if environment is not None and any( event["environment"] != environment for event in select(events, "BLOCKED") @@ -147,11 +179,66 @@ def place(self) -> None: elif self.scenario in {"deadline-during-step", "peer-recovery"}: self.boundary = self.cloud.phase(victim, "BLOCKED")[0] if self.scenario == "peer-recovery": - self.launch("peer", "barrier", gate=self.gate("peer")) - self.wait_overlap(("victim", "peer"), self.boundary["environment"]) + self._prepare_recovery_peer() else: self.cloud.phase(victim, "WRAPPER_ENTER") + def _prepare_recovery_peer(self) -> None: + if self.boundary is None: + raise CollectionError( + "Recovery preparation has no original victim boundary" + ) + deadline = self.boundary["deadline"] + lead = self.cloud.manifest["invocationTimeout"] / 2 + self.cloud.poll( + lambda: time.time() >= deadline - lead, + seconds=self.cloud.manifest["invocationTimeout"], + message="Original victim did not reach the peer-admission window", + ) + if time.time() >= deadline: + raise PlacementError("Original victim expired before peer admission") + peer = self.launch( + "peer", + "barrier", + gate=self.gate("peer"), + target_environment=self.boundary["environment"], + ) + self.peer_boundary = self.cloud.phase(peer, "BLOCKED")[0] + required_window = self.seconds("recoveryMs", 15) + self.seconds( + "observationGraceMs", 1 + ) + if self.peer_boundary["deadline"] <= deadline + required_window: + raise PlacementError( + "Original peer has insufficient time remaining for recovery" + ) + self.wait_overlap(("victim", "peer"), self.boundary["environment"]) + # Establish the original healthy cohort before admitting the actual fault. + self.cloud.release(self.gate("prepare")) + request = self.boundary["request"] + self.cloud.poll( + lambda: [ + event + for event in canonical(self.cloud.for_item(self.items["victim"])) + if event.get("requestId") == request + and event.get("kind") == "work_in_flight" + and event.get("atMs", deadline * 1000) < deadline * 1000 + ], + seconds=max(0.001, deadline - time.time()), + message="Original victim never entered the fault before its deadline", + ) + + def _peer_still_active(self) -> None: + if self.peer_boundary is None: + raise CollectionError("Recovery lost the original peer boundary") + if time.time() >= self.peer_boundary["deadline"]: + raise PlacementError("Original healthy peer expired during recovery") + if any( + event["request"] == self.peer_boundary["request"] + and event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} + for event in self.cloud.for_item(self.items["peer"]) + ): + raise PlacementError("Original healthy peer ended during recovery") + def _callback(self, item: dict[str, Any]) -> None: self.cloud.poll( lambda: [ @@ -195,6 +282,7 @@ def _deadline(self) -> None: seconds=self.cloud.manifest["invocationTimeout"] + 5, ) if self.scenario == "peer-recovery": + self._peer_still_active() recovery = self.launch( "recovery", "barrier", @@ -204,6 +292,7 @@ def _deadline(self) -> None: self.wait_overlap(("peer", "recovery"), self.boundary["environment"]) self.cloud.release(self.gate("recovery")) self.cloud.finish(recovery) + self._peer_still_active() self.cloud.release(self.gate("peer")) self.cloud.finish(self.items["peer"]) self.cloud.platform_timeout(victim, self.boundary["request"]) @@ -216,7 +305,8 @@ def _deadline(self) -> None: ), seconds=self.cloud.manifest["invocationTimeout"] + 30, ) - self.cloud.wait_terminal(victim) + # RUNNING is a valid pre-cleanup execution state for this measurement. + # Collection records it; retirement stops/reconciles the execution later. def execute(self) -> None: victim = self.items["victim"] diff --git a/lmi_tests/tests/e2e/test_peer_recovery_local.py b/lmi_tests/tests/e2e/test_peer_recovery_local.py new file mode 100644 index 000000000..438f9f3f7 --- /dev/null +++ b/lmi_tests/tests/e2e/test_peer_recovery_local.py @@ -0,0 +1,44 @@ +"""The real SDK checkpoints preparation before the controlled recovery fault.""" + +import json + +from aws_durable_execution_sdk_python_testing import DurableFunctionTestRunner +from lmi_tests import fixture +from lmi_tests.tests.unit.test_shared_observations import install_ledger, shared_payload + + +def test_preparation_gate_precedes_fault_and_keeps_one_original_barrier(monkeypatch): + ledger = install_ledger(monkeypatch) + payload = shared_payload("deadline") + payload.update( + sharedScenario="peer-recovery", preparation_gate="prepare", gate="fault" + ) + prefix = payload["controlPrefix"] + for gate in ("prepare", "fault", "release-all"): + ledger.objects[prefix + gate] = b"hold" + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + arn = runner.run_async(input=json.dumps(payload), execution_timeout=15) + prepared = ledger.wait_for("blocked")["observation"] + assert prepared["gateId"] == "prepare" + assert not [ + e + for e in ledger.observations() + if e["observation"]["kind"] == "work_in_flight" + ] + ledger.objects[prefix + "prepare"] = b"release" + fault = ledger.wait_for("work_in_flight")["observation"] + assert fault["gateId"] == "fault" + ledger.objects[prefix + "fault"] = b"release" + result = runner.wait_for_result(arn, timeout=15) + assert result.result == '"marker"' + observations = [event["observation"] for event in ledger.observations()] + barriers = [event for event in observations if event["kind"] == "barrier_arrived"] + assert len(barriers) == 1 and barriers[0]["gateId"] == "prepare" + checkpoint = next( + event + for event in observations + if event["kind"] == "checkpoint_ack" + and result.get_step("prepare-recovery").operation_id in event["operationIds"] + ) + assert prepared["atMs"] < checkpoint["atMs"] <= fault["atMs"] + assert not [event for event in observations if event["kind"] == "observation_error"] diff --git a/lmi_tests/tests/unit/test_lifecycle_mapping.py b/lmi_tests/tests/unit/test_lifecycle_mapping.py index 4a4130d62..c33b9fc9c 100644 --- a/lmi_tests/tests/unit/test_lifecycle_mapping.py +++ b/lmi_tests/tests/unit/test_lifecycle_mapping.py @@ -14,8 +14,17 @@ @pytest.mark.parametrize( "cell", required_cases("python", "lifecycle"), ids=lambda c: c.scenario ) -def test_each_shared_case_uses_its_required_original_roles(cell): +def test_each_shared_case_uses_its_required_original_roles(cell, monkeypatch): cloud = ScriptCloud(cell.minimum_native_concurrency) + if cell.scenario == "peer-recovery": + from types import SimpleNamespace + from lmi_tests import shared_scenarios + from lmi_tests.tests.unit.test_peer_recovery import RecoveryCloud + + cloud = RecoveryCloud() + monkeypatch.setattr( + shared_scenarios, "time", SimpleNamespace(time=lambda: cloud.now) + ) driver = ScenarioDriver( context( scenario=cell.scenario, diff --git a/lmi_tests/tests/unit/test_peer_recovery.py b/lmi_tests/tests/unit/test_peer_recovery.py new file mode 100644 index 000000000..bb36a68e3 --- /dev/null +++ b/lmi_tests/tests/unit/test_peer_recovery.py @@ -0,0 +1,206 @@ +"""Recovery scheduling uses original native deadlines, not late I/O or retries.""" + +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from aws_durable_execution_conformance_tests.lmi.coordinator import ( + Profile, + Scenario, + ScenarioContext, +) +from lmi_tests import shared_scenarios +from lmi_tests.evidence import PlacementError +from lmi_tests.shared_scenarios import ScenarioDriver +from lmi_tests.tests.unit.test_shared_adapter import ScriptCloud + + +class RecoveryCloud(ScriptCloud): + def __init__(self): + super().__init__(2) + self.now = 0.0 + self.events = [] + self.retiring = False + self.configuration = "c2" + self.manifest["driverTimeout"] = 120 + + def emit(self, item, phase, **extra): + request = item["native_requests"][0] + value = { + "phase": phase, + "time": self.now, + "deadline": item["deadline"], + "marker": item["marker"], + "request": request, + "execution": item["arn"], + "environment": "environment", + "process": item["process"], + "pid": item["pid"], + "gate": item.get("held_gate"), + "sequence": 1 + sum(e["request"] == request for e in self.events), + **extra, + } + self.events.append(value) + return value + + def start(self, fixture, **options): + item = super().start(fixture, **options) + role = options["role"] + item.update( + native_requests=[f"request-{role}"], + deadline=self.now + 60, + process="peer-worker" if role == "peer" else "victim-worker", + pid=2 if role == "peer" else 1, + payload={"scenario": fixture, **options}, + held_gate=options.get("preparation_gate") or options.get("gate"), + ) + self.emit(item, "WRAPPER_ENTER") + self.now += 3 if role == "recovery" else 0.25 + self.emit(item, "BLOCKED") + return item + + def phase(self, item, phase, **options): + return [ + e + for e in self.events + if e["marker"] == item["marker"] and e["phase"] == phase + ] + + def for_item(self, item): + return [e for e in self.refresh() if e["marker"] == item["marker"]] + + def refresh(self, **options): + # Deliberately allow I/O liveness after a native deadline: this must not + # be accepted as proof that the original invocation is still healthy. + for item in self.invocations: + if not item.get("finished"): + self.emit(item, "ALIVE") + return list(self.events) + + def poll(self, predicate, seconds=30, **options): + deadline = self.now + seconds + while True: + value = predicate() + if value: + return value + if self.now >= deadline: + raise AssertionError(options.get("message", "poll timed out")) + self.now = min(deadline, self.now + 0.5) + + def release(self, gate): + super().release(gate) + for item in self.invocations: + if item["payload"].get("preparation_gate") == gate: + self.emit(item, "IO_EXIT") + item["held_gate"] = item["payload"]["gate"] + self.emit(item, "BLOCKED") + self.emit( + item, + "CONFORMANCE", + observation={"kind": "work_in_flight", "atMs": self.now * 1000}, + ) + + def finish(self, item, status="SUCCEEDED"): + assert self.now < item["deadline"], "The selected original invocation expired" + super().finish(item, status) + self.now += 0.25 + self.emit(item, "WRAPPER_RETURN", status=status) + item["finished"] = True + + def wait_terminal(self, item): + assert self.retiring, "An active victim must reach assertions before cleanup" + return super().wait_terminal(item) + + def wait(self, seconds): + self.now += seconds + + def collect(self, **options): + self.actions.append(("collect",)) + + def settle_case(self): + self.retiring = True + self.actions.append(("settle",)) + self.wait_terminal(self.invocations[0]) + + +def recovery_driver(monkeypatch): + cloud = RecoveryCloud() + monkeypatch.setattr( + shared_scenarios, "time", SimpleNamespace(time=lambda: cloud.now) + ) + profile = Profile("c2", "function", 2, 2, "sdk", "configuration") + scenario = Scenario( + "case", + "lmi-lifecycle-4", + "c2", + "peer-recovery", + roles=("victim", "peer", "recovery"), + ) + return ScenarioDriver(ScenarioContext("run", scenario, profile), cloud), cloud + + +def test_peer_has_recovery_headroom_and_fault_is_armed_after_cohort_placement( + monkeypatch, +): + driver, cloud = recovery_driver(monkeypatch) + driver.start() + assert [i["marker"] for i in cloud.invocations] == ["victim"] + driver.place() + victim, peer = cloud.invocations + assert peer["deadline"] - victim["deadline"] >= 30 + armed = next( + e + for e in cloud.events + if e.get("observation", {}).get("kind") == "work_in_flight" + ) + peer_ready = next( + e for e in cloud.events if e["phase"] == "BLOCKED" and e["marker"] == "peer" + ) + assert peer_ready["time"] <= armed["time"] < victim["deadline"] + driver.execute() + assert ("finish", "peer", "SUCCEEDED") in cloud.actions + assert not any(action[0] == "terminal" for action in cloud.actions) + assert ("release", driver.gate("fault")) not in cloud.actions + driver.collect() + driver.quiescent = Mock(return_value=True) + driver.retire() + assert cloud.actions.index(("collect",)) < cloud.actions.index(("settle",)) + + +def test_late_io_and_a_peer_retry_cannot_replace_the_expired_original(monkeypatch): + driver, cloud = recovery_driver(monkeypatch) + driver.start() + driver.place() + peer = driver.items["peer"] + cloud.now = peer["deadline"] + 1 + driver.launch("recovery", "barrier", gate=driver.gate("recovery")) + original = peer["native_requests"][0] + peer["native_requests"].append("peer-retry") + cloud.events.append( + { + **next(e for e in cloud.events if e["request"] == original), + "request": "peer-retry", + "time": cloud.now, + "deadline": cloud.now + 60, + } + ) + with pytest.raises(PlacementError, match="Original peer invocation expired"): + driver.wait_overlap(("peer", "recovery"), "environment") + + +def test_original_peer_cannot_expire_while_waiting_for_recovery_completion(monkeypatch): + driver, cloud = recovery_driver(monkeypatch) + driver.start() + driver.place() + finish = cloud.finish + + def delayed_finish(item, status="SUCCEEDED"): + finish(item, status) + if item["marker"] == "recovery": + cloud.now = driver.peer_boundary["deadline"] + 1 + + cloud.finish = delayed_finish + with pytest.raises(PlacementError, match="Original healthy peer expired"): + driver.execute() + assert ("release", driver.gate("peer")) not in cloud.actions From e148ac81d87c4a4c4ec9eb74c48bf65d48d6a19d Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 15:58:24 -0700 Subject: [PATCH 25/32] test: temporarily skip lifecycle case 2 --- .github/workflows/lmi-e2e-tests.yml | 2 ++ lmi_tests/shared.py | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 2bca89579..0cc2c747c 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -130,6 +130,8 @@ jobs: allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - name: Run seven shared lifecycle requirements timeout-minutes: 60 + env: + LMI_SKIP_CASE_2: "1" run: hatch run lmi:shared - name: Refresh credentials for evidence and cleanup if: always() && hashFiles('lmi_tests/artifacts/manifest.json') != '' diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 5d5aed898..f97d7fc91 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -96,6 +96,12 @@ def make_manifest( ] scenarios = [] for required in required_cases("python", "lifecycle", coverage): + # Temporary diagnostic run; restore the full catalog after validation. + if ( + os.environ.get("LMI_SKIP_CASE_2") == "1" + and required.requirement_id == "lmi-lifecycle-2" + ): + continue concurrency = required.minimum_native_concurrency case_id = f"{required.requirement_id}-{required.scenario}" scenarios.append( From c0eb43a33bd6944324008ac36ba216061d09488c Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:02:18 -0700 Subject: [PATCH 26/32] fix: record recovery cohort admission --- lmi_tests/fixture.py | 13 ++++++++++++- lmi_tests/tests/e2e/test_peer_recovery_local.py | 12 ++++++++++++ lmi_tests/tests/unit/test_peer_recovery.py | 10 ++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index fbda43858..652d05df1 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -258,6 +258,17 @@ def gate_release( ) if on_release is not None: on_release(released) + if preparation and mode == "normal": + # The driver releases preparation only after the peer + # joined. This observes the cohort barrier within the + # original invocations' overlap, not at early warm-up. + observe( + self, + "barrier_arrived", + gateId=name, + atMs=released.observed_at_ms, + source="cohort-admission", + ) observe( self, "fault_released", @@ -282,7 +293,7 @@ def gate_release( on_hold(held_at_ms) self.emit("BLOCKED", gate=name) observe(self, "blocked", gateId=name, atMs=held_at_ms) - if preparation or not self.payload.get("preparation_gate"): + if not preparation and not self.payload.get("preparation_gate"): observe(self, "barrier_arrived", gateId=name, atMs=held_at_ms) if self.payload["scenario"] == "deadline" and not preparation: observe(self, "work_in_flight", gateId=name, atMs=held_at_ms) diff --git a/lmi_tests/tests/e2e/test_peer_recovery_local.py b/lmi_tests/tests/e2e/test_peer_recovery_local.py index 438f9f3f7..3aaf4fbb1 100644 --- a/lmi_tests/tests/e2e/test_peer_recovery_local.py +++ b/lmi_tests/tests/e2e/test_peer_recovery_local.py @@ -20,6 +20,11 @@ def test_preparation_gate_precedes_fault_and_keeps_one_original_barrier(monkeypa arn = runner.run_async(input=json.dumps(payload), execution_timeout=15) prepared = ledger.wait_for("blocked")["observation"] assert prepared["gateId"] == "prepare" + assert not [ + e + for e in ledger.observations() + if e["observation"]["kind"] == "barrier_arrived" + ] assert not [ e for e in ledger.observations() @@ -34,6 +39,13 @@ def test_preparation_gate_precedes_fault_and_keeps_one_original_barrier(monkeypa observations = [event["observation"] for event in ledger.observations()] barriers = [event for event in observations if event["kind"] == "barrier_arrived"] assert len(barriers) == 1 and barriers[0]["gateId"] == "prepare" + released = next( + e + for e in observations + if e["kind"] == "fault_released" and e["gateId"] == "prepare" + ) + assert barriers[0]["atMs"] == released["atMs"] + assert prepared["atMs"] < barriers[0]["atMs"] <= fault["atMs"] checkpoint = next( event for event in observations diff --git a/lmi_tests/tests/unit/test_peer_recovery.py b/lmi_tests/tests/unit/test_peer_recovery.py index bb36a68e3..e62274ca5 100644 --- a/lmi_tests/tests/unit/test_peer_recovery.py +++ b/lmi_tests/tests/unit/test_peer_recovery.py @@ -204,3 +204,13 @@ def delayed_finish(item, status="SUCCEEDED"): with pytest.raises(PlacementError, match="Original healthy peer expired"): driver.execute() assert ("release", driver.gate("peer")) not in cloud.actions + + +def test_expired_victim_does_not_start_a_peer_or_arm_the_fault(monkeypatch): + driver, cloud = recovery_driver(monkeypatch) + driver.start() + cloud.now = cloud.invocations[0]["deadline"] + 1 + with pytest.raises(PlacementError, match="Original victim expired"): + driver.place() + assert [item["marker"] for item in cloud.invocations] == ["victim"] + assert ("release", driver.gate("prepare")) not in cloud.actions From 45d6a4360d958c1ae74745e163270d2bf92cd099 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:13:48 -0700 Subject: [PATCH 27/32] fix: scope recovery faults to the original request --- lmi_tests/README.md | 6 ++- lmi_tests/fixture.py | 41 ++++++++++++----- .../tests/e2e/test_peer_recovery_local.py | 45 +++++++++++++++++++ 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 83a4a0f17..685d44740 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -96,8 +96,10 @@ that event requires the original request's service `InvocationCompleted` history The recovery victim first waits in a checkpointed preparation step. The driver starts the peer halfway through the victim's native timeout, verifies the original cohort and the peer's remaining recovery window, then releases preparation to -admit the fault. Recovery overlap excludes expired original requests and their -retries. The original peer remains held until recovery completes. A still-running +admit the fault. Preparation checkpoints the original native request ID, so +service retries keep the same durable sequence without reinjecting the victim's +fault or admission probes. Recovery overlap excludes expired original requests +and their retries. The original peer remains held until recovery completes. A still-running durable execution is collected as such; terminal-state waiting and stops belong to retirement after assertions. Releasing a victim fault cannot establish the preceding cleanup or recovery assertion. diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 652d05df1..08334cadb 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -458,19 +458,35 @@ def run_workflow(event, context, trace): if scenario == "failure": raise ValueError("expected:" + marker) if scenario in {"barrier", "deadline"}: + original_request = None if preparation_gate := event.get("preparation_gate"): - context.step( - lambda _: trace.gate(preparation_gate, preparation=True), - name="prepare-recovery", - config=NO_RETRY, + + def prepare(_): + trace.gate(preparation_gate, preparation=True) + return trace.identity["request"] + + original_request = context.step( + prepare, name="prepare-recovery", config=NO_RETRY ) - context.step( - lambda step: trace.gate( + trace.recovery_original_request = original_request + + def controlled_io(step): + # Fault injection belongs to the selected native invocation. + # This decision is inside the step; replay still consumes the + # same preparation checkpoint and durable operation sequence. + if ( + original_request is not None + and trace.identity["request"] != original_request + ): + observe( + trace, "fault_not_reapplied", originalRequestId=original_request + ) + return None + return trace.gate( event["gate"], effects=scenario == "deadline", attempt=step.attempt - ), - name="controlled-io", - config=NO_RETRY, - ) + ) + + context.step(controlled_io, name="controlled-io", config=NO_RETRY) context.step(lambda s: trace.body("after-io", marker, s), name="after-io") elif scenario == "warm-owner": context.step( @@ -753,6 +769,11 @@ def invoke(plugins=None): "peer-recovery", } and payload.get("role") == "victim" + and ( + payload.get("sharedScenario") != "peer-recovery" + or trace.identity["request"] + == getattr(trace, "recovery_original_request", None) + ) ): start_admission_probes(trace) return result diff --git a/lmi_tests/tests/e2e/test_peer_recovery_local.py b/lmi_tests/tests/e2e/test_peer_recovery_local.py index 3aaf4fbb1..9685ee363 100644 --- a/lmi_tests/tests/e2e/test_peer_recovery_local.py +++ b/lmi_tests/tests/e2e/test_peer_recovery_local.py @@ -54,3 +54,48 @@ def test_preparation_gate_precedes_fault_and_keeps_one_original_barrier(monkeypa ) assert prepared["atMs"] < checkpoint["atMs"] <= fault["atMs"] assert not [event for event in observations if event["kind"] == "observation_error"] + + +def test_service_retry_replays_preparation_without_reinjecting_the_original_fault( + monkeypatch, +): + from aws_durable_execution_sdk_python.exceptions import RetryableSerDesError + + ledger = install_ledger(monkeypatch) + payload = shared_payload("deadline") + payload.update( + sharedScenario="peer-recovery", preparation_gate="prepare", gate="fault" + ) + prefix = payload["controlPrefix"] + ledger.objects[prefix + "prepare"] = b"release" + ledger.objects[prefix + "fault"] = b"hold" + ledger.objects[prefix + "release-all"] = b"hold" + gate = fixture.Trace.gate + fault_requests = [] + + def interrupted(trace, name, **options): + if name == "fault": + fault_requests.append(trace.identity["request"]) + # Trigger the SDK's actual invocation-retry path after preparation + # checkpointed, leaving controlled-io incomplete for replay. + raise RetryableSerDesError("transient interruption") + return gate(trace, name, **options) + + monkeypatch.setattr(fixture.Trace, "gate", interrupted) + with DurableFunctionTestRunner(handler=fixture.handler) as runner: + result = runner.run(input=json.dumps(payload), execution_timeout=20) + assert result.result == '"marker"' + assert len(fault_requests) == 1 + observations = ledger.observations() + skipped = [ + e for e in observations if e["observation"]["kind"] == "fault_not_reapplied" + ] + assert len(skipped) == 1 + assert skipped[0]["request"] != fault_requests[0] + assert skipped[0]["observation"]["originalRequestId"] == fault_requests[0] + assert not [ + e + for e in observations + if e["request"] == skipped[0]["request"] + and e["observation"]["kind"] == "admission_probe" + ] From a1d43d44ff39ad5427604cde71a8b78b06b99e36 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:13:53 -0700 Subject: [PATCH 28/32] test: prioritize case 7 during diagnostics --- lmi_tests/shared.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index f97d7fc91..b9246a7bf 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -122,6 +122,9 @@ def make_manifest( }, } ) + if os.environ.get("LMI_SKIP_CASE_2") == "1": + # Diagnostic only: observe case 7 before case 6 can quarantine c1. + scenarios.sort(key=lambda case: case["requirement_id"] != "lmi-lifecycle-7") return { "schemaVersion": 1, "runId": actual["run"], From 014bb17f19107298bd624c4841b701e79e7161cd Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:25:27 -0700 Subject: [PATCH 29/32] fix: observe admission after exceptional exit --- lmi_tests/README.md | 4 +- lmi_tests/fixture.py | 72 +++++++++++++------ .../tests/e2e/test_peer_recovery_local.py | 13 ++++ .../tests/unit/test_shared_observations.py | 40 +++++++++++ 4 files changed, 105 insertions(+), 24 deletions(-) diff --git a/lmi_tests/README.md b/lmi_tests/README.md index 685d44740..d8ae1ccbb 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -85,8 +85,8 @@ For normal return/failure, the adapter records the actual durable wrapper bounda and probes the original context after that boundary. The operation probe uses `DurableContext.step`; the checkpoint probe calls the SDK's `context.state.create_checkpoint(is_sync=False)` admission path without issuing a -raw Lambda request. Only an actual `OrphanedChildException` is recorded as -rejection. An accepted checkpoint, another error, or a still-pending operation +raw Lambda request. SDK closure/revocation errors are recorded as rejection only when no user code +or new SDK request ran. Probes run after both wrapper return and wrapper raise. An accepted checkpoint, another error, or a still-pending operation remains visible. Probe tasks are tracked separately; a timeout cannot make them quiescent. Current SDK gaps are failures, not expected passes or exclusions. diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 08334cadb..58e162e08 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -27,7 +27,12 @@ ) from aws_durable_execution_sdk_python.lambda_service import LambdaClient from aws_durable_execution_sdk_python.retries import RetryDecision -from aws_durable_execution_sdk_python.exceptions import OrphanedChildException +from aws_durable_execution_sdk_python.exceptions import ( + BackgroundThreadError, + CheckpointError, + OrphanedChildException, + SuspendExecution, +) from lmi_tests.shared_observations import ( GateRelease, LifecycleObservationFactory, @@ -101,6 +106,7 @@ def __init__(self, payload, context, execution_arn): self.capture_sequence = 0 self.sequence = 0 self.call_sequence = 0 + self.sdk_request_count = 0 self.sdk_completed_at = None self.sdk_observation = None self.sdk_resources = None @@ -185,6 +191,8 @@ def observe(self, kind, **fields): def next_call_id(self, kind): with self.lock: self.call_sequence += 1 + if kind in {"checkpoint", "poll"}: + self.sdk_request_count += 1 return f"{self.identity['request']}:{kind}:{self.call_sequence}" def signal(self, name): @@ -567,6 +575,13 @@ def retry(step): def admission_probes(trace): trace.observe("task_start", taskId="admission-probes", source="fixture") closed = False + body_executed = False + + def body(_): + nonlocal body_executed + body_executed = True + trace.observe("probe_body_executed") + try: for probe, operation in ( ( @@ -576,18 +591,28 @@ def admission_probes(trace): ( "operation", lambda: trace.durable_context.step( - lambda _: trace.observe("probe_body_executed"), - name="operation-admission-probe", - config=NO_RETRY, + body, name="operation-admission-probe", config=NO_RETRY ), ), ): trace.observe("probe_enter", probe=probe) + calls_before = trace.sdk_request_count error_record = None try: operation() - except OrphanedChildException as error: - outcome = "REJECTED" + except ( + OrphanedChildException, + BackgroundThreadError, + CheckpointError, + SuspendExecution, + ) as error: + # Failed/revoked SDK state can reject by re-raising its stored + # error. It is a rejection only before user code or a new RPC. + outcome = ( + "REJECTED" + if not body_executed and trace.sdk_request_count == calls_before + else "ERROR" + ) error_record = {"type": type(error).__name__, "message": str(error)} except BaseException as error: outcome = "ERROR" @@ -606,6 +631,8 @@ def admission_probes(trace): outcome=outcome, error=error_record, atMs=observed_at, + bodyExecuted=body_executed, + newSdkRequests=trace.sdk_request_count - calls_before, ) finally: trace.observe("task_exit", taskId="admission-probes", source="fixture") @@ -760,22 +787,6 @@ def invoke(plugins=None): atMs=trace.sdk_completed_at_ms, source="sdk-wrapper", ) - if ( - payload.get("sharedScenario") - in { - "normal-return", - "normal-failure", - "deadline-during-step", - "peer-recovery", - } - and payload.get("role") == "victim" - and ( - payload.get("sharedScenario") != "peer-recovery" - or trace.identity["request"] - == getattr(trace, "recovery_original_request", None) - ) - ): - start_admission_probes(trace) return result except BaseException as error: if payload.get("coverageProfile") and trace.sdk_observation is None: @@ -797,3 +808,20 @@ def invoke(plugins=None): } ) raise + finally: + if ( + payload.get("sharedScenario") + in { + "normal-return", + "normal-failure", + "deadline-during-step", + "peer-recovery", + } + and payload.get("role") == "victim" + and ( + payload.get("sharedScenario") != "peer-recovery" + or trace.identity["request"] + == getattr(trace, "recovery_original_request", None) + ) + ): + start_admission_probes(trace) diff --git a/lmi_tests/tests/e2e/test_peer_recovery_local.py b/lmi_tests/tests/e2e/test_peer_recovery_local.py index 9685ee363..8fcb3e132 100644 --- a/lmi_tests/tests/e2e/test_peer_recovery_local.py +++ b/lmi_tests/tests/e2e/test_peer_recovery_local.py @@ -87,6 +87,19 @@ def interrupted(trace, name, **options): assert result.result == '"marker"' assert len(fault_requests) == 1 observations = ledger.observations() + raised = next( + e["observation"] + for e in observations + if e["request"] == fault_requests[0] + and e["observation"]["kind"] == "wrapper_raise" + ) + original_probes = [ + e["observation"] + for e in observations + if e["request"] == fault_requests[0] + and e["observation"]["kind"] == "admission_probe" + ] + assert original_probes and all(e["atMs"] >= raised["atMs"] for e in original_probes) skipped = [ e for e in observations if e["observation"]["kind"] == "fault_not_reapplied" ] diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py index 5a78892fb..a6e4ab856 100644 --- a/lmi_tests/tests/unit/test_shared_observations.py +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -423,3 +423,43 @@ def test_pending_and_wrapper_return_share_one_captured_timestamp( wrapper = next(item for item in observations if item["kind"] == "wrapper_return") pending = next(item for item in observations if item["kind"] == "pending") assert pending["atMs"] == wrapper["atMs"] == completed_ns / 1_000_000 + + +@pytest.mark.parametrize("kind", ["checkpoint", "background", "suspend"]) +@pytest.mark.parametrize("effect", ["none", "body", "rpc"]) +def test_stored_sdk_errors_are_rejections_only_before_new_work( + monkeypatch, kind, effect +): + ledger = install_ledger(monkeypatch) + trace = fixture.Trace(shared_payload(), lambda_context(30000), "execution") + errors = { + "checkpoint": lambda: fixture.CheckpointError("Invalid checkpoint token"), + "background": lambda: fixture.BackgroundThreadError( + "failed", ValueError("source") + ), + "suspend": lambda: fixture.SuspendExecution("revoked"), + } + + def checkpoint(**options): + if effect == "rpc": + trace.next_call_id("checkpoint") + raise errors[kind]() + + def step(body, **options): + if effect == "body": + body(None) + if effect == "rpc": + trace.next_call_id("checkpoint") + raise errors[kind]() + + trace.durable_context = SimpleNamespace( + state=SimpleNamespace(create_checkpoint=checkpoint), step=step + ) + fixture.admission_probes(trace) + probes = [ + r["observation"] + for r in ledger.observations() + if r["observation"]["kind"] == "admission_probe" + ] + assert probes[0]["outcome"] == ("ERROR" if effect == "rpc" else "REJECTED") + assert probes[1]["outcome"] == ("REJECTED" if effect == "none" else "ERROR") From 58bf50509a40f36ca956819a621e1415b10fb5b1 Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:36:59 -0700 Subject: [PATCH 30/32] test: restore lifecycle case 2 --- .github/workflows/lmi-e2e-tests.yml | 2 -- lmi_tests/shared.py | 9 --------- 2 files changed, 11 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 0cc2c747c..2bca89579 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -130,8 +130,6 @@ jobs: allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - name: Run seven shared lifecycle requirements timeout-minutes: 60 - env: - LMI_SKIP_CASE_2: "1" run: hatch run lmi:shared - name: Refresh credentials for evidence and cleanup if: always() && hashFiles('lmi_tests/artifacts/manifest.json') != '' diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index b9246a7bf..5d5aed898 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -96,12 +96,6 @@ def make_manifest( ] scenarios = [] for required in required_cases("python", "lifecycle", coverage): - # Temporary diagnostic run; restore the full catalog after validation. - if ( - os.environ.get("LMI_SKIP_CASE_2") == "1" - and required.requirement_id == "lmi-lifecycle-2" - ): - continue concurrency = required.minimum_native_concurrency case_id = f"{required.requirement_id}-{required.scenario}" scenarios.append( @@ -122,9 +116,6 @@ def make_manifest( }, } ) - if os.environ.get("LMI_SKIP_CASE_2") == "1": - # Diagnostic only: observe case 7 before case 6 can quarantine c1. - scenarios.sort(key=lambda case: case["requirement_id"] != "lmi-lifecycle-7") return { "schemaVersion": 1, "runId": actual["run"], From e42887291ed191c4ed8133ad5d67afad9bece7fb Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 16:45:52 -0700 Subject: [PATCH 31/32] refactor: remove redundant cohort ID --- .github/workflows/lmi-e2e-tests.yml | 2 +- lmi_tests/README.md | 7 +++++-- lmi_tests/cloud.py | 6 +++--- lmi_tests/fixture.py | 2 -- lmi_tests/shared.py | 5 +---- lmi_tests/shared_adapter.py | 4 +--- lmi_tests/shared_contract.py | 2 +- lmi_tests/shared_evidence.py | 2 -- lmi_tests/shared_scenarios.py | 2 +- lmi_tests/tests/unit/test_shared_cleanup.py | 6 +++--- lmi_tests/tests/unit/test_shared_observations.py | 1 - 11 files changed, 16 insertions(+), 23 deletions(-) diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index 2bca89579..c3404c41f 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -19,7 +19,7 @@ env: # Ignore the CodeBuild image's shared pip target and Python search path. PIP_CONFIG_FILE: /dev/null PYTHONPATH: "" - CONFORMANCE_REVISION: cc1eae152d9d0a49ecb777db140339f290b43567 + CONFORMANCE_REVISION: 15ea7714de3c06b6552c4bd33fd036272a7e800b LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} jobs: diff --git a/lmi_tests/README.md b/lmi_tests/README.md index d8ae1ccbb..efa488150 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -29,7 +29,7 @@ and check out the exact `CONFORMANCE_REVISION` from `shared_contract.py`: ```bash git clone https://github.com/aws/aws-durable-execution-conformance-tests.git .conformance -git -C .conformance checkout --detach cc1eae152d9d0a49ecb777db140339f290b43567 +git -C .conformance checkout --detach 15ea7714de3c06b6552c4bd33fd036272a7e800b hatch run lmi:shared-setup hatch run lmi:lint hatch run lmi:shared-tests @@ -65,7 +65,10 @@ control. Direct CLI calls do not acquire a lock and are not serialized with CI. `shared plan` writes the seven-case manifest without invoking functions. `shared run` writes under `lmi_tests/artifacts/shared/standard/`: the manifest, coverage, JSON/JUnit reports, launch journal and retained case evidence. Each invocation -uses a case-owned control/evidence prefix. A service resume is journaled under +uses a case-owned control/evidence prefix derived from `runId` and `caseId`. +The case hash uses only `caseId` within the run prefix. Plans, payloads and evidence +have no additional cohort identifier. Generate new run artifacts with this pinned +revision; inspect or clean up older artifacts with their original adapter revision. A service resume is journaled under the same durable execution; it is not another driver Invoke. Trusted same-repository pull requests and manual workflow runs exercise the cloud diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 6efabcc8c..75c34bb7b 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -83,9 +83,9 @@ def __init__( self.callback_records: list[dict[str, Any]] = [] self.control_prefix = f"runs/{manifest['run']}/control/" if case_context is not None: - owner = hashlib.sha256( - f"{case_context.scenario.case_id}:{case_context.scenario.cohort_id}".encode() - ).hexdigest()[:20] + owner = hashlib.sha256(case_context.scenario.case_id.encode()).hexdigest()[ + :20 + ] self.control_prefix = f"runs/{manifest['run']}/cases/{owner}/control/" def check_active(self) -> None: diff --git a/lmi_tests/fixture.py b/lmi_tests/fixture.py index 58e162e08..0fc103c71 100644 --- a/lmi_tests/fixture.py +++ b/lmi_tests/fixture.py @@ -86,7 +86,6 @@ def __init__(self, payload, context, execution_arn): for key in ( "runId", "caseId", - "cohortId", "role", "sharedScenario", "controlPrefix", @@ -128,7 +127,6 @@ def __init__(self, payload, context, execution_arn): for key in ( "runId", "caseId", - "cohortId", "role", "coverageProfile", "sharedScenario", diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index 5d5aed898..c4e838db5 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -101,7 +101,6 @@ def make_manifest( scenarios.append( { "case_id": case_id, - "cohort_id": "cohort-1", "requirement_id": required.requirement_id, "profile_id": f"c{concurrency}", "scenario": required.scenario, @@ -227,9 +226,7 @@ def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: target = profiles[case["profile_id"]]["resource_id"] if target not in deployment["functions"].values(): raise CollectionError("Cleanup target is outside the owned deployment") - owner = hashlib.sha256( - f"{case['case_id']}:{case.get('cohort_id', 'cohort-1')}".encode() - ).hexdigest()[:20] + owner = hashlib.sha256(case["case_id"].encode()).hexdigest()[:20] prefix = f"runs/{deployment['run']}/cases/{owner}/control/" case_dir = directory / coverage / "source" / "cases" / owner try: diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py index 26aabeaae..88daf41a4 100644 --- a/lmi_tests/shared_adapter.py +++ b/lmi_tests/shared_adapter.py @@ -127,9 +127,7 @@ async def start(self, context: ScenarioContext) -> None: ) if key is None: raise ProvisioningError("Case target is outside the owned deployment") - owner = hashlib.sha256( - f"{context.scenario.case_id}:{context.scenario.cohort_id}".encode() - ).hexdigest()[:20] + owner = hashlib.sha256(context.scenario.case_id.encode()).hexdigest()[:20] cloud = await asyncio.to_thread( self.cloud_factory, self.deployment, diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py index 300f07c0a..369debeee 100644 --- a/lmi_tests/shared_contract.py +++ b/lmi_tests/shared_contract.py @@ -10,7 +10,7 @@ CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" # Updated only with the matching reviewed shared evidence contract. -CONFORMANCE_REVISION = "cc1eae152d9d0a49ecb777db140339f290b43567" +CONFORMANCE_REVISION = "15ea7714de3c06b6552c4bd33fd036272a7e800b" def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py index f7efcb19f..8c40ae82d 100644 --- a/lmi_tests/shared_evidence.py +++ b/lmi_tests/shared_evidence.py @@ -103,7 +103,6 @@ def observations(self) -> tuple[list[dict[str, Any]], list[str]]: raw.get("run") != self.context.run_id or raw.get("commit") != self.cloud.manifest["commit"] or raw.get("caseId") != self.context.scenario.case_id - or raw.get("cohortId") != self.context.scenario.cohort_id ): errors.append( "Raw evidence does not belong to the admitted run/artifact/case" @@ -605,7 +604,6 @@ def build_evidence( "schemaVersion": 1, "runId": context.run_id, "caseId": context.scenario.case_id, - "cohortId": context.scenario.cohort_id, "coverageProfile": context.scenario.coverage_profile, "scenario": context.scenario.scenario, "requirementId": context.scenario.requirement_id, diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py index a9b97abb7..151305e7b 100644 --- a/lmi_tests/shared_scenarios.py +++ b/lmi_tests/shared_scenarios.py @@ -69,7 +69,7 @@ def seconds(self, budget: str, fallback: float) -> float: def gate(self, label: str) -> str: owner = hashlib.sha256( - f"{self.context.run_id}:{self.context.scenario.case_id}:{self.context.scenario.cohort_id}".encode() + f"{self.context.run_id}:{self.context.scenario.case_id}".encode() ).hexdigest()[:20] return f"case-{owner}-{label}" diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py index 0c3a3df23..657055168 100644 --- a/lmi_tests/tests/unit/test_shared_cleanup.py +++ b/lmi_tests/tests/unit/test_shared_cleanup.py @@ -33,7 +33,7 @@ def test_case_identity_is_checked_before_control_release( "region": "us-west-2", "functions": {"c1": target}, } - case = {"case_id": "case", "cohort_id": "cohort", "profile_id": "c1"} + case = {"case_id": "case", "profile_id": "c1"} plan = { "runId": "run", "profiles": [{"profile_id": "c1", "resource_id": target}], @@ -42,7 +42,7 @@ def test_case_identity_is_checked_before_control_release( output = tmp_path / "standard" output.mkdir() (output / "manifest.json").write_text(json.dumps(plan)) - identity = hashlib.sha256(b"case:cohort").hexdigest()[:20] + identity = hashlib.sha256(b"case").hexdigest()[:20] invocations = output / "source/cases" / identity / "invocations" invocations.mkdir(parents=True) (invocations / "intent.json").write_text( @@ -137,7 +137,7 @@ def test_one_identified_execution_does_not_hide_another_unreconciled_invoke( } ) ) - owner = hashlib.sha256(b"case:cohort-1").hexdigest()[:20] + owner = hashlib.sha256(b"case").hexdigest()[:20] intents = output / "source/cases" / owner / "invocations" intents.mkdir(parents=True) known, missing = intents / "known.json", intents / "missing.json" diff --git a/lmi_tests/tests/unit/test_shared_observations.py b/lmi_tests/tests/unit/test_shared_observations.py index a6e4ab856..e1335c580 100644 --- a/lmi_tests/tests/unit/test_shared_observations.py +++ b/lmi_tests/tests/unit/test_shared_observations.py @@ -95,7 +95,6 @@ def shared_payload(scenario: str = "success") -> dict[str, Any]: return { "runId": "unit", "caseId": "case", - "cohortId": "cohort-1", "role": "victim", "coverageProfile": "standard", "sharedScenario": scenario, From 044fbd8e9132f9933056204f2b59877826fae5ef Mon Sep 17 00:00:00 2001 From: Frank Chen <65260095+zhongkechen@users.noreply.github.com> Date: Fri, 13 Nov 2026 17:05:57 -0700 Subject: [PATCH 32/32] refactor: consume shared LMI workflow and driver --- .github/scripts/lmi-ci.sh | 34 + .github/workflows/lmi-e2e-tests.yml | 156 +--- lmi_tests/README.md | 18 +- lmi_tests/cloud.py | 692 +--------------- lmi_tests/deploy.py | 531 +----------- lmi_tests/evidence.py | 75 -- lmi_tests/runtime.py | 7 + lmi_tests/shared-suite-design.md | 7 +- lmi_tests/shared.py | 283 +------ lmi_tests/shared_adapter.py | 248 +----- lmi_tests/shared_contract.py | 32 +- lmi_tests/shared_evidence.py | 780 +----------------- lmi_tests/shared_scenarios.py | 402 +-------- lmi_tests/summary.py | 56 +- lmi_tests/tests/unit/test_cloud.py | 430 ---------- lmi_tests/tests/unit/test_deploy.py | 447 ---------- lmi_tests/tests/unit/test_evidence.py | 65 -- .../tests/unit/test_lifecycle_mapping.py | 99 --- lmi_tests/tests/unit/test_peer_recovery.py | 216 ----- lmi_tests/tests/unit/test_shared_adapter.py | 547 ------------ lmi_tests/tests/unit/test_shared_cleanup.py | 167 ---- .../unit/test_shared_control_receipts.py | 46 -- lmi_tests/tests/unit/test_shared_manifest.py | 4 +- .../unit/test_shared_partial_evidence.py | 159 ---- lmi_tests/tests/unit/test_shared_workflow.py | 164 ++-- 25 files changed, 277 insertions(+), 5388 deletions(-) create mode 100644 .github/scripts/lmi-ci.sh delete mode 100644 lmi_tests/evidence.py create mode 100644 lmi_tests/runtime.py delete mode 100644 lmi_tests/tests/unit/test_cloud.py delete mode 100644 lmi_tests/tests/unit/test_evidence.py delete mode 100644 lmi_tests/tests/unit/test_lifecycle_mapping.py delete mode 100644 lmi_tests/tests/unit/test_peer_recovery.py delete mode 100644 lmi_tests/tests/unit/test_shared_adapter.py delete mode 100644 lmi_tests/tests/unit/test_shared_cleanup.py delete mode 100644 lmi_tests/tests/unit/test_shared_control_receipts.py delete mode 100644 lmi_tests/tests/unit/test_shared_partial_evidence.py diff --git a/.github/scripts/lmi-ci.sh b/.github/scripts/lmi-ci.sh new file mode 100644 index 000000000..9c9a9a895 --- /dev/null +++ b/.github/scripts/lmi-ci.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# Python commands for the shared LMI reusable workflow's stage contract. +set -euo pipefail + +case "${1:?Expected a workflow stage}" in + setup) + uv pip install hatch==1.16.5 + hatch run lmi:shared-setup + ;; + harness) + hatch run lmi:lint + hatch run lmi:shared-tests --junitxml=lmi_tests/artifacts/unit.xml + hatch run examples:python - <<'PYTHON' +from pathlib import Path +import aws_durable_execution_sdk_python as sdk + +source = Path('packages/aws-durable-execution-sdk-python/src').resolve() +assert Path(sdk.__file__).resolve().is_relative_to(source), sdk.__file__ +PYTHON + ;; + build) hatch run lmi:build ;; + deploy) + hatch run lmi:python -m lmi_tests.deploy deploy \ + --run-id "$LMI_SHARED_RUN_ID" \ + --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" + ;; + run) hatch run lmi:shared ;; + collect) hatch run lmi:python -m lmi_tests.deploy collect ;; + cleanup) hatch run lmi:shared-cleanup ;; + summary) hatch run lmi:python -m lmi_tests.summary ;; + *) printf 'Unknown LMI workflow stage: %s\n' "$1" >&2; exit 2 ;; +esac diff --git a/.github/workflows/lmi-e2e-tests.yml b/.github/workflows/lmi-e2e-tests.yml index c3404c41f..85ff5efda 100644 --- a/.github/workflows/lmi-e2e-tests.yml +++ b/.github/workflows/lmi-e2e-tests.yml @@ -15,153 +15,21 @@ on: permissions: contents: read -env: - # Ignore the CodeBuild image's shared pip target and Python search path. - PIP_CONFIG_FILE: /dev/null - PYTHONPATH: "" - CONFORMANCE_REVISION: 15ea7714de3c06b6552c4bd33fd036272a7e800b - LMI_SHARED_RUN_ID: ${{ github.run_id }}-${{ github.run_attempt }} - jobs: - harness: - # Keep fork and Dependabot code outside the testing account. - runs-on: ${{ github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]') && 'ubuntu-latest' || format('codebuild-github-actions-runner-{0}-{1}', github.run_id, github.run_attempt) }} - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check out pinned shared conformance - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: aws/aws-durable-execution-conformance-tests - ref: ${{ env.CONFORMANCE_REVISION }} - path: .conformance - persist-credentials: false - # uv supplies Python on the CodeBuild Amazon Linux image and on Ubuntu. - - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - with: - python-version: '3.14' - activate-environment: true - - run: uv pip install hatch==1.16.5 - - run: hatch run lmi:shared-setup - - run: hatch run lmi:lint - - run: hatch run lmi:shared-tests --junitxml=lmi_tests/artifacts/unit.xml - - name: Verify examples workspace initialization - run: | - hatch run examples:python - <<'PY' - from pathlib import Path - import aws_durable_execution_sdk_python as sdk - - source = Path('packages/aws-durable-execution-sdk-python/src').resolve() - assert Path(sdk.__file__).resolve().is_relative_to(source), sdk.__file__ - PY - - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - if: always() - with: - name: lmi-harness-tests - path: lmi_tests/artifacts/ - - cloud: - name: cloud (Python 3.14, concurrency 1 and 2) - needs: harness - # Trusted PRs and manual runs exercise the shared lifecycle suite. SDK - # failures remain strict; fork and Dependabot PRs run only the harness. - if: github.actor != 'dependabot[bot]' && (github.event_name == 'workflow_dispatch' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)) - runs-on: codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }} - timeout-minutes: 120 + lifecycle: permissions: contents: read id-token: write - # GitHub Actions is the cross-run concurrency guard. Hold this slot through - # cleanup and use the same group for every PR/ref targeting this provider. - concurrency: - group: lmi-e2e-shared-capacity-provider - cancel-in-progress: false - queue: max - env: - LMI_STACK_NAME: python-lmi-e2e - AWS_REGION: ${{ inputs.region || 'us-west-2' }} - CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} + uses: aws/aws-durable-execution-conformance-tests/.github/workflows/lmi-lifecycle.yml@56e7e444457653775579c120469d931b71c8cfd8 + with: + conformance-ref: 56e7e444457653775579c120469d931b71c8cfd8 + sdk-script: .github/scripts/lmi-ci.sh + runtime-label: Python 3.14 + lambda-runtime: python3.14 + stack-name: python-lmi-e2e + region: ${{ inputs.region || 'us-west-2' }} + secrets: + TEST_ROLE_ARN: ${{ secrets.TEST_ROLE_ARN }} TEST_ACCOUNT_ID: ${{ secrets.TEST_ACCOUNT_ID }} TEST_LAMBDA_EXECUTION_ROLE_ARN: ${{ secrets.TEST_LAMBDA_EXECUTION_ROLE_ARN }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - name: Check out pinned shared conformance - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: aws/aws-durable-execution-conformance-tests - ref: ${{ env.CONFORMANCE_REVISION }} - path: .conformance - persist-credentials: false - # uv supplies Python on the CodeBuild Amazon Linux image and on Ubuntu. - - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - with: - python-version: '3.14' - activate-environment: true - - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 - with: - role-to-assume: ${{ secrets.TEST_ROLE_ARN }} - role-session-name: python-lmi-e2e - aws-region: ${{ env.AWS_REGION }} - allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Verify test account and prerequisites - run: | - test -n "$CAPACITY_PROVIDER_ARN" - test -n "$TEST_LAMBDA_EXECUTION_ROLE_ARN" - test "$(aws sts get-caller-identity --query Account --output text)" = "$TEST_ACCOUNT_ID" - - run: uv pip install hatch==1.16.5 - - run: hatch run lmi:shared-setup - - name: Build SDK checkout - timeout-minutes: 5 - run: hatch run lmi:build - - name: Create or update persistent LMI functions - timeout-minutes: 35 - env: - LMI_RUNTIME: python3.14 - run: | - hatch run lmi:python -m lmi_tests.deploy deploy \ - --run-id "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" \ - --runtime "$LMI_RUNTIME" --stack-name "$LMI_STACK_NAME" - - name: Refresh credentials for cloud assertions - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 - with: - role-to-assume: ${{ secrets.TEST_ROLE_ARN }} - role-session-name: python-lmi-assertions - aws-region: ${{ env.AWS_REGION }} - allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Run seven shared lifecycle requirements - timeout-minutes: 60 - run: hatch run lmi:shared - - name: Refresh credentials for evidence and cleanup - if: always() && hashFiles('lmi_tests/artifacts/manifest.json') != '' - uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6.3.0 - with: - role-to-assume: ${{ secrets.TEST_ROLE_ARN }} - role-session-name: python-lmi-cleanup - aws-region: ${{ env.AWS_REGION }} - allowed-account-ids: ${{ secrets.TEST_ACCOUNT_ID }} - - name: Collect service history and runtime evidence - if: always() - timeout-minutes: 5 - run: | - if test -f lmi_tests/artifacts/manifest.json; then - hatch run lmi:python -m lmi_tests.deploy collect - fi - - name: Release this run's work and retain persistent functions - if: always() - timeout-minutes: 5 - run: | - if test -f lmi_tests/artifacts/manifest.json; then - hatch run lmi:shared-cleanup - fi - - name: Retain diagnostics - if: always() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: lmi-3.14-${{ github.run_id }}-${{ github.run_attempt }} - retention-days: 7 - path: lmi_tests/artifacts/ - - name: Summarize outcomes - if: always() - run: | - python -m lmi_tests.summary >> "$GITHUB_STEP_SUMMARY" - find lmi_tests/artifacts/shared -name report.json -print 2>/dev/null >> "$GITHUB_STEP_SUMMARY" || true + CAPACITY_PROVIDER_ARN: ${{ secrets.CAPACITY_PROVIDER_ARN }} diff --git a/lmi_tests/README.md b/lmi_tests/README.md index efa488150..d9859bec2 100644 --- a/lmi_tests/README.md +++ b/lmi_tests/README.md @@ -16,9 +16,12 @@ shared matrix directly rather than maintaining a second requirement catalog. | `lmi-lifecycle-6` | `warm-reuse` | c1 | victim, warm | | `lmi-lifecycle-7` | `normal-failure` | c1 | victim | -The shared runner owns assertions, strict reports, placement identity, per-case -scheduling, launch journals and independent retirement auditing. This repository owns the -real public SDK handlers, AWS transport, source observations and deployment. +The shared package owns assertions, AWS transport, persistent deployment, scenario +choreography, source normalization, strict reports, scheduling and retirement. +This repository owns the real Python SDK handlers, observation hooks, ZIP build +and thin runtime bindings. The workflow definition also lives in the shared +repository; `.github/workflows/lmi-e2e-tests.yml` calls its pinned +`lmi-lifecycle.yml`, using `.github/scripts/lmi-ci.sh` for Python stage commands. OTel/exporter tests and the previous Python-specific matrix are deferred. There is no local conformance profile or validator that replaces a shared rule. @@ -29,7 +32,7 @@ and check out the exact `CONFORMANCE_REVISION` from `shared_contract.py`: ```bash git clone https://github.com/aws/aws-durable-execution-conformance-tests.git .conformance -git -C .conformance checkout --detach 15ea7714de3c06b6552c4bd33fd036272a7e800b +git -C .conformance checkout --detach 56e7e444457653775579c120469d931b71c8cfd8 hatch run lmi:shared-setup hatch run lmi:lint hatch run lmi:shared-tests @@ -42,7 +45,9 @@ out core SDK and testing library; the Lambda bundle contains the core SDK and its dependencies plus fixture/observation modules. It does not include the OTel SDK, an OTel plugin, local catalogs, validators or the shared runner itself. -Unit tests cover manifest completeness, case ownership, cancellation, source +Python unit tests cover pinned workflow delegation, the Python artifact/manifest +binding, fixture observations and bootstrap. Shared driver regression tests now +live in the conformance repository and cover case ownership, cancellation, source normalization, driver ordering and cleanup. Local e2e tests use the actual public SDK and its in-process test service, replacing only AWS transport and evidence storage. They verify checkpoint/replay and preserve genuine admission failures. @@ -112,7 +117,8 @@ invocation must find it in the same process; it observes the old callback using the original request identity. Failure to reuse that process is a placement or collection failure, not permission to select a replacement victim. -`shared_evidence.py` normalizes retained observations into the shared schema. +`shared_evidence.py` supplies Python error expectations and process identity to +the shared source-evidence normalizer. Service payload wrappers are decoded consistently with checkpoint snapshots; truncated results are rejected. Replay compares a real resume's initial checkpoints to the final service history and counts actual user-function entries. diff --git a/lmi_tests/cloud.py b/lmi_tests/cloud.py index 75c34bb7b..ceb3c2830 100644 --- a/lmi_tests/cloud.py +++ b/lmi_tests/cloud.py @@ -1,691 +1,19 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Bounded real-service driver using the repository's cloud runner.""" +"""Python fixture binding to the shared AWS driver.""" -import json -import hashlib -from pathlib import Path -from threading import Event, Lock -from typing import Any -from concurrent.futures import ThreadPoolExecutor -from copy import deepcopy -import time -import uuid +from aws_durable_execution_conformance_tests.lmi.cloud import Cloud as SharedCloud +from lmi_tests.deploy import ARTIFACTS, manager -from botocore.exceptions import ClientError -from aws_durable_execution_sdk_python_testing import DurableFunctionCloudTestRunner -from lmi_tests.deploy import ARTIFACTS, CONCURRENCIES, QUALIFIER, client, save, verify -from lmi_tests.evidence import ( - CollectionError, - ProvisioningError, - check_controls, - select, -) - - -class RunnerClient: - """Share paced history reads with the public runner; other APIs are unchanged.""" - - def __init__(self, cloud, *, before_call=None): - self.cloud = cloud - self.before_call = before_call - - def invoke(self, **kwargs): - if self.before_call is not None: - self.before_call() - return self.cloud.lam.invoke(**kwargs) - - def __getattr__(self, name): - operation = getattr(self.cloud.lam, name) - if not callable(operation): - return operation - - def guarded(**kwargs): - if self.before_call is not None: - self.before_call() - return operation(**kwargs) - - return guarded - - def get_durable_execution_history(self, **kwargs): - return self.cloud.history_page(**kwargs) - - -class Cloud: +class Cloud(SharedCloud): def __init__( self, manifest, configuration=None, *, artifacts_dir=None, case_context=None ): - self.manifest = manifest - self.configuration = configuration - self.lam, self.s3, self.logs = client("lambda"), client("s3"), client("logs") - self.events = {} - self.invocations = [] - self.gates = set() - self.released_gates: set[str] = set() - self._next_history_read = 0.0 - self.artifacts_dir = ( - Path(artifacts_dir) if artifacts_dir is not None else ARTIFACTS - ) - self.case_context = case_context - self.cancel_event = Event() - self._history_lock = Lock() - self._retiring = False - self.request_roles: dict[str, str] = {} - self.request_launches: dict[str, str] = {} - self.raw_histories: dict[str, list[dict[str, Any]]] = {} - self.execution_results: dict[str, dict[str, Any]] = {} - self.control_records: list[dict[str, Any]] = [] - self.source_uris: set[str] = set() - self.observation_ended_at: float | None = None - self.deadline_measurement: dict[str, Any] | None = None - self.retired_at: float | None = None - self.callback_records: list[dict[str, Any]] = [] - self.control_prefix = f"runs/{manifest['run']}/control/" - if case_context is not None: - owner = hashlib.sha256(case_context.scenario.case_id.encode()).hexdigest()[ - :20 - ] - self.control_prefix = f"runs/{manifest['run']}/cases/{owner}/control/" - - def check_active(self) -> None: - if self.cancel_event.is_set(): - raise CollectionError("Case driver was cancelled") - - def cancel(self) -> None: - """Request cooperative stop before joining this case's blocking worker.""" - self.cancel_event.set() - - def begin_retirement(self) -> None: - """Called only after the cancelled phase worker has actually exited.""" - self._retiring = True - self.cancel_event.clear() - - def wait(self, seconds: float) -> None: - self.check_active() - if self.case_context is None: - time.sleep(max(0, seconds)) - elif self.cancel_event.wait(max(0, seconds)): - self.check_active() - - def _save(self, relative: str, data: Any) -> None: - save(self.artifacts_dir / relative, data) - self.source_uris.add((self.artifacts_dir / relative).resolve().as_uri()) - - def _require_execution_target(self, arn: str) -> None: - if self.case_context is not None and not arn.startswith( - self.manifest["functions"][self.configuration] + "/durable-execution/" - ): - raise CollectionError("Execution is outside the case-owned function") - - def _bind_requests(self, events: list[dict[str, Any]]) -> None: - if self.case_context is None: - return - by_marker = {item["marker"]: item for item in self.invocations} - for event in sorted( - events, key=lambda item: (item["time"], item["request"], item["sequence"]) - ): - if ( - event["phase"] != "WRAPPER_ENTER" - or event["request"] in self.request_roles - ): - continue - item = by_marker.get(event["marker"]) - if item is None: - raise CollectionError( - "Observed an invocation outside this case's launch journal" - ) - if item.get("arn") is not None and item["arn"] != event["execution"]: - raise CollectionError( - "An invocation changed its original durable execution" - ) - self._require_execution_target(event["execution"]) - item["arn"] = event["execution"] - ordinal = len(item["native_requests"]) - role = item["role"] - if ordinal and item["resume_roles"]: - role = item["resume_roles"][ - min(ordinal - 1, len(item["resume_roles"]) - 1) - ] - launch_id = ( - item["launch_id"] - if ordinal == 0 - else f"{item['launch_id']}:native:{event['request']}" - ) - if ordinal == 0: - self.case_context.update_attempt( - launch_id, - requestId=event["request"], - executionArn=event["execution"], - ) - else: - self.case_context.record_attempt( - launch_id, - role=role, - requestId=event["request"], - executionArn=event["execution"], - source="service-observed", - ) - item["native_requests"].append(event["request"]) - self.request_roles[event["request"]] = role - self.request_launches[event["request"]] = launch_id - - def verify(self): - if self.manifest.get("concurrencies") != CONCURRENCIES or set( - self.manifest["functions"] - ) != set(CONCURRENCIES): - raise ProvisioningError( - "One deployment must contain both c1 and c2 functions" - ) - for key, arn in self.manifest["functions"].items(): - config = self.lam.get_function_configuration(FunctionName=arn) - scaling = self.lam.get_function_scaling_config( - FunctionName=arn.rsplit(":", 1)[0], Qualifier=QUALIFIER - ) - verify(config, scaling, self.manifest, key) - - @property - def concurrency(self): - return self.manifest["concurrencies"][self.configuration] - - def start( - self, - scenario, - gate=None, - target_environment=None, - *, - preparation_gate=None, - role=None, - resume_roles=(), - dispatcher_payload=None, - marker=None, - ): - self.check_active() - if self._retiring: - raise CollectionError("A retiring case cannot launch new executions") - if self.configuration not in self.manifest["functions"]: - raise ProvisioningError( - "Select a deployed concurrency configuration before Invoke" - ) - marker = marker or scenario + "-" + uuid.uuid4().hex[:12] - payload = dict(dispatcher_payload or {}) - if dispatcher_payload: - payload["sharedScenario"] = dispatcher_payload["scenario"] - payload.update(scenario=scenario, marker=marker, run=self.manifest["run"]) - if self.case_context is not None: - payload["coverageProfile"] = "standard" - payload["behaviorBudgets"] = dict( - self.case_context.scenario.behavior_budgets - ) - payload["controlPrefix"] = self.control_prefix - if "release-all" not in self.gates: - self.hold("release-all") - if role not in self.case_context.scenario.roles: - raise ValueError("Launch role is not part of the admitted scenario") - if any( - resume not in self.case_context.scenario.roles - for resume in resume_roles - ): - raise ValueError("Resume role is not part of the admitted scenario") - if target_environment is not None: - payload["target_environment"] = target_environment - if gate: - payload["gate"] = gate - self.hold(gate) - if preparation_gate: - payload["preparation_gate"] = preparation_gate - self.hold(preparation_gate) - runner = DurableFunctionCloudTestRunner( - self.manifest["functions"][self.configuration], - region=self.manifest["region"], - ) - runner.lambda_client = RunnerClient(self, before_call=self.check_active) - item = { - "marker": marker, - "arn": None, - "configuration": self.configuration, - "runner": runner, - "started": time.time(), - "payload": payload, - "role": role, - "resume_roles": tuple(resume_roles), - "native_requests": [], - "launch_id": marker, - } - if self.case_context is not None: - # Retain attempted writes before the transport, even if its response - # is lost. Later ledger observations can recover their native IDs. - self.invocations.append(item) - self.case_context.record_attempt( - marker, role=role, marker=marker, source="driver-invoke" - ) - self._save( - f"invocations/{marker}.json", - {"payload": payload, "launched": item["started"]}, - ) - self.check_active() - try: - arn = runner.run_async(input=payload) - if not arn: - raise CollectionError("Invoke did not return a durable execution ARN") - self._require_execution_target(arn) - except Exception as error: - self._save( - f"invocations/{marker}.json", {"payload": payload, "error": str(error)} - ) - raise CollectionError(f"{marker}: invocation API failed") from error - item["arn"] = arn - if self.case_context is None: - self.invocations.append(item) - print(f"LMI {scenario}: marker={marker}, execution={arn}", flush=True) - self._save( - f"invocations/{marker}.json", - { - "payload": payload, - "arn": arn, - "function": runner.function_name, - "started": item["started"], - }, - ) - return item - - def hold(self, gate): - self.check_active() - if gate in self.gates: - return # another invocation in this case uses the same barrier - try: - self.s3.put_object( - Bucket=self.manifest["bucket"], - Key=f"{self.control_prefix}{gate}", - Body=b"hold", - IfNoneMatch="*", - ) - except ClientError as error: - if error.response["Error"]["Code"] != "PreconditionFailed": - raise CollectionError( - f"Cannot initialize control {gate}: {error}" - ) from error - self.gates.add(gate) - self.control_records.append( - {"gate": gate, "action": "hold", "time": time.time()} - ) - - def release(self, gate): - self.check_active() - if gate in self.released_gates: - return - if self.case_context is not None and gate not in self.gates: - raise CollectionError( - "Refusing to release a control not owned by this case" - ) - intent = { - "gate": gate, - "action": "release", - "started": time.time(), - "state": "PENDING", - } - self.control_records.append(intent) - self._save("control-records.json", self.control_records) - self.s3.put_object( - Bucket=self.manifest["bucket"], - Key=f"{self.control_prefix}{gate}", - Body=b"release", - ) - intent.update(time=time.time(), state="ACKNOWLEDGED") - self.released_gates.add(gate) - self._save("control-records.json", self.control_records) - - def refresh(self, markers=None, *, validate_controls=True): - self.check_active() - if self.case_context is not None and markers == []: - raise CollectionError( - "A shared case cannot collect another case's run prefix" - ) - # A new pytest fixture must not scan every earlier case before it can - # observe its first event. Explicit [] requests the complete run for collection. - if markers is None: - markers = [item["marker"] for item in self.invocations] - if self.case_context is not None: - if not markers: - return [] - if not set(markers) <= {item["marker"] for item in self.invocations}: - raise CollectionError("A shared case cannot read another case's marker") - prefix = f"runs/{self.manifest['run']}/events/" - prefixes = [f"{prefix}{marker}/" for marker in markers] if markers else [prefix] - missing = [] - try: - for prefix in prefixes: - for page in self.s3.get_paginator("list_objects_v2").paginate( - Bucket=self.manifest["bucket"], Prefix=prefix - ): - missing.extend( - obj["Key"] - for obj in page.get("Contents", []) - if obj["Key"] not in self.events - ) - - def read(key): - self.check_active() - response = self.s3.get_object(Bucket=self.manifest["bucket"], Key=key) - with response["Body"] as stream: - event = json.loads(stream.read()) - if ( - event["run"] != self.manifest["run"] - or event["commit"] != self.manifest["commit"] - ): - raise CollectionError("Event came from another build/run") - self.check_active() - return key, event - - if missing: - with ThreadPoolExecutor(max_workers=8) as readers: - self.events.update(readers.map(read, missing)) - except ClientError as error: - raise CollectionError( - "Could not retrieve external lifecycle/side-effect ledger" - ) from error - events = sorted( - self.events.values(), key=lambda e: (e["time"], e["request"], e["sequence"]) - ) - self.check_active() - self._bind_requests(events) - self.source_uris.update( - f"s3://{self.manifest['bucket']}/{key}" for key in self.events - ) - self._save("events.json", events) - selected = [e for e in events if not markers or e["marker"] in markers] - if markers and validate_controls: - check_controls(selected) - return selected - - def for_item(self, item): - return self.refresh(markers=[item["marker"]]) - - def poll( - self, - predicate, - seconds=30, - category=CollectionError, - message="Evidence deadline exceeded", - ): - end = time.monotonic() + seconds - while True: - self.check_active() - value = predicate() - if value: - return value - if time.monotonic() >= end: - raise category(message) - self.wait(min(0.5, max(0, end - time.monotonic()))) - - def phase(self, item, phase, **kwargs): - return self.poll(lambda: select(self.for_item(item), phase), **kwargs) - - def record_deadline_measurement(self, snapshot: dict[str, Any]) -> None: - """Persist the completed measurement before any victim-fault release.""" - self.check_active() - recorded = deepcopy(snapshot) - self._save("measurements/deadline.json", recorded) - self.deadline_measurement = recorded - - def wait_terminal(self, item: dict[str, Any]) -> dict[str, Any]: - """Observe actual terminal state and wrapper exits without asserting success. - - Deadline fixtures can legitimately finish with several service outcomes. - Their declared conformance expectations decide acceptance; this method - only observes completion and does not stop or retry an execution. - """ - self._require_execution_target(item["arn"]) - - def terminal() -> dict[str, Any] | None: - result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) - if result["Status"] in {"SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"}: - return result - return None - - result = self.poll( - terminal, - seconds=self.manifest["driverTimeout"], - message="Released victim execution did not reach an actual terminal state", - ) - self.execution_results[item["arn"]] = result - self._save(f"executions/{item['marker']}.json", result) - - def wrappers_closed() -> bool: - events = self.for_item(item) - entered = {event["request"] for event in select(events, "WRAPPER_ENTER")} - exited = { - event["request"] - for event in events - if event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - } - return bool(entered) and entered <= exited - - self.poll( - wrappers_closed, - seconds=self.manifest["driverTimeout"], - message="Released victim has native invocations without actual wrapper exits", + super().__init__( + manifest, + configuration, + artifacts_dir=artifacts_dir or ARTIFACTS, + case_context=case_context, + verify_profile=manager().verify, ) - self.history(item) - return result - - def finish(self, item, status="SUCCEEDED"): - try: - item["runner"].wait_for_result( - item["arn"], timeout=self.manifest["driverTimeout"] - ) - except TimeoutError as error: - raise CollectionError( - "Driver timeout, not Lambda invocation timeout evidence" - ) from error - result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) - self.execution_results[item["arn"]] = result - self._save(f"executions/{item['marker']}.json", result) - assert result["Status"] == status, ( - f"{item['marker']}: {result['Status']} (expected {status})" - ) - if status == "SUCCEEDED": - assert json.loads(result["Result"]) == item["marker"], ( - "Result crossed execution boundaries" - ) - self.phase(item, "WRAPPER_RETURN") - return self.history(item) - - def history_page(self, **request): - """Serialize paced reads within one case, without shared-case throttling state.""" - with self._history_lock: - return self._history_page(**request) - - def _history_page(self, **request): - """Bound read-only throttling retries without replaying invocation writes.""" - deadline = time.monotonic() + 20 - last_error = None - for attempt in range(5): - self.check_active() - delay = max(0, self._next_history_read - time.monotonic()) - if time.monotonic() + delay >= deadline: - break - if delay: - self.wait(delay) - try: - return self.lam.get_durable_execution_history(**request) - except ClientError as error: - if error.response["Error"]["Code"] not in { - "TooManyRequestsException", - "ThrottlingException", - }: - raise - last_error = error - finally: - self._next_history_read = time.monotonic() + 1 - self._next_history_read = time.monotonic() + min(2**attempt, 4) - raise CollectionError( - "Execution-history API throttle budget exhausted" - ) from last_error - - def history(self, item): - events, marker = [], None - while True: - request = {"DurableExecutionArn": item["arn"], "IncludeExecutionData": True} - if marker: - request["Marker"] = marker - result = self.history_page(**request) - events.extend(result.get("Events", [])) - marker = result.get("NextMarker") - if not marker: - break - self.raw_histories[item["arn"]] = events - self._save(f"histories/{item['marker']}.json", events) - return events - - def platform_timeout(self, item, request): - # A TIMED_OUT logical execution or a client poll limit is insufficient. - # Require a real invocation-completed service event for this request. - def observed(): - for event in self.history(item): - details = event.get("InvocationCompletedDetails", {}) - error = json.dumps(details.get("Error", {})).lower().replace(" ", "") - if details.get("RequestId") == request and any( - label in error for label in ("timeout", "timedout") - ): - return event - return None - - return self.poll( - observed, - seconds=self.manifest["invocationTimeout"] + 40, - message="No service invocation timeout evidence for " + request, - ) - - def collect(self, *, full_run=True): - self.check_active() - if self.case_context is not None and full_run: - raise CollectionError("A shared case may collect only its own invocations") - errors = [] - if not full_run and not self.invocations: - return - try: - events = self.refresh( - markers=[] if full_run else [i["marker"] for i in self.invocations], - validate_controls=False, - ) - # Independent collect steps can recover execution ARNs without the - # pytest process or successful invoke response. - items = { - e["execution"]: {"arn": e["execution"], "marker": e["marker"]} - for e in events - } - except Exception as error: - errors.append(str(error)) - items = {} - if not full_run: - items.update({i["arn"]: i for i in self.invocations if i.get("arn")}) - for item in items.values(): - try: - self.history(item) - result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) - self.execution_results[item["arn"]] = result - self._save(f"executions/{item['marker']}.json", result) - except Exception as error: - errors.append(str(error)) - for key, arn in self.manifest["functions"].items() if full_run else []: - try: - config = self.lam.get_function_configuration(FunctionName=arn) - logs = [] - for page in self.logs.get_paginator("filter_log_events").paginate( - logGroupName=config["LoggingConfig"]["LogGroup"], - startTime=int(self.manifest["created"] * 1000), - ): - logs.extend(page.get("events", [])) - self._save(f"logs/{key}.json", logs) - except Exception as error: - errors.append(str(error)) - if errors: - self._save("collection-errors.json", errors) - raise CollectionError("; ".join(errors)) - - def release_all(self): - for gate in sorted( - self.gates - self.released_gates, key=lambda value: value != "release-all" - ): - self.release(gate) - self.gates.clear() - - def run_invocations(self): - """Recover only this run's invocations, including failed Invoke responses.""" - events = self.refresh(markers=[], validate_controls=False) - items = { - e["execution"]: {"arn": e["execution"], "marker": e["marker"]} - for e in events - } - for path in (self.artifacts_dir / "invocations").glob("*.json"): - record = json.loads(path.read_text()) - if ( - record.get("arn") - and record.get("payload", {}).get("run") == self.manifest["run"] - ): - items[record["arn"]] = { - "arn": record["arn"], - "marker": record["payload"]["marker"], - } - targets = tuple( - arn + "/durable-execution/" for arn in self.manifest["functions"].values() - ) - if any(not item["arn"].startswith(targets) for item in items.values()): - raise CollectionError( - "Refusing to stop an execution outside this deployment" - ) - return list(items.values()) - - def settle_case(self): - """Release fault work and drain this case before reusing its function.""" - self.release_all() - pending = [] - for item in self.invocations: - result = self.lam.get_durable_execution(DurableExecutionArn=item["arn"]) - if result["Status"] == "RUNNING": - try: - self.lam.stop_durable_execution(DurableExecutionArn=item["arn"]) - pending.append(item) - except ClientError as error: - if ( - error.response["Error"]["Code"] - != "InvalidParameterValueException" - ): - raise - # Completion can race the stop request. A still-running - # execution must not be ignored or leak into the next case. - if ( - self.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ - "Status" - ] - == "RUNNING" - ): - raise - - def settled(): - pending[:] = [ - item - for item in pending - if self.lam.get_durable_execution(DurableExecutionArn=item["arn"])[ - "Status" - ] - == "RUNNING" - ] - if pending: - return False - events = self.refresh(validate_controls=False) - entered = {e["request"] for e in select(events, "WRAPPER_ENTER")} - exited = { - e["request"] - for e in events - if e["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - } - return entered <= exited - - if self.invocations: - self.poll( - settled, - seconds=30, - message="Released case still owns runtime workers; shared function is not ready for reuse", - ) diff --git a/lmi_tests/deploy.py b/lmi_tests/deploy.py index 2da90e2c5..95d166b79 100644 --- a/lmi_tests/deploy.py +++ b/lmi_tests/deploy.py @@ -1,84 +1,42 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Build, update, verify, and exercise persistent LMI test resources.""" +"""Build the Python fixture and delegate deployment to the shared LMI harness.""" import argparse -import base64 import hashlib import json import os from pathlib import Path -import re import shutil import subprocess import sys -import time import zipfile -import boto3 -from botocore.config import Config -from botocore.exceptions import ClientError - -from lmi_tests.evidence import ProvisioningError - +from aws_durable_execution_conformance_tests.lmi.artifacts import ( + clear_run_artifacts, + save, +) +from aws_durable_execution_conformance_tests.lmi.deployment import DeploymentManager +from aws_durable_execution_conformance_tests.lmi.errors import ProvisioningError +from lmi_tests.runtime import RUNTIME ROOT = Path(__file__).resolve().parents[1] ARTIFACTS = ROOT / "lmi_tests/artifacts" -OWNER = "python-sdk-lmi-e2e" -SCALING = {"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1} -QUALIFIER = "$LATEST.PUBLISHED" -CONCURRENCIES = {"c1": 1, "c2": 2} DEFAULT_STACK = "python-lmi-e2e" -def client(service, region=None): - return boto3.client( - service, - region_name=region or os.environ["AWS_REGION"], - config=Config( - connect_timeout=3, read_timeout=10, retries={"total_max_attempts": 2} - ), +def manager(): + return DeploymentManager( + runtime=RUNTIME, + owner="python-sdk-lmi-e2e", + artifacts=ARTIFACTS, + bundle=ROOT / "lmi_tests/build/function.zip", + bucket_prefix="py-lmi-e2e", ) -def scrub(value): - if isinstance(value, dict): - return { - k: "" if k in {"CheckpointToken", "CallbackId"} else scrub(v) - for k, v in value.items() - } - if isinstance(value, list): - return [scrub(v) for v in value] - return value - - -def save(path, data): - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(scrub(data), indent=2, default=str) + "\n") - - -def clear_run_artifacts() -> None: - """Invalidate prior run evidence before fallible build/deployment work. - - Keep the build manifest, which deploy needs to verify the packaged wheel. - Remove the run manifest first: even if a later local deletion fails, an exit - trap cannot use that prior run's identity to collect, release, or stop work. - """ - if not ARTIFACTS.exists(): - return - (ARTIFACTS / "manifest.json").unlink(missing_ok=True) - for path in ARTIFACTS.iterdir(): - if path.name == "build.json": - continue - if path.is_dir() and not path.is_symlink(): - shutil.rmtree(path) - else: - path.unlink() - - def build(): - clear_run_artifacts() + clear_run_artifacts(ARTIFACTS) directory = ROOT / "lmi_tests/build" if directory.exists(): shutil.rmtree(directory) @@ -163,459 +121,6 @@ def build(): ) -def template(manifest, functions=True): - bucket = manifest["bucket"] - resources = { - "Bucket": { - "Type": "AWS::S3::Bucket", - "Properties": { - "BucketName": bucket, - "PublicAccessBlockConfiguration": { - "BlockPublicAcls": True, - "IgnorePublicAcls": True, - "BlockPublicPolicy": True, - "RestrictPublicBuckets": True, - }, - "BucketEncryption": { - "ServerSideEncryptionConfiguration": [ - {"ServerSideEncryptionByDefault": {"SSEAlgorithm": "AES256"}} - ] - }, - "LifecycleConfiguration": { - "Rules": [ - { - "Id": "expire-run-data", - "Status": "Enabled", - "Prefix": "runs/", - "ExpirationInDays": 1, - "AbortIncompleteMultipartUpload": { - "DaysAfterInitiation": 1 - }, - } - ] - }, - }, - }, - "BucketPolicy": { - "Type": "AWS::S3::BucketPolicy", - "Properties": { - "Bucket": {"Ref": "Bucket"}, - "PolicyDocument": { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": {"AWS": manifest["role"]}, - "Action": ["s3:GetObject", "s3:PutObject"], - "Resource": [ - f"arn:aws:s3:::{bucket}/runs/*", - ], - }, - { - "Effect": "Deny", - "Principal": "*", - "Action": "s3:*", - "Resource": [ - f"arn:aws:s3:::{bucket}", - f"arn:aws:s3:::{bucket}/*", - ], - "Condition": {"Bool": {"aws:SecureTransport": "false"}}, - }, - ], - }, - }, - }, - } - outputs = {} - if functions: - for key, concurrency in manifest["concurrencies"].items(): - name = manifest["stack"] + "-" + key - resources[key + "Logs"] = { - "Type": "AWS::Logs::LogGroup", - "Properties": { - "LogGroupName": "/aws/lambda/" + name, - "RetentionInDays": 1, - }, - } - resources[key + "Function"] = { - "Type": "AWS::Lambda::Function", - "DependsOn": "BucketPolicy", - "Properties": { - "FunctionName": name, - "Runtime": manifest["runtime"], - "Architectures": ["arm64"], - "Handler": "lmi_tests.fixture.handler", - "Role": manifest["role"], - "MemorySize": 2048, - "Timeout": manifest["invocationTimeout"], - "Code": {"S3Bucket": bucket, "S3Key": manifest["codeKey"]}, - "FunctionScalingConfig": SCALING, - "DurableConfig": { - "ExecutionTimeout": manifest["executionTimeout"], - "RetentionPeriodInDays": 1, - }, - "CapacityProviderConfig": { - "LambdaManagedInstancesCapacityProviderConfig": { - "CapacityProviderArn": manifest["provider"], - "PerExecutionEnvironmentMaxConcurrency": concurrency, - "ExecutionEnvironmentMemoryGiBPerVCpu": 2, - } - }, - "Environment": { - "Variables": { - "LMI_BUCKET": bucket, - "LMI_COMMIT": manifest["commit"], - "LMI_RUN_ID": manifest["run"], - "AWS_RETRY_MODE": "standard", - } - }, - "LoggingConfig": { - "LogFormat": "JSON", - "ApplicationLogLevel": "INFO", - "SystemLogLevel": "INFO", - "LogGroup": {"Ref": key + "Logs"}, - }, - }, - } - # LMI publishes this qualified version automatically, including its - # scaling settings. An extra numbered version would allocate more capacity. - outputs[key] = { - "Value": { - "Fn::Join": [ - "", - [{"Fn::GetAtt": [key + "Function", "Arn"]}, ":" + QUALIFIER], - ] - } - } - return { - "AWSTemplateFormatVersion": "2010-09-09", - "Resources": resources, - "Outputs": outputs, - } - - -def wait_stack(cfn, name, expected, seconds=1500): - deadline = time.monotonic() + seconds - while time.monotonic() < deadline: - try: - result = cfn.describe_stacks(StackName=name)["Stacks"][0] - except ClientError as error: - if expected == "DELETE_COMPLETE" and "does not exist" in str(error): - return None - raise - status = result["StackStatus"] - if status == expected: - return result - if "FAILED" in status or "ROLLBACK" in status: - save( - ARTIFACTS / "stack-events.json", - cfn.describe_stack_events(StackName=name), - ) - raise ProvisioningError(f"{name}: {status}; no fallback to standard Lambda") - time.sleep(3) - raise ProvisioningError(f"{name}: provisioning deadline exceeded; stack retained") - - -def verify(config, scaling, manifest, key): - actual = config.get("CapacityProviderConfig", {}).get( - "LambdaManagedInstancesCapacityProviderConfig", {} - ) - checks = [ - actual.get("CapacityProviderArn") == manifest["provider"], - actual.get("PerExecutionEnvironmentMaxConcurrency") - == manifest["concurrencies"][key], - config.get("Runtime") == manifest["runtime"], - config.get("Role") == manifest["role"], - config.get("Handler") == "lmi_tests.fixture.handler", - not config.get("Layers", []), - config.get("Environment", {}).get("Variables", {}).get("LMI_BUCKET") - == manifest["bucket"], - config.get("Architectures") == ["arm64"], - config.get("Version") == QUALIFIER, - config.get("CodeSha256") == manifest["codeSha256"], - config.get("MemorySize") == 2048, - config.get("State") == "Active", - config.get("DurableConfig", {}).get("ExecutionTimeout") - == manifest["executionTimeout"], - config.get("Timeout") == manifest["invocationTimeout"], - scaling.get("AppliedFunctionScalingConfig") == SCALING, - config.get("Environment", {}).get("Variables", {}).get("LMI_COMMIT") - == manifest["commit"], - config.get("Environment", {}).get("Variables", {}).get("LMI_RUN_ID") - == manifest["run"], - ] - if not all(checks): - raise ProvisioningError( - f"{key}: LMI configuration/artifact/scaling readback mismatch" - ) - - -def deploy(args): - clear_run_artifacts() - if not args.run_id or not re.fullmatch(r"[a-z0-9-]{1,24}", args.run_id): - raise ProvisioningError( - "run-id must be 1-24 lowercase letters, digits or hyphens" - ) - if not re.fullmatch(r"[a-z][a-z0-9-]{0,49}", args.stack_name): - raise ProvisioningError( - "stack-name must be 1-50 lowercase letters, digits or hyphens, starting with a letter" - ) - provider = os.environ["CAPACITY_PROVIDER_ARN"] - account = client("sts").get_caller_identity()["Account"] - if account != os.environ["TEST_ACCOUNT_ID"] or provider.split(":")[3:5] != [ - os.environ["AWS_REGION"], - account, - ]: - raise ProvisioningError( - "Authenticated account/region must match the test account and capacity provider" - ) - capacity = client("lambda").get_capacity_provider( - CapacityProviderName=provider.rsplit(":", 1)[-1].split("/")[-1] - ) - save(ARTIFACTS / "capacity-provider.json", capacity) - provider_config = capacity["CapacityProvider"] - if ( - not 2 - <= provider_config.get("CapacityProviderScalingConfig", {}).get( - "MaxVCpuCount", 0 - ) - <= 128 - ): - raise ProvisioningError( - "Use a bounded, test-owned capacity provider (2-128 maximum vCPUs)" - ) - data = (ROOT / "lmi_tests/build/function.zip").read_bytes() - digest = hashlib.sha256(data).digest() - built = json.loads((ARTIFACTS / "build.json").read_text()) - if built["sha256"] != digest.hex(): - raise ProvisioningError("Artifact changed since build") - name = args.stack_name - bucket_scope = hashlib.sha256( - f"{os.environ['AWS_REGION']}:{name}".encode() - ).hexdigest()[:12] - manifest = { - "run": args.run_id, - "stack": name, - "bucket": f"py-lmi-e2e-{account}-{bucket_scope}", - "persistent": True, - "account": account, - "region": os.environ["AWS_REGION"], - "provider": provider, - "role": os.environ["TEST_LAMBDA_EXECUTION_ROLE_ARN"], - "runtime": args.runtime, - "concurrencies": CONCURRENCIES, - "commit": built["commit"], - "wheels": built.get("wheels", {}), - "dependencies": built.get("dependencies", []), - "codeKey": "code/" + digest.hex() + ".zip", - "codeSha256": base64.b64encode(digest).decode(), - "invocationTimeout": 60, - "executionTimeout": 240, - "driverTimeout": 120, - "cleanupGrace": 5, - "created": time.time(), - "functions": {}, - } - save(ARTIFACTS / "manifest.json", manifest) - deploy_stack(manifest, data) - - -def deploy_stack(manifest, data): - """Bootstrap the bucket once, then update the same two functions in place.""" - cfn = client("cloudformation") - tags = [ - {"Key": "Suite", "Value": OWNER}, - {"Key": "Stack", "Value": manifest["stack"]}, - {"Key": "Persistent", "Value": "true"}, - ] - stack = owned_stack(cfn, manifest) - if stack is None: - cfn.create_stack( - StackName=manifest["stack"], - TemplateBody=json.dumps(template(manifest, False)), - Tags=tags, - ) - manifest["owned"] = True - save(ARTIFACTS / "manifest.json", manifest) - wait_stack(cfn, manifest["stack"], "CREATE_COMPLETE", 180) - else: - if stack["StackStatus"] not in { - "CREATE_COMPLETE", - "UPDATE_COMPLETE", - "UPDATE_ROLLBACK_COMPLETE", - }: - raise ProvisioningError( - f"Persistent stack requires recovery from {stack['StackStatus']}; it was retained" - ) - storage = client("s3") - storage.put_public_access_block( - Bucket=manifest["bucket"], - PublicAccessBlockConfiguration={ - "BlockPublicAcls": True, - "IgnorePublicAcls": True, - "BlockPublicPolicy": True, - "RestrictPublicBuckets": True, - }, - ) - protection = storage.get_public_access_block(Bucket=manifest["bucket"])[ - "PublicAccessBlockConfiguration" - ] - if not all( - protection.get(name) is True - for name in ( - "BlockPublicAcls", - "IgnorePublicAcls", - "BlockPublicPolicy", - "RestrictPublicBuckets", - ) - ): - raise ProvisioningError( - "Evidence bucket must be private before uploading the tested artifact" - ) - if stack is not None: - wait_for_idle_functions(stack) - manifest["owned"] = True - save(ARTIFACTS / "manifest.json", manifest) - client("s3").put_object( - Bucket=manifest["bucket"], Key=manifest["codeKey"], Body=data - ) - client("s3").put_object( - Bucket=manifest["bucket"], - Key=f"runs/{manifest['run']}/control/release-all", - Body=b"hold", - ) - spec = template(manifest) - save(ARTIFACTS / "template.json", spec) - try: - cfn.update_stack( - StackName=manifest["stack"], TemplateBody=json.dumps(spec), Tags=tags - ) - except ClientError as error: - if "No updates are to be performed" not in error.response["Error"].get( - "Message", "" - ): - raise - stack = owned_stack(cfn, manifest) - else: - stack = wait_stack(cfn, manifest["stack"], "UPDATE_COMPLETE") - if {output["OutputKey"] for output in stack.get("Outputs", [])} != set( - CONCURRENCIES - ): - raise ProvisioningError( - "The persistent stack must expose both shared LMI functions" - ) - for output in stack["Outputs"]: - key, arn = output["OutputKey"], output["OutputValue"] - manifest["functions"][key] = arn - save(ARTIFACTS / "manifest.json", manifest) - config = client("lambda").get_function_configuration(FunctionName=arn) - scaling = client("lambda").get_function_scaling_config( - FunctionName=arn.rsplit(":", 1)[0], Qualifier=QUALIFIER - ) - save( - ARTIFACTS / f"configuration/{key}.json", - {"function": config, "scaling": scaling}, - ) - verify(config, scaling, manifest, key) - save( - ARTIFACTS / "function-name-map.json", - { - key: {"lmi_tests.fixture.handler": arn} - for key, arn in manifest["functions"].items() - }, - ) - - -def owned_stack(cfn, manifest): - try: - stack = cfn.describe_stacks(StackName=manifest["stack"])["Stacks"][0] - except ClientError as error: - if error.response["Error"][ - "Code" - ] == "ValidationError" and "does not exist" in str(error): - return None - raise - tags = {t["Key"]: t["Value"] for t in stack.get("Tags", [])} - if ( - tags.get("Suite") != OWNER - or tags.get("Stack") != manifest["stack"] - or tags.get("Persistent") != "true" - ): - raise ProvisioningError( - "Refusing to modify a stack not owned by this persistent suite" - ) - return stack - - -def wait_for_idle_functions(stack, seconds=270): - """Wait for prior runs without stopping them or updating active functions.""" - lam = client("lambda") - deadline = time.monotonic() + seconds - while True: - running = [] - for output in stack.get("Outputs", []): - if output["OutputKey"] not in CONCURRENCIES: - raise ProvisioningError( - "Unexpected function in persistent stack outputs" - ) - for page in lam.get_paginator( - "list_durable_executions_by_function" - ).paginate( - FunctionName=output["OutputValue"].rsplit(":", 1)[0], - Statuses=["RUNNING"], - ): - running.extend(page.get("DurableExecutions", [])) - if page.get("NextMarker") and time.monotonic() >= deadline: - raise ProvisioningError( - "Could not finish checking previous executions; code was not updated" - ) - save(ARTIFACTS / "pre-deploy-executions.json", running) - if not running: - return - if time.monotonic() >= deadline: - raise ProvisioningError( - "Previous durable executions are still running; persistent code was not updated" - ) - time.sleep(2) - - -def cleanup(manifest): - """Release only this run's work; retain functions, stack, bucket, and code.""" - if not manifest.get("owned"): - return - cfn = client("cloudformation") - if owned_stack(cfn, manifest) is None: - return - resources = cfn.describe_stack_resources(StackName=manifest["stack"])[ - "StackResources" - ] - if any( - r["ResourceType"] == "AWS::S3::Bucket" - and r.get("PhysicalResourceId") == manifest["bucket"] - and r["ResourceStatus"] != "DELETE_COMPLETE" - for r in resources - ): - client("s3").put_object( - Bucket=manifest["bucket"], - Key=f"runs/{manifest['run']}/control/release-all", - Body=b"release", - ) - from lmi_tests.cloud import Cloud - - driver = Cloud(manifest) - driver.invocations = driver.run_invocations() - driver.settle_case() - save( - ARTIFACTS / "cleanup.json", - { - "stack": manifest["stack"], - "run": manifest["run"], - "status": "RETAINED", - "message": "Run work released; persistent functions, bucket, and code retained", - }, - ) - - def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("command", choices=["build", "deploy", "collect", "cleanup"]) @@ -627,11 +132,11 @@ def main(): if args.command == "build": build() elif args.command == "deploy": - deploy(args) + manager().deploy(args) else: manifest = json.loads((ARTIFACTS / "manifest.json").read_text()) if args.command == "cleanup": - cleanup(manifest) + manager().cleanup(manifest) else: from lmi_tests.cloud import Cloud diff --git a/lmi_tests/evidence.py b/lmi_tests/evidence.py deleted file mode 100644 index 6439741fd..000000000 --- a/lmi_tests/evidence.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Assertions on service and external evidence, including negative controls.""" - - -class ProvisioningError(RuntimeError): - """Unsupported configuration, unavailable capacity, or failed deployment.""" - - -class PlacementError(RuntimeError): - """The required runtime placement was not established; never a test pass.""" - - -class CollectionError(RuntimeError): - """Evidence is missing/unavailable; distinct from an SDK regression assertion.""" - - -def select(events, phase, marker=None): - return [ - e - for e in events - if e["phase"] == phase and (marker is None or e["marker"] == marker) - ] - - -def check_controls(events): - errors = select(events, "CONTROL_ERROR") - if errors: - error = errors[0] - raise CollectionError(f"Control {error['gate']} failed: {error['error']}") - - -def overlap(events, markers, count, environment=None): - blocked = [e for e in select(events, "BLOCKED") if e["marker"] in markers] - for start in blocked: - env = start["environment"] - if environment is not None and env != environment: - continue - active = [] - for candidate in blocked: - if candidate["environment"] != env or candidate["time"] > start["time"]: - continue - alive = [ - e - for e in events - if e["phase"] in {"ALIVE", "EFFECT"} - and all( - e.get(k) == candidate.get(k) - for k in ("request", "environment", "process", "gate") - ) - and e["time"] >= candidate["time"] - ] - # Missing exit records (e.g. a retired process) do not prove liveness. - if not alive or max(e["time"] for e in alive) < start["time"]: - continue - exits = [ - e - for e in select(events, "IO_EXIT") - if e["request"] == candidate["request"] - and e.get("gate") == candidate.get("gate") - and e["time"] >= candidate["time"] - ] - if not exits or min(e["time"] for e in exits) > start["time"]: - active.append(candidate) - if ( - len({e["request"] for e in active}) >= count - and len({e["process"] for e in active}) >= count - and len({e["pid"] for e in active}) >= count - and len({e["execution"] for e in active}) >= count - ): - return env - raise PlacementError(f"No {count}-process overlap in one execution environment") - - # A runtime-supported worker retirement can have no final diagnostic. Recovery - # and full-capacity placement must therefore also be asserted by the driver. diff --git a/lmi_tests/runtime.py b/lmi_tests/runtime.py new file mode 100644 index 000000000..019d47b59 --- /dev/null +++ b/lmi_tests/runtime.py @@ -0,0 +1,7 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Python fixture configuration for the shared LMI harness.""" + +from aws_durable_execution_conformance_tests.lmi.runtime import RuntimeDefinition + +RUNTIME = RuntimeDefinition("python", "lmi_tests.fixture.handler") diff --git a/lmi_tests/shared-suite-design.md b/lmi_tests/shared-suite-design.md index c4fe3f888..c8a7c9d6b 100644 --- a/lmi_tests/shared-suite-design.md +++ b/lmi_tests/shared-suite-design.md @@ -6,8 +6,11 @@ Normal return is requirement 2; normal failure is requirement 7. There is no Python-owned replacement catalog, OTel profile or exporter matrix in this PR. The shared repository owns requirement descriptions, roles, minimum native -concurrency, scheduling, behavior validators and strict result aggregation. The -SDK repository maps each scenario to its real public API fixture and retains +concurrency, scheduling, behavior validators, AWS transport, deployment, scenario +driving, evidence normalization and strict result aggregation. It also owns the +reusable `lmi-lifecycle.yml` workflow, called at the same pinned SHA as the shared +package. The SDK repository keeps `.github/scripts/lmi-ci.sh` for Python toolchain +and stage commands, and maps each scenario to its real public API fixture and retains source observations. `make_manifest` reads the shared matrix, uses c1 for single-worker cases and c2 for overlap/recovery, and preserves every required role. diff --git a/lmi_tests/shared.py b/lmi_tests/shared.py index c4e838db5..f717f0b1b 100644 --- a/lmi_tests/shared.py +++ b/lmi_tests/shared.py @@ -1,25 +1,19 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Run the seven shared LMI lifecycle requirements.""" +"""Bootstrap the pinned shared harness and run it with the Python adapter.""" from __future__ import annotations - import argparse -import asyncio -import hashlib -from copy import deepcopy import json import os from pathlib import Path import subprocess import sys from typing import Any, Mapping +from lmi_tests.shared_contract import CONFORMANCE_REVISION -from lmi_tests.deploy import ARTIFACTS, ROOT, client, save -from lmi_tests.shared_contract import CONFORMANCE_REVISION, configuration_identity -from lmi_tests.evidence import CollectionError - - +ROOT = Path(__file__).resolve().parents[1] +ARTIFACTS = ROOT / "lmi_tests/artifacts" COVERAGE_PROFILES = ("standard",) @@ -68,264 +62,46 @@ def verify_install(checkout: Path) -> None: def make_manifest( deployment: Mapping[str, Any], coverage: str, directory: Path ) -> dict[str, Any]: - from aws_durable_execution_conformance_tests.lmi.matrix import required_cases + from aws_durable_execution_conformance_tests.lmi.runner import ( + make_manifest as shared_manifest, + ) + from lmi_tests.shared_contract import configuration_identity - if coverage not in COVERAGE_PROFILES: - raise ValueError("Unknown Python adapter coverage profile") - actual = deepcopy(dict(deployment)) - actual["conformanceRevision"] = CONFORMANCE_REVISION core_wheels = { name: digest - for name, digest in actual["wheels"].items() + for name, digest in deployment["wheels"].items() if name.startswith("aws_durable_execution_sdk_python-") } if len(core_wheels) != 1: raise ValueError("Deployment does not retain an exact SDK wheel identity") - artifact = "sha256:" + next(iter(core_wheels.values())) - profiles = [ - { - "profile_id": key, - "resource_id": actual["functions"][key], - "native_concurrency": actual["concurrencies"][key], - "capacity": actual["concurrencies"][key], - "artifact_id": artifact, - "configuration_id": configuration_identity(actual, key), - "recovery_policy": "reuse-worker", - } - for key in ("c1", "c2") - ] - scenarios = [] - for required in required_cases("python", "lifecycle", coverage): - concurrency = required.minimum_native_concurrency - case_id = f"{required.requirement_id}-{required.scenario}" - scenarios.append( - { - "case_id": case_id, - "requirement_id": required.requirement_id, - "profile_id": f"c{concurrency}", - "scenario": required.scenario, - "roles": list(required.roles), - "slots": concurrency, - "exclusive": required.exclusive, - "behavior_budgets": { - "observationGraceMs": 1000, - "admissionMs": 5000, - "cleanupMs": 5000, - "recoveryMs": 15000, - }, - } - ) - return { - "schemaVersion": 1, - "runId": actual["run"], - "runtime": "python", - "region": actual["region"], - "coverageProfile": coverage, - "capabilityProfile": "lifecycle", - "scenarioConcurrency": 8, - "profiles": profiles, - "scenarios": scenarios, - "budgets": { - "provisioning": 120, - "lease": 1800, - "placement": 120, - "execution": 240, - "collection": 180, - "release": 60, - "retirement": 180, - }, - "sdk": { - "deployment": actual, - "artifacts_dir": str(directory.resolve() / "source"), - }, - } + return shared_manifest( + deployment, + coverage, + directory, + runtime="python", + artifact_id="sha256:" + next(iter(core_wheels.values())), + conformance_revision=CONFORMANCE_REVISION, + configuration_identity=configuration_identity, + ) def run(deployment: Mapping[str, Any], directory: Path) -> int: + from aws_durable_execution_conformance_tests.lmi.runner import run as shared_run - from aws_durable_execution_conformance_tests.lmi.app import ( - build_report, - coverage_inventory, - live_results, - load_manifest, + return shared_run( + deployment, + directory, + manifest_factory=make_manifest, + adapter_name="lmi_tests.shared_adapter:create_adapter", ) - from aws_durable_execution_conformance_tests.reporters import ( - render_console, - write_report, - ) - - directory.mkdir(parents=True, exist_ok=True) - if any(directory.iterdir()): - raise ValueError( - "Shared run output must be empty; preserve prior attempts and failure evidence" - ) - exit_code = 0 - for coverage in COVERAGE_PROFILES: - output = directory / coverage - output.mkdir() - plan = make_manifest(deployment, coverage, output) - path = output / "manifest.json" - save(path, plan) - validated = load_manifest(path) - results = asyncio.run( - live_results( - validated, - "lmi_tests.shared_adapter:create_adapter", - audit_dir=output, - ) - ) - report = build_report(validated, results) - save(output / "evidence.json", results) - save(output / "coverage.json", coverage_inventory(validated)) - print(render_console(report)) - for format_name in ("json", "junit"): - write_report(report, format_name, str(output / "report")) - exit_code = max(exit_code, report.exit_code()) - return exit_code def cleanup(deployment: Mapping[str, Any], directory: Path) -> int: - """Release this run's controls and request stops without altering its reports. - - GitHub Actions holds the cloud job's concurrency slot through this step. - Successful requests do not imply that every runtime task has retired. - """ - plans = [] - for coverage in COVERAGE_PROFILES: - path = directory / coverage / "manifest.json" - if not path.exists(): - continue - plan = json.loads(path.read_text()) - if plan.get("runId") != deployment["run"]: - raise CollectionError("Cleanup manifest belongs to another run") - profiles = {profile["profile_id"]: profile for profile in plan["profiles"]} - for case in plan["scenarios"]: - target = profiles[case["profile_id"]]["resource_id"] - if target not in deployment["functions"].values(): - raise CollectionError("Cleanup target is outside this deployment") - plans.append((coverage, plan)) - if not plans: - save( - directory / "cleanup.json", - { - "runId": deployment["run"], - "status": "NO_CASES", - "errors": [], - "reason": "No shared case manifests were recorded; no cleanup requests were sent", - }, - ) - return 0 - storage, service = ( - client("s3", deployment["region"]), - client("lambda", deployment["region"]), - ) - errors: list[str] = [] - for coverage, plan in plans: - profiles = {profile["profile_id"]: profile for profile in plan["profiles"]} - for case in plan["scenarios"]: - target = profiles[case["profile_id"]]["resource_id"] - if target not in deployment["functions"].values(): - raise CollectionError("Cleanup target is outside the owned deployment") - owner = hashlib.sha256(case["case_id"].encode()).hexdigest()[:20] - prefix = f"runs/{deployment['run']}/cases/{owner}/control/" - case_dir = directory / coverage / "source" / "cases" / owner - try: - objects = [ - item["Key"] - for page in storage.get_paginator("list_objects_v2").paginate( - Bucket=deployment["bucket"], Prefix=prefix - ) - for item in page.get("Contents", []) - ] - if any(not key.startswith(prefix) for key in objects): - raise CollectionError( - "Control listing escaped the case-owned prefix" - ) - executions: set[str] = set() - for path in (case_dir / "invocations").glob("*.json"): - invocation = json.loads(path.read_text()) - payload = invocation.get("payload", {}) - if ( - payload.get("run") != deployment["run"] - or payload.get("caseId") != case["case_id"] - ): - raise CollectionError( - "Invocation intent belongs to another case" - ) - if invocation.get("arn"): - executions.add(invocation["arn"]) - else: - recovered_executions: set[str] = set() - marker = payload["marker"] - event_prefix = f"runs/{deployment['run']}/events/{marker}/" - for page in storage.get_paginator("list_objects_v2").paginate( - Bucket=deployment["bucket"], Prefix=event_prefix - ): - for obj in page.get("Contents", []): - with storage.get_object( - Bucket=deployment["bucket"], Key=obj["Key"] - )["Body"] as stream: - event = json.loads(stream.read()) - if ( - event.get("run") != deployment["run"] - or event.get("caseId") != case["case_id"] - ): - raise CollectionError( - "Recovered invocation evidence belongs to another case" - ) - recovered_executions.add(event["execution"]) - executions.update(recovered_executions) - if not recovered_executions: - errors.append( - f"Unreconciled Invoke: {case['case_id']}/{marker}" - ) - if any( - not arn.startswith(target + "/durable-execution/") - for arn in executions - ): - raise CollectionError("Refusing to stop a foreign execution") - for key in sorted( - objects, key=lambda value: value != prefix + "release-all" - ): - with storage.get_object(Bucket=deployment["bucket"], Key=key)[ - "Body" - ] as stream: - state = stream.read() - if state not in {b"hold", b"release"}: - raise CollectionError( - "Unknown control content; refusing to overwrite it" - ) - if state == b"hold": - storage.put_object( - Bucket=deployment["bucket"], Key=key, Body=b"release" - ) - for arn in executions: - if ( - service.get_durable_execution(DurableExecutionArn=arn)["Status"] - == "RUNNING" - ): - try: - service.stop_durable_execution(DurableExecutionArn=arn) - except Exception: - if ( - service.get_durable_execution(DurableExecutionArn=arn)[ - "Status" - ] - == "RUNNING" - ): - raise - except Exception as error: - errors.append(f"{case['case_id']}: {type(error).__name__}: {error}") - save( - directory / "cleanup.json", - { - "runId": deployment["run"], - "errors": errors, - "status": "FAILED" if errors else "CLEANUP_REQUESTED", - "reason": "Run-scoped control release and execution-stop requests; case reports retain task retirement findings", - }, + from aws_durable_execution_conformance_tests.lmi.runner import ( + cleanup as shared_cleanup, ) - return int(bool(errors)) + + return shared_cleanup(deployment, directory) def main() -> None: @@ -344,6 +120,9 @@ def main() -> None: if args.command == "setup": setup(args.checkout) return + from aws_durable_execution_conformance_tests.lmi.artifacts import save + from aws_durable_execution_conformance_tests.lmi.errors import CollectionError + verify_install(args.checkout) deployment = json.loads(args.manifest.read_text()) expected_run = os.environ.get("LMI_SHARED_RUN_ID") diff --git a/lmi_tests/shared_adapter.py b/lmi_tests/shared_adapter.py index 88daf41a4..0da3220c9 100644 --- a/lmi_tests/shared_adapter.py +++ b/lmi_tests/shared_adapter.py @@ -1,243 +1,19 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""SDK-owned bridge to the pinned shared LMI scenario coordinator.""" +"""Compose the shared LMI adapter with Python fixture policies.""" -from __future__ import annotations - -import asyncio -from collections.abc import Callable, Mapping -from concurrent.futures import TimeoutError as FutureTimeout -from copy import deepcopy -import hashlib -from pathlib import Path -import time -from typing import Any, Protocol - -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - ScenarioContext, -) +from aws_durable_execution_conformance_tests.lmi.adapter import SharedAdapter from lmi_tests.cloud import Cloud -from lmi_tests.evidence import CollectionError, ProvisioningError -from lmi_tests.shared_scenarios import ScenarioDriver from lmi_tests.shared_contract import configuration_identity +from lmi_tests.shared_evidence import build_evidence +from lmi_tests.shared_scenarios import ScenarioDriver -class EvidenceBuilder(Protocol): - def __call__( - self, context: ScenarioContext, cloud: Cloud, *, phase: str - ) -> dict[str, Any]: ... - - -class _Case: - def __init__(self, driver: ScenarioDriver) -> None: - self.driver = driver - self.worker: asyncio.Task[Any] | None = None - - async def call(self, operation: Callable[[], Any]) -> Any: - if self.worker is not None and not self.worker.done(): - raise CollectionError( - "A previous case worker still owns blocking service calls" - ) - self.worker = asyncio.create_task(asyncio.to_thread(operation)) - try: - # Keep the worker task alive on outer cancellation so retirement can - # join the actual thread before issuing cleanup writes. - return await asyncio.shield(self.worker) - except asyncio.CancelledError: - self.driver.cloud.cancel() - raise - - async def prepare_retirement(self) -> None: - if self.worker is not None: - try: - await asyncio.shield(self.worker) - except Exception: - # The coordinator retains the original phase error; this only - # confirms its service-call worker has really stopped. - pass - finally: - if self.worker.done(): - self.worker = None - self.driver.cloud.begin_retirement() - - -class SharedAdapter: - """Each shared case has one Cloud, worker, control namespace and evidence sink.""" - - def __init__( - self, - manifest: Mapping[str, Any], - *, - evidence_builder: EvidenceBuilder, - cloud_factory: Callable[..., Cloud] = Cloud, - ) -> None: - self.manifest = deepcopy(dict(manifest)) - self.sdk = self.manifest["sdk"] - self.deployment = self.sdk["deployment"] - self.artifacts_dir = Path(self.sdk["artifacts_dir"]) - self.evidence_builder = evidence_builder - self.cloud_factory = cloud_factory - self.cases: dict[str, _Case] = {} - - async def prepare(self, profile: Profile) -> None: - def verify() -> None: - if profile.resource_id not in self.deployment["functions"].values(): - raise ProvisioningError( - "Shared profile is not part of the owned SDK deployment" - ) - key = next( - key - for key, arn in self.deployment["functions"].items() - if arn == profile.resource_id - ) - if self.deployment["concurrencies"][key] != profile.native_concurrency: - raise ProvisioningError( - "Native concurrency differs from the shared profile" - ) - if profile.configuration_id != configuration_identity(self.deployment, key): - raise ProvisioningError( - "Shared configuration identity differs from the deployed fixture" - ) - cloud = self.cloud_factory( - self.deployment, key, artifacts_dir=self.artifacts_dir / "configuration" - ) - cloud.verify() - - await asyncio.to_thread(verify) - - def _case(self, context: ScenarioContext) -> _Case: - try: - return self.cases[context.scenario.case_id] - except KeyError as error: - raise CollectionError( - "Case was not initialized by the shared adapter" - ) from error - - async def start(self, context: ScenarioContext) -> None: - if context.scenario.case_id in self.cases: - raise CollectionError("A later scenario cannot replace an admitted case") - key = next( - ( - key - for key, arn in self.deployment["functions"].items() - if arn == context.profile.resource_id - ), - None, - ) - if key is None: - raise ProvisioningError("Case target is outside the owned deployment") - owner = hashlib.sha256(context.scenario.case_id.encode()).hexdigest()[:20] - cloud = await asyncio.to_thread( - self.cloud_factory, - self.deployment, - key, - artifacts_dir=self.artifacts_dir / "cases" / owner, - case_context=context, - ) - loop = asyncio.get_running_loop() - - def capacity(operation: Callable[[], Any]) -> None: - cloud.check_active() - pending = asyncio.run_coroutine_threadsafe(operation(), loop) - try: - while True: - try: - pending.result(timeout=0.2) - return - except FutureTimeout: - cloud.check_active() - finally: - if not pending.done(): - pending.cancel() - - driver = ScenarioDriver( - context, - cloud, - release_capacity=lambda: capacity(context.release_capacity), - reserve_capacity=lambda: capacity(context.reserve_capacity), - ) - case = self.cases[context.scenario.case_id] = _Case(driver) - await case.call(driver.start) - - async def place(self, context: ScenarioContext) -> None: - case = self._case(context) - await case.call(case.driver.place) - - async def execute(self, context: ScenarioContext) -> None: - case = self._case(context) - await case.call(case.driver.execute) - - async def collect(self, context: ScenarioContext) -> dict[str, Any]: - case = self._case(context) - - def collect() -> dict[str, Any]: - case.driver.collect() - return self.evidence_builder( - context, case.driver.cloud, phase="observations" - ) - - return await case.call(collect) - - async def collect_partial(self, context: ScenarioContext) -> dict[str, Any]: - """Retain a failed phase's facts before release, without awaiting cleanup.""" - case = self._case(context) - - def capture() -> dict[str, Any]: - cloud = case.driver.cloud - read_error = None - try: - cloud.collect(full_run=False) - except Exception as error: - read_error = ( - f"Partial source collection: {type(error).__name__}: {error}" - ) - cloud.observation_ended_at = time.time() - partial = self.evidence_builder(context, cloud, phase="observations") - if read_error is not None: - partial.setdefault("collectionErrors", []).append(read_error) - partial.setdefault("collection", {})["watermarks"] = {} - cloud._save("shared-partial-evidence.json", partial) - return partial - - return await case.call(capture) - - async def release(self, context: ScenarioContext) -> None: - case = self.cases.get(context.scenario.case_id) - if case is None: - return - await case.prepare_retirement() - await case.call(case.driver.cloud.release_all) - - async def retire(self, context: ScenarioContext) -> bool: - case = self.cases.get(context.scenario.case_id) - if case is None: - return True - await case.prepare_retirement() - - def retire() -> bool: - complete = case.driver.retire() - final = self.evidence_builder(context, case.driver.cloud, phase="final") - # The coordinator freezes observation evidence before fault release. - # Only actual retirement facts may be appended by this phase. - context.evidence["retirement"] = final.get("retirement", {}) - # Fresh completion facts prove physical retirement separately; they - # must never replace the case's pre-release behavior observations. - from lmi_tests.shared_evidence import retirement_snapshot - - context.retirement_evidence = retirement_snapshot(final) - return complete - - return await case.call(retire) - - async def finish(self, profile: Profile, *, quiescent: bool) -> None: - # The shared runner retains per-case retirement results. Cross-run - # serialization ends with the GitHub Actions cloud job. - return None - - -def create_adapter(manifest: Mapping[str, Any]) -> SharedAdapter: - """Create an adapter for the cloud job serialized by GitHub Actions.""" - from lmi_tests.shared_evidence import build_evidence - - return SharedAdapter(manifest, evidence_builder=build_evidence) +def create_adapter(manifest): + return SharedAdapter( + manifest, + cloud_factory=Cloud, + driver_factory=ScenarioDriver, + evidence_builder=build_evidence, + configuration_identity=configuration_identity, + ) diff --git a/lmi_tests/shared_contract.py b/lmi_tests/shared_contract.py index 369debeee..5f0207c2c 100644 --- a/lmi_tests/shared_contract.py +++ b/lmi_tests/shared_contract.py @@ -4,37 +4,17 @@ from __future__ import annotations -import hashlib -import json from typing import Any, Mapping CONFORMANCE_REPOSITORY = "aws/aws-durable-execution-conformance-tests" # Updated only with the matching reviewed shared evidence contract. -CONFORMANCE_REVISION = "15ea7714de3c06b6552c4bd33fd036272a7e800b" +CONFORMANCE_REVISION = "56e7e444457653775579c120469d931b71c8cfd8" def configuration_identity(deployment: Mapping[str, Any], key: str) -> str: - fields = ( - "runtime", - "role", - "bucket", - "commit", - "run", - "provider", - "codeSha256", - "invocationTimeout", - "executionTimeout", - ) - configuration = {name: deployment[name] for name in fields} - configuration.update( - function=deployment["functions"][key], - nativeConcurrency=deployment["concurrencies"][key], - architecture="arm64", - memorySize=2048, - handler="lmi_tests.fixture.handler", - scaling={"MinExecutionEnvironments": 1, "MaxExecutionEnvironments": 1}, - ) - return ( - "sha256:" - + hashlib.sha256(json.dumps(configuration, sort_keys=True).encode()).hexdigest() + from aws_durable_execution_conformance_tests.lmi.configuration import ( + configuration_identity as identity, ) + from lmi_tests.runtime import RUNTIME + + return identity(deployment, key, RUNTIME) diff --git a/lmi_tests/shared_evidence.py b/lmi_tests/shared_evidence.py index 8c40ae82d..4b9a2a675 100644 --- a/lmi_tests/shared_evidence.py +++ b/lmi_tests/shared_evidence.py @@ -1,749 +1,47 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Normalize observed SDK/service evidence for the pinned shared validator. - -This module runs only in the driver. It never fills in a missing SDK boundary or -uses a fixture's expected answer as an observed result. Raw source objects remain -in the case artifact directory and S3 ledger. -""" - -from __future__ import annotations - -from collections import Counter, defaultdict -from collections.abc import Mapping -from dataclasses import dataclass -from datetime import datetime -import json -from typing import Any - -from aws_durable_execution_conformance_tests.lmi.coordinator import ScenarioContext -from lmi_tests.cloud import Cloud - - -TERMINAL = {"SUCCEEDED", "FAILED", "TIMED_OUT", "STOPPED"} -_ACTIVE_FAULTS = {"deadline-during-step", "peer-recovery"} -_OPERATION_TYPES = { - "Step": "STEP", - "Wait": "WAIT", - "Callback": "CALLBACK", - "Context": "CONTEXT", - "ChainedInvoke": "CHAINED_INVOKE", -} - - -def _milliseconds(value: Any) -> float: - if isinstance(value, datetime): - return value.timestamp() * 1000 - if isinstance(value, str): - return datetime.fromisoformat(value.replace("Z", "+00:00")).timestamp() * 1000 - if type(value) in (int, float): - return float(value) * 1000 - raise ValueError("Service evidence is missing its actual timestamp") - - -def _decoded(value: Any) -> Any: - if isinstance(value, Mapping) and "Payload" in value: - if value.get("Truncated") is True: - raise ValueError( - "Truncated service data cannot establish checkpoint equality" - ) - value = value["Payload"] - if isinstance(value, str): - try: - return json.loads(value) - except json.JSONDecodeError: - # Callback payloads may be plain UTF-8, rather than JSON results. - return value - return value - - -def _error(value: Any) -> dict[str, Any] | None: - if not isinstance(value, Mapping): - return None - if "Payload" in value: - value = value["Payload"] - if not isinstance(value, Mapping): - return None - return { - "type": value.get("ErrorType", value.get("type")), - "message": value.get("ErrorMessage", value.get("message")), - } - - -def _resources(value: Mapping[str, Any]) -> dict[str, Any]: - result: dict[str, Any] = {} - if "threads" in value: - result["threads"] = [ - {**thread, "id": str(thread["id"])} for thread in value["threads"] - ] - if "fds" in value: - result["fds"] = value["fds"] - if "rssBytes" in value or "rss" in value: - result["rssBytes"] = value.get("rssBytes", value.get("rss")) - return result - - -@dataclass(frozen=True) -class CaseSources: - context: ScenarioContext - cloud: Cloud - raw: tuple[Mapping[str, Any], ...] - - @classmethod - def capture(cls, context: ScenarioContext, cloud: Cloud) -> CaseSources: - return cls(context, cloud, tuple(cloud.events.values())) - - def observations(self) -> tuple[list[dict[str, Any]], list[str]]: - events: list[dict[str, Any]] = [] - errors: list[str] = [] - by_request: dict[str, list[int]] = defaultdict(list) - for raw in self.raw: - request = raw.get("request") - if ( - raw.get("run") != self.context.run_id - or raw.get("commit") != self.cloud.manifest["commit"] - or raw.get("caseId") != self.context.scenario.case_id - ): - errors.append( - "Raw evidence does not belong to the admitted run/artifact/case" - ) - continue - if not isinstance(request, str) or type(raw.get("sequence")) is not int: - errors.append( - "Raw evidence has no native request identity or source sequence" - ) - continue - by_request[request].append(raw["sequence"]) - observation = raw.get("observation") - if raw.get("phase") != "CONFORMANCE" or not isinstance( - observation, Mapping - ): - continue - event = { - "requestId": request, - "ownerRequestId": request, - "executionArn": raw["execution"], - **observation, - "sourceSequence": raw["sequence"], - "observedAtMs": raw["time"] * 1000, - } - if "resources" in event: - event["resources"] = _resources(event["resources"]) - if event.get("kind") == "wrapper_raise" and event.get("outcome") == "THREW": - event["sourceOutcome"] = "THREW" - event["outcome"] = ( - "FAILED" # Actual exception, not an inferred platform timeout. - ) - if event.get("kind") in { - "observation_error", - "control_error", - }: - errors.append( - str(event.get("message", event.get("error", event["kind"]))) - ) - events.append(event) - for request, sequences in by_request.items(): - if len(set(sequences)) != len(sequences) or sorted(sequences) != list( - range(1, max(sequences) + 1) - ): - errors.append(f"Raw source publication is incomplete for {request}") - return events, errors - - def operation_snapshots( - self, events: list[dict[str, Any]] - ) -> dict[str, dict[str, Mapping[str, Any]]]: - snapshots: dict[str, dict[str, Mapping[str, Any]]] = defaultdict(dict) - for event in events: - records = list(event.get("operations", [])) - if event.get("operationId") and event.get("operationType"): - records.append(event) - for record in records: - snapshots[event["executionArn"]][record["operationId"]] = record - return snapshots - - def histories( - self, events: list[dict[str, Any]] - ) -> tuple[list[dict[str, Any]], dict[str, Any], list[str]]: - snapshots = self.operation_snapshots(events) - histories: list[dict[str, Any]] = [] - expected: dict[str, Any] = {} - errors: list[str] = [] - for item in self.cloud.invocations: - arn = item.get("arn") - if ( - not arn - or arn not in self.cloud.raw_histories - or arn not in self.cloud.execution_results - ): - errors.append( - "An admitted execution is missing actual service history or state" - ) - continue - operations: dict[str, dict[str, Any]] = {} - for raw in self.cloud.raw_histories[arn]: - event_type = raw.get("EventType", "") - kind = next( - ( - value - for prefix, value in _OPERATION_TYPES.items() - if event_type.startswith(prefix) - ), - None, - ) - identity = raw.get("Id") - if kind is None or not isinstance(identity, str): - continue - info = snapshots.get(arn, {}).get(identity, {}) - operation = operations.setdefault( - identity, - { - "id": identity, - "type": kind, - "name": raw.get("Name", info.get("name")), - "parentId": raw.get("ParentId", info.get("parentId")), - "status": "STARTED", - }, - ) - if "Name" in raw: - operation["name"] = raw["Name"] - if "ParentId" in raw: - operation["parentId"] = raw["ParentId"] - details = raw.get(event_type + "Details", {}) - # Service operation lifetime is independent of native invocation - # and exporter timing. A retry cannot replace its original start. - if ( - event_type in {prefix + "Started" for prefix in _OPERATION_TYPES} - and raw.get("EventTimestamp") is not None - ): - started_at = _milliseconds(raw["EventTimestamp"]) - operation["startedAtMs"] = min( - operation.get("startedAtMs", started_at), started_at - ) - if ( - any( - event_type.endswith(suffix) - for suffix in ("Succeeded", "Failed", "TimedOut", "Stopped") - ) - and raw.get("EventTimestamp") is not None - ): - ended_at = _milliseconds(raw["EventTimestamp"]) - operation["endedAtMs"] = ended_at - if event_type.endswith("Succeeded"): - operation["status"] = "SUCCEEDED" - result = details.get("Result", info.get("serializedResult")) - if result is not None: - operation["result"] = _decoded(result) - elif event_type.endswith("Failed"): - operation["status"] = "FAILED" - operation["error"] = _error(details.get("Error")) or info.get( - "error" - ) - elif event_type.endswith("TimedOut"): - operation["status"] = "TIMED_OUT" - elif event_type.endswith("Stopped"): - operation["status"] = "STOPPED" - for operation in operations.values(): - operation.pop("startedAtMs", None) - operation.pop("endedAtMs", None) - result = self.cloud.execution_results[arn] - started = result.get("StartTimestamp") - if started is None: - started = next( - ( - event.get("EventTimestamp") - for event in self.cloud.raw_histories[arn] - if event.get("EventType") == "ExecutionStarted" - ), - None, - ) - history: dict[str, Any] = { - "executionArn": arn, - "ownerCaseId": self.context.scenario.case_id, - "status": result.get("Status"), - "operations": list(operations.values()), - } - if started is not None: - history["startedAtMs"] = _milliseconds(started) - if "Result" in result: - history["result"] = _decoded(result["Result"]) - if "Error" in result: - history["error"] = _error(result["Error"]) - histories.append(history) - fixture = item.get("payload", {}).get("scenario") - if fixture == "failure": - expected[arn] = { - "status": "FAILED", - "error": { - "type": "ValueError", - "message": "expected:" + item["marker"], - }, - } - elif ( - fixture == "deadline" - and self.context.scenario.scenario in _ACTIVE_FAULTS - ): - # This fixture asserts native deadline/retry behavior. Its logical - # execution may still be active or may finish after normal release. - expected[arn] = { - "allowedStatuses": [ - "RUNNING", - "SUCCEEDED", - "FAILED", - "TIMED_OUT", - "STOPPED", - ] - } - else: - expected[arn] = {"status": "SUCCEEDED", "result": item["marker"]} - return histories, expected, errors - - def attempts( - self, events: list[dict[str, Any]] - ) -> tuple[list[dict[str, Any]], dict[str, str], list[str]]: - attempts: list[dict[str, Any]] = [] - selected: dict[str, str] = {} - errors: list[str] = [] - journal = {item.get("requestId"): item for item in self.context.attempts} - raw_by_request: dict[str, Mapping[str, Any]] = {} - for raw in self.raw: - if raw.get("phase") == "WRAPPER_ENTER": - raw_by_request[raw["request"]] = raw - for request, raw in raw_by_request.items(): - item = next( - ( - item - for item in self.cloud.invocations - if item["marker"] == raw["marker"] - ), - None, - ) - if ( - item is None - or request not in journal - or request not in self.cloud.request_roles - ): - errors.append( - f"Native request {request} was not bound to an original launch" - ) - continue - owned = [event for event in events if event.get("requestId") == request] - boundaries = [ - event - for event in owned - if event.get("kind") in {"wrapper_return", "wrapper_raise"} - ] - deadline = [ - event for event in owned if event.get("kind") == "runtime_deadline" - ] - timeout = None - for service_event in self.cloud.raw_histories.get(raw["execution"], []): - details = service_event.get("InvocationCompletedDetails", {}) - message = json.dumps(details.get("Error", {})).lower().replace(" ", "") - if details.get("RequestId") == request and any( - word in message for word in ("timeout", "timedout") - ): - timeout = { - "kind": "platform_timeout", - "requestId": request, - "ownerRequestId": request, - "executionArn": raw["execution"], - "atMs": _milliseconds(service_event.get("EventTimestamp")), - "source": "platform", - "sourceEventId": service_event.get("EventId"), - } - if timeout is not None: - events.append(timeout) - role = self.cloud.request_roles[request] - attempt: dict[str, Any] = { - "requestId": request, - "executionArn": raw["execution"], - "environmentId": raw["environment"], - "processId": raw["process"], - "workerId": raw["process"], - "pid": raw.get("pid"), - "role": role, - "attempt": journal[request]["attempt"], - "marker": raw["marker"], - "fixture": item.get("payload", {}).get("scenario"), - "startedAtMs": min( - [raw["time"] * 1000, *(event["atMs"] for event in owned)] - ), - } - if boundaries: - attempt["outcome"] = boundaries[-1]["outcome"] - attempt["endedAtMs"] = boundaries[-1]["atMs"] - if timeout is not None: - attempt.update(outcome="TIMED_OUT", endedAtMs=timeout["atMs"]) - if deadline: - attempt["deadlineAtMs"] = deadline[0]["deadlineAtMs"] - if "outcome" not in attempt: - errors.append( - f"Native request {request} has no observed completion boundary" - ) - if journal[request]["attempt"] == 1: - selected[role] = request - attempts.append(attempt) - return attempts, selected, errors - - -def _causal_order(events: list[dict[str, Any]]) -> None: - requests = {event["requestId"] for event in events} - for request in requests: - owned = sorted( - (event for event in events if event["requestId"] == request), - key=lambda event: ( - event["atMs"], - event.get("captureSequence", event.get("sourceSequence", -1)), - ), - ) - for sequence, event in enumerate(owned, 1): - event["sequence"] = sequence - - -def _quiescent(request: str, events: list[dict[str, Any]]) -> bool: - local = [event for event in events if event.get("requestId") == request] - if not any( - event.get("kind") in {"wrapper_return", "wrapper_raise"} for event in local - ): - return False - if not any( - event.get("kind") == "task_exit" and event.get("taskId") == "root" - for event in local - ): - return False - for start, end, key in ( - ("task_start", "task_exit", "taskId"), - ("io_start", "io_exit", "taskId"), - ("checkpoint_call", "checkpoint_exit", "callId"), - ("poll_call", "poll_exit", "callId"), - ): - begun = Counter(event.get(key) for event in local if event.get("kind") == start) - ended = Counter(event.get(key) for event in local if event.get("kind") == end) - if begun - ended: - return False - return not any( - event.get("kind") in {"observation_error", "control_error"} for event in local - ) - - -def replay_evidence(events, histories, selected): - """Compare actual invocation-start checkpoints with final service history.""" - first_resume = next( - ( - event - for event in events - if event.get("kind") == "invocation_start" - and event.get("requestId") == selected.get("resume") - ), - {}, - ) - before = [] - for saved in first_resume.get("operations", []): - if saved.get("operationType") == "EXECUTION" or saved.get("status") not in { - "SUCCEEDED", - "FAILED", - }: - continue - operation = { - "id": saved["operationId"], - "name": saved.get("name"), - "type": saved["operationType"], - "parentId": saved.get("parentId"), - "status": saved["status"], - } - if saved["status"] == "SUCCEEDED": - operation["result"] = _decoded(saved.get("serializedResult")) - else: - operation["error"] = saved.get("error") - before.append(operation) - victim = next( - (e for e in events if e.get("requestId") == selected.get("victim")), {} - ) - after = next( - ( - h["operations"] - for h in histories - if h["executionArn"] == victim.get("executionArn") - ), - [], - ) - bodies = [e for e in events if e.get("kind") == "body"] - executions = [ - { - "requestId": e["requestId"], - "operationId": e["operationId"], - "attemptId": f"{e['requestId']}:{e['attempt']}", +"""Python expected outcomes and worker identity for shared evidence normalization.""" + +from aws_durable_execution_conformance_tests.lmi.source_evidence import ( + build_evidence as shared_build_evidence, +) + + +def expected_outcome(item, scenario): + fixture = item.get("payload", {}).get("scenario") + if fixture == "failure": + return { + "status": "FAILED", + "error": { + "type": "ValueError", + "message": "expected:" + item["marker"], + }, } - for e in bodies - ] - retry = [e for e in bodies if e.get("operation") == "retry"] - attempts = [] - for body in retry: - ending = next( - ( - e - for e in events - if e.get("kind") == "task_exit" - and e.get("operationId") == body["operationId"] - and e.get("requestId") == body["requestId"] - and e.get("attempt") == body["attempt"] - ), - {}, - ) - attempts.append( - { - "attemptId": f"{body['requestId']}:{body['attempt']}", - "requestId": body["requestId"], - "outcome": ending.get("outcome"), - } - ) - return { - "before": before, - "after": after, - "operationExecutions": executions, - "retry": { - "operationId": retry[0]["operationId"] if retry else "", - "attempts": attempts, - }, - "sideEffects": [ - { - "operationId": body["operationId"], - "count": sum(e["operationId"] == body["operationId"] for e in bodies), - "expectedCount": 2 if body.get("operation") == "retry" else 1, - } - for index, body in enumerate(bodies) - if body["operationId"] not in {e["operationId"] for e in bodies[:index]} - ], - } - - -def assertion_events(events, scenario, selected): - """Expose one scenario-owned task boundary while retaining every task fact. - - The shared behavior contract's task_exit is singular. Other real tasks stay - in the envelope with their source kind, and are restored for the separate - all-task retirement audit. - """ - if scenario not in { - "normal-return", - "normal-failure", + elif fixture == "deadline" and scenario in { "deadline-during-step", "peer-recovery", - "warm-reuse", }: - return events - result = [] - for event in events: - item = dict(event) - target = ( - "late-callback" - if scenario == "warm-reuse" - and event.get("requestId") == selected.get("victim") - else "root" - ) - if item["kind"] in {"task_start", "task_exit"} and item.get("taskId") != target: - item["sourceKind"] = item["kind"] - item["kind"] = "observed_" + item["kind"] - result.append(item) - return result - - -def retirement_snapshot(evidence): - """Restore every observed task for the common independent retirement audit.""" - from copy import deepcopy - - snapshot = deepcopy(evidence) - for event in snapshot.get("events", []): - if event.get("kind") in {"observed_task_start", "observed_task_exit"}: - event["kind"] = event["sourceKind"] - return snapshot - - -def build_evidence( - context: ScenarioContext, cloud: Cloud, *, phase: str = "observations" -) -> dict[str, Any]: - sources = CaseSources.capture(context, cloud) - events, errors = sources.observations() - histories, expected, history_errors = sources.histories(events) - attempts, selected, attempt_errors = sources.attempts(events) - errors.extend(history_errors + attempt_errors) - _causal_order(events) - # The final snapshot describes actual retirement collection, independently - # of a failed phase's earlier (possibly incomplete) observation window. - ended = ( - (cloud.retired_at if phase == "final" else cloud.observation_ended_at) or 0 - ) * 1000 - complete = bool(attempts) and all( - _quiescent(item["requestId"], events) for item in attempts - ) - if not complete: - errors.append( - "Actual wrapper/task publication has not established a complete observation window" - ) - latest = max((event["atMs"] for event in events), default=0) - if ended < latest: - errors.append("Collection predates a retained observation") - watermarks = ( - {name: ended for name in ("lifecycle", "sdkRequests", "history")} - if not errors - else {} - ) - artifact = { - "sdk": context.profile.artifact_id, - "conformance": cloud.manifest.get("conformanceRevision", "unverified"), - } - profile = { - "functionArn": context.profile.resource_id, - "configurationDigest": context.profile.configuration_id, - "nativeConcurrency": context.profile.native_concurrency, - "timeoutMs": cloud.manifest["invocationTimeout"] * 1000, - } - if context.profile.recovery_policy: - profile["recoveryPolicy"] = context.profile.recovery_policy - evidence: dict[str, Any] = { - "schemaVersion": 1, - "runId": context.run_id, - "caseId": context.scenario.case_id, - "coverageProfile": context.scenario.coverage_profile, - "scenario": context.scenario.scenario, - "requirementId": context.scenario.requirement_id, - "runtime": "python", - "artifact": artifact, - "profile": profile, - "attempts": attempts, - "selectedRequests": selected, - "events": assertion_events(events, context.scenario.scenario, selected), - "histories": histories, - "expectedHistories": expected, - "observationEndedAtMs": ended, - "sdkRequests": [], - "collection": { - "watermarks": watermarks, - "evidenceUris": sorted(cloud.source_uris), - }, - "collectionErrors": errors, - } - evidence["budgets"] = dict(context.scenario.behavior_budgets) - if context.scenario.scenario == "suspension-retry": - evidence["replay"] = replay_evidence(events, histories, selected) - for event in events: - if event.get("kind") in {"checkpoint_call", "poll_call"}: - evidence["sdkRequests"].append( - { - **event, - "method": "CheckpointDurableExecution" - if event["kind"] == "checkpoint_call" - else "GetDurableExecutionState", - "outcome": "DISPATCHED", - } - ) - created = [ - { - "kind": "invocation", - "id": item["requestId"], - "ownerCaseId": context.scenario.case_id, + # This fixture asserts native deadline/retry behavior. Its logical + # execution may still be active or may finish after normal release. + return { + "allowedStatuses": [ + "RUNNING", + "SUCCEEDED", + "FAILED", + "TIMED_OUT", + "STOPPED", + ] } - for item in attempts - ] - created.extend( - {"kind": "execution", "id": arn, "ownerCaseId": context.scenario.case_id} - for arn in dict.fromkeys(item["executionArn"] for item in attempts) + else: + return {"status": "SUCCEEDED", "result": item["marker"]} + + +def build_evidence(context, cloud, *, phase="observations"): + return shared_build_evidence( + context, + cloud, + runtime="python", + expected_outcome=expected_outcome, + worker_identity=lambda raw: raw["process"], + phase=phase, ) - evidence["resourcesCreated"] = created - # Coordination barriers are not injected victim faults. Retain their full - # journal separately; releasing a healthy peer is not evidence of victim cleanup. - evidence["controlRecords"] = list(cloud.control_records) - victim_faults = { - item.get("payload", {}).get("gate") - for item in cloud.invocations - if item.get("role") == "victim" - and item.get("payload", {}).get("scenario") == "deadline" - } - faults = [] - for gate in dict.fromkeys( - record["gate"] for record in cloud.control_records if record["action"] == "hold" - ): - if gate not in victim_faults: - continue - fault: dict[str, Any] = {"id": gate, "ownerCaseId": context.scenario.case_id} - receipts = [ - event["atMs"] - for event in events - if event.get("gateId") == gate - and event.get("kind") in {"fault_released"} - and event.get("mode") == "normal" - ] - releases = [ - record - for record in cloud.control_records - if record["gate"] == gate - and record["action"] == "release" - and record.get("state", "ACKNOWLEDGED") == "ACKNOWLEDGED" - ] - if receipts: - fault["releasedAtMs"] = min(receipts) - if releases: - release = releases[0] - requested = release.get("started", release["time"]) * 1000 - if not receipts or requested <= min(receipts): - fault.update( - releaseRequestedAtMs=requested, - releaseAcknowledgedAtMs=release["time"] * 1000, - ) - fault.setdefault("releasedAtMs", release["time"] * 1000) - faults.append(fault) - evidence["faultsCreated"] = faults - if phase == "final": - retired = (cloud.retired_at or 0) * 1000 - settled = [] - for resource in created: - item = { - **resource, - "settledAtMs": retired, - "evidenceUri": (cloud.artifacts_dir / "events.json").resolve().as_uri(), - } - if resource["kind"] == "invocation": - item["outcome"] = ( - "QUIESCENT" if _quiescent(resource["id"], events) else "UNKNOWN" - ) - elif resource["kind"] == "execution": - item["outcome"] = cloud.execution_results.get(resource["id"], {}).get( - "Status", "UNKNOWN" - ) - else: - completion = [ - event - for event in events - if event.get("kind") == resource["kind"] + "_completed" - and event.get(resource["kind"] + "Id") == resource["id"] - ] - item["outcome"] = ( - completion[0].get("outcome") if len(completion) == 1 else "UNKNOWN" - ) - settled.append(item) - evidence["retirement"] = { - "retiredAtMs": retired, - "resources": settled, - "faults": [ - { - "id": fault["id"], - "ownerCaseId": context.scenario.case_id, - "releasedAtMs": fault.get( - "releaseAcknowledgedAtMs", fault.get("releasedAtMs", 0) - ), - "mode": "emergency" - if any( - event.get("gateId") == fault["id"] - and event.get("kind") in {"fault_released"} - and event.get("mode") == "emergency" - for event in events - ) - else "normal", - } - for fault in faults - ], - } - if errors: - evidence["collection"]["watermarks"] = {} - cloud._save(f"shared-{phase}-evidence.json", evidence) - return evidence diff --git a/lmi_tests/shared_scenarios.py b/lmi_tests/shared_scenarios.py index 151305e7b..797b64902 100644 --- a/lmi_tests/shared_scenarios.py +++ b/lmi_tests/shared_scenarios.py @@ -1,19 +1,10 @@ # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. # SPDX-License-Identifier: Apache-2.0 -"""Drive the seven shared lifecycle scenarios through real SDK handlers.""" +"""Bind shared lifecycle choreography to Python fixture entry points.""" -from __future__ import annotations - -from collections import Counter -from collections.abc import Callable -import hashlib -import time -from typing import Any -import uuid - -from aws_durable_execution_conformance_tests.lmi.coordinator import ScenarioContext -from lmi_tests.cloud import Cloud -from lmi_tests.evidence import CollectionError, PlacementError, overlap, select +from aws_durable_execution_conformance_tests.lmi.scenarios import ( + ScenarioDriver as SharedScenarioDriver, +) FIXTURES = { "native-overlap": "barrier", @@ -26,376 +17,17 @@ } -def canonical(events: list[dict[str, Any]]) -> list[dict[str, Any]]: - return [ - { - "requestId": event.get("request"), - "ownerRequestId": event.get("request"), - "executionArn": event.get("execution"), - **event["observation"], - } - for event in events - if event.get("phase") == "CONFORMANCE" - and isinstance(event.get("observation"), dict) - ] - - -class ScenarioDriver: - def __init__( - self, - context: ScenarioContext, - cloud: Cloud, - *, - release_capacity: Callable[[], None] | None = None, - reserve_capacity: Callable[[], None] | None = None, - ) -> None: - self.context, self.cloud = context, cloud - self.items: dict[str, dict[str, Any]] = {} - self.release_capacity, self.reserve_capacity = ( - release_capacity, - reserve_capacity, - ) - self.boundary: dict[str, Any] | None = None - self.peer_boundary: dict[str, Any] | None = None - - @property - def scenario(self) -> str: - return self.context.scenario.scenario - - def seconds(self, budget: str, fallback: float) -> float: - return ( - self.context.scenario.behavior_budgets.get(budget, fallback * 1000) / 1000 - ) - - def gate(self, label: str) -> str: - owner = hashlib.sha256( - f"{self.context.run_id}:{self.context.scenario.case_id}".encode() - ).hexdigest()[:20] - return f"case-{owner}-{label}" - - def launch( - self, - role: str, - fixture: str, - *, - gate: str | None = None, - preparation_gate: str | None = None, - resume_roles: tuple[str, ...] = (), - target_environment: str | None = None, - ) -> dict[str, Any]: - if role in self.items: - raise CollectionError("A later launch cannot replace a selected original") - item = self.cloud.start( - fixture, - gate=gate, - preparation_gate=preparation_gate, - target_environment=target_environment, - role=role, - resume_roles=resume_roles, - dispatcher_payload=self.context.dispatcher(role), - marker=self.gate(role) + "-" + uuid.uuid4().hex[:12], - ) - self.items[role] = item - return item - - def start(self) -> None: - if self.scenario not in FIXTURES: - raise NotImplementedError(f"No Python fixture for {self.scenario}") - if self.scenario == "native-overlap": - for role in self.context.scenario.roles: - self.launch(role, "barrier", gate=self.gate("cohort")) - return - self.launch( - "victim", - FIXTURES[self.scenario], - gate=self.gate("fault") - if self.scenario in {"deadline-during-step", "peer-recovery"} - else None, - preparation_gate=self.gate("prepare") - if self.scenario == "peer-recovery" - else None, - resume_roles=("resume",) if self.scenario == "suspension-retry" else (), - ) - - def wait_overlap( - self, roles: tuple[str, ...], environment: str | None = None - ) -> list[dict[str, Any]]: - markers = {self.items[role]["marker"] for role in roles} - - def established(): - observed = [ - event for event in self.cloud.refresh() if event["marker"] in markers - ] - originals = {} - for role in roles: - entries = [ - event - for event in observed - if event["phase"] == "WRAPPER_ENTER" - and event["marker"] == self.items[role]["marker"] - ] - if not entries: - return None - entry = min( - entries, key=lambda event: (event["time"], event["sequence"]) - ) - originals[role] = entry["request"] - if time.time() >= entry["deadline"]: - raise PlacementError( - f"Original {role} invocation expired before overlap was established" - ) - events = [ - event for event in observed if event["request"] in originals.values() - ] - if any( - event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - for event in events - ): - raise PlacementError("An original cohort invocation already ended") - if environment is not None and any( - event["environment"] != environment - for event in select(events, "BLOCKED") - ): - raise PlacementError( - "Selected cohort reached a replacement environment" - ) - try: - overlap(events, markers, len(roles), environment) - except PlacementError: - return None - return events - - return self.cloud.poll( - established, - seconds=self.cloud.manifest.get("driverTimeout", 120), - category=PlacementError, - message="Original cohort placement/overlap was not established", - ) - - def place(self) -> None: - victim = self.items["victim"] - if self.scenario == "native-overlap": - self.wait_overlap(("victim", "peer")) - elif self.scenario in {"deadline-during-step", "peer-recovery"}: - self.boundary = self.cloud.phase(victim, "BLOCKED")[0] - if self.scenario == "peer-recovery": - self._prepare_recovery_peer() - else: - self.cloud.phase(victim, "WRAPPER_ENTER") - - def _prepare_recovery_peer(self) -> None: - if self.boundary is None: - raise CollectionError( - "Recovery preparation has no original victim boundary" - ) - deadline = self.boundary["deadline"] - lead = self.cloud.manifest["invocationTimeout"] / 2 - self.cloud.poll( - lambda: time.time() >= deadline - lead, - seconds=self.cloud.manifest["invocationTimeout"], - message="Original victim did not reach the peer-admission window", - ) - if time.time() >= deadline: - raise PlacementError("Original victim expired before peer admission") - peer = self.launch( - "peer", - "barrier", - gate=self.gate("peer"), - target_environment=self.boundary["environment"], - ) - self.peer_boundary = self.cloud.phase(peer, "BLOCKED")[0] - required_window = self.seconds("recoveryMs", 15) + self.seconds( - "observationGraceMs", 1 - ) - if self.peer_boundary["deadline"] <= deadline + required_window: - raise PlacementError( - "Original peer has insufficient time remaining for recovery" - ) - self.wait_overlap(("victim", "peer"), self.boundary["environment"]) - # Establish the original healthy cohort before admitting the actual fault. - self.cloud.release(self.gate("prepare")) - request = self.boundary["request"] - self.cloud.poll( - lambda: [ - event - for event in canonical(self.cloud.for_item(self.items["victim"])) - if event.get("requestId") == request - and event.get("kind") == "work_in_flight" - and event.get("atMs", deadline * 1000) < deadline * 1000 - ], - seconds=max(0.001, deadline - time.time()), - message="Original victim never entered the fault before its deadline", - ) - - def _peer_still_active(self) -> None: - if self.peer_boundary is None: - raise CollectionError("Recovery lost the original peer boundary") - if time.time() >= self.peer_boundary["deadline"]: - raise PlacementError("Original healthy peer expired during recovery") - if any( - event["request"] == self.peer_boundary["request"] - and event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - for event in self.cloud.for_item(self.items["peer"]) - ): - raise PlacementError("Original healthy peer ended during recovery") - - def _callback(self, item: dict[str, Any]) -> None: - self.cloud.poll( - lambda: [ - e - for e in self.cloud.for_item(item) - if e["phase"] == "WRAPPER_RETURN" and e.get("status") == "PENDING" - ], - seconds=self.cloud.manifest.get("driverTimeout", 120), - message="Callback fixture did not suspend", - ) - if self.release_capacity: - self.release_capacity() - try: - callback = item["runner"].wait_for_callback( - item["arn"], - name="completion", - timeout=self.cloud.manifest.get("driverTimeout", 120), - ) - finally: - if self.reserve_capacity: - self.reserve_capacity() - self.cloud.for_item(item) - item["runner"].send_callback_success(callback, result=item["marker"].encode()) - self.cloud.callback_records.append( - { - "executionArn": item["arn"], - "callbackId": callback, - "action": "success", - "time": time.time(), - } - ) - - def _deadline(self) -> None: - if self.boundary is None: - raise CollectionError("Deadline fixture lost its original placement record") - victim = self.items["victim"] - # The fixture cooperatively exits real held I/O after the runtime deadline; - # no control release is allowed to establish the cleanup assertion. - self.cloud.poll( - lambda: time.time() > self.boundary["deadline"], - seconds=self.cloud.manifest["invocationTimeout"] + 5, - ) - if self.scenario == "peer-recovery": - self._peer_still_active() - recovery = self.launch( - "recovery", - "barrier", - gate=self.gate("recovery"), - target_environment=self.boundary["environment"], - ) - self.wait_overlap(("peer", "recovery"), self.boundary["environment"]) - self.cloud.release(self.gate("recovery")) - self.cloud.finish(recovery) - self._peer_still_active() - self.cloud.release(self.gate("peer")) - self.cloud.finish(self.items["peer"]) - self.cloud.platform_timeout(victim, self.boundary["request"]) - self.cloud.poll( - lambda: ( - time.time() - > self.boundary["deadline"] - + self.seconds("cleanupMs", 5) - + self.seconds("observationGraceMs", 1) - ), - seconds=self.cloud.manifest["invocationTimeout"] + 30, - ) - # RUNNING is a valid pre-cleanup execution state for this measurement. - # Collection records it; retirement stops/reconciles the execution later. - - def execute(self) -> None: - victim = self.items["victim"] - if self.scenario == "native-overlap": - self.cloud.release(self.gate("cohort")) - for item in self.items.values(): - self.cloud.finish(item) - elif self.scenario in {"deadline-during-step", "peer-recovery"}: - self._deadline() - elif self.scenario == "suspension-retry": - self._callback(victim) - self.cloud.finish(victim) - elif self.scenario == "warm-reuse": - self.cloud.finish(victim) - self.cloud.finish(self.launch("warm", "warm-probe")) - else: - self.cloud.finish( - victim, "FAILED" if self.scenario == "normal-failure" else "SUCCEEDED" - ) - self.cloud.observation_ended_at = time.time() - - def collect(self) -> None: - self.cloud.wait(self.seconds("observationGraceMs", 1)) - self.cloud.collect(full_run=False) - self.cloud.observation_ended_at = time.time() - - def quiescent(self) -> bool: - events = self.cloud.refresh(validate_controls=False) - observations = canonical(events) - if any(item.get("kind") == "observation_error" for item in observations): - return False - if not self.cloud.invocations: - return True - if any( - not item.get("arn") or not item["native_requests"] - for item in self.cloud.invocations - ): - return False - entered = {event["request"] for event in select(events, "WRAPPER_ENTER")} - ended = { - event["request"] - for event in events - if event["phase"] in {"WRAPPER_RETURN", "WRAPPER_RAISE"} - } - if not entered or not entered <= ended: - return False - for start, end, identity in ( - ("task_start", "task_exit", "taskId"), - ("io_start", "io_exit", "taskId"), - ("checkpoint_call", "checkpoint_exit", "callId"), - ("poll_call", "poll_exit", "callId"), - ): - starts = Counter( - (item.get("requestId"), item.get(identity)) - for item in observations - if item.get("kind") == start - ) - ends = Counter( - (item.get("requestId"), item.get(identity)) - for item in observations - if item.get("kind") == end - ) - if starts - ends: - return False - # The public root task hook is a positive witness that task instrumentation - # ran; absent instrumentation must not look like an empty, settled pool. - root_ends = { - item.get("requestId") - for item in observations - if item.get("kind") == "task_exit" and item.get("taskId") == "root" - } - if not entered <= root_ends: - return False - return True - - def retire(self) -> bool: - # Recover native request IDs for responses lost during the cancelled - # phase before deciding which owned service executions can be stopped. - self.cloud.refresh(validate_controls=False) - if any(not item.get("arn") for item in self.cloud.invocations): - raise CollectionError( - "A launched invocation has no proven execution identity" - ) - self.cloud.settle_case() - self.cloud.poll( - self.quiescent, - seconds=self.cloud.manifest.get("driverTimeout", 120), - message="Case-owned task/request work is not quiescent", +class ScenarioDriver(SharedScenarioDriver): + def __init__(self, context, cloud, **options): + super().__init__( + context, + cloud, + runtime="python", + fixtures={ + **FIXTURES, + "peer": "barrier", + "recovery": "barrier", + "warm": "warm-probe", + }, + **options, ) - self.cloud.collect(full_run=False) - self.cloud.retired_at = time.time() - return True diff --git a/lmi_tests/summary.py b/lmi_tests/summary.py index 79644f8f2..0a2e9fad2 100644 --- a/lmi_tests/summary.py +++ b/lmi_tests/summary.py @@ -1,57 +1,9 @@ -"""Summarize the shared lifecycle report without inventing cloud acceptance.""" +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Render the shared report for the Python fixture's artifact directory.""" -import json from pathlib import Path - - -def summarize(directory): - directory = Path(directory) - manifest = directory / "manifest.json" - config = json.loads(manifest.read_text()) if manifest.exists() else None - lines = ["Python shared LMI lifecycle conformance."] - if config: - lines.append( - f"Commit `{config['commit']}`; {config['runtime']}; native concurrency {list(config['concurrencies'].values())}." - ) - shared = directory / "shared" - path = shared / "standard/report.json" - report = json.loads(path.read_text()) if path.exists() else None - if report and config is None: - lines.append( - "Deployment identity is missing; current cloud acceptance is unavailable." - ) - elif report and config and report.get("run", {}).get("name") != config["run"]: - lines.append( - "Report belongs to another run; current cloud acceptance is unavailable." - ) - elif report and report.get("results"): - lines.extend(f"- {case['id']}: {case['status']}" for case in report["results"]) - lines.append( - f"Shared runner exit code: {report.get('ci', {}).get('exit_code', 'unknown')}." - ) - else: - lines.append( - "Cloud scenarios did not run. Inspect provisioning/build diagnostics." - ) - for path in sorted(directory.glob("*-error.json")): - data = json.loads(path.read_text()) - lines.append(f"- {path.stem}: {data['type']}") - path = shared / "cleanup.json" - cleanup = json.loads(path.read_text()) if path.exists() else {} - if config is None or cleanup.get("runId") != config["run"]: - lines.append("Run cleanup not confirmed; persistent resources are retained.") - elif cleanup.get("status") == "NO_CASES": - lines.append("No scenario work was recorded for cleanup.") - elif cleanup.get("status") == "CLEANUP_REQUESTED" and cleanup.get("errors") == []: - lines.append( - "Run-scoped cleanup requests completed; see case reports for task retirement." - ) - else: - lines.append( - "Run cleanup failed or is incomplete; inspect cleanup diagnostics." - ) - return "\n\n".join(lines) - +from aws_durable_execution_conformance_tests.lmi.summary import summarize if __name__ == "__main__": print(summarize(Path(__file__).parent / "artifacts")) diff --git a/lmi_tests/tests/unit/test_cloud.py b/lmi_tests/tests/unit/test_cloud.py deleted file mode 100644 index 156779586..000000000 --- a/lmi_tests/tests/unit/test_cloud.py +++ /dev/null @@ -1,430 +0,0 @@ -import json -import io -from unittest.mock import Mock - -import boto3 -from botocore.stub import ANY, Stubber -from botocore.exceptions import ClientError -import pytest - -from lmi_tests import cloud as module -from lmi_tests.evidence import CollectionError -from lmi_tests.summary import summarize - - -@pytest.fixture -def driver(monkeypatch, tmp_path): - lam = boto3.client( - "lambda", - region_name="us-west-2", - aws_access_key_id="test", - aws_secret_access_key="test", - ) - monkeypatch.setattr(module, "client", lambda _: lam) - monkeypatch.setattr(module, "ARTIFACTS", tmp_path) - # CloudRunner.__init__ normally discovers credentials; the real client below - # is still used for invoke serialization/validation through botocore Stubber. - monkeypatch.setattr( - "aws_durable_execution_sdk_python_testing.runner.boto3.client", - lambda *a, **kw: lam, - ) - driver = module.Cloud( - { - "functions": { - "c1": "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" - }, - "region": "us-west-2", - "run": "unit", - "bucket": "unit-bucket", - "commit": "commit", - "invocationTimeout": 60, - "concurrencies": {"c1": 1}, - }, - "c1", - ) - driver.s3 = Mock() - return driver - - -@pytest.mark.parametrize("configuration", ["c1", "c2"]) -@pytest.mark.parametrize("scenario", ["success", "deadline"]) -def test_scenarios_reuse_the_qualified_function_for_their_concurrency( - driver, configuration, scenario -): - driver.configuration = configuration - driver.manifest["concurrencies"] = module.CONCURRENCIES - driver.manifest["functions"]["c2"] = ( - "arn:aws:lambda:us-west-2:123456789012:function:lmi-c2:$LATEST.PUBLISHED" - ) - assert driver.concurrency == module.CONCURRENCIES[configuration] - arn = "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED/durable-execution/name/id" - with Stubber(driver.lam) as stub: - stub.add_response( - "invoke", - {"StatusCode": 202, "DurableExecutionArn": arn}, - { - "FunctionName": driver.manifest["functions"][configuration], - "InvocationType": "Event", - "Payload": ANY, - }, - ) - item = driver.start(scenario, gate="fault" if scenario == "deadline" else None) - assert item["arn"] == arn - stub.assert_no_pending_responses() - - -def test_verify_rejects_a_deployment_missing_a_concurrency_configuration(driver): - driver.manifest["concurrencies"] = module.CONCURRENCIES - with pytest.raises(module.ProvisioningError, match="both c1 and c2"): - driver.verify() - - -def test_settlement_releases_controls_stops_retries_and_waits_for_every_wrapper(driver): - driver.invocations = [{"arn": "fault"}, {"arn": "completed"}] - calls = [] - driver.release_all = Mock(side_effect=lambda: calls.append("release")) - statuses = {"fault": "RUNNING", "completed": "SUCCEEDED"} - driver.lam.get_durable_execution = Mock( - side_effect=lambda **kw: {"Status": statuses[kw["DurableExecutionArn"]]} - ) - - def stop(**kw): - calls.append("stop") - statuses[kw["DurableExecutionArn"]] = "STOPPED" - - driver.lam.stop_durable_execution = Mock(side_effect=stop) - events = [ - {"phase": "WRAPPER_ENTER", "request": "first"}, - {"phase": "WRAPPER_ENTER", "request": "retry"}, - {"phase": "WRAPPER_RETURN", "request": "first"}, - ] - driver.refresh = Mock(return_value=events) - - def poll(predicate, **_): - # A terminal durable execution does not prove a stopped LMI worker. - assert not predicate() - events.append({"phase": "WRAPPER_RAISE", "request": "retry"}) - assert predicate() - calls.append("settled") - - driver.poll = poll - driver.settle_case() - assert calls == ["release", "stop", "settled"] - driver.lam.stop_durable_execution.assert_called_once_with( - DurableExecutionArn="fault" - ) - - -def test_settlement_reports_a_worker_still_running_after_release(driver): - driver.invocations = [{"arn": "execution"}] - driver.lam.get_durable_execution = Mock(return_value={"Status": "STOPPED"}) - driver.refresh = Mock(return_value=[{"phase": "WRAPPER_ENTER", "request": "stuck"}]) - - def poll(predicate, **kwargs): - assert not predicate() - raise CollectionError(kwargs["message"]) - - driver.poll = poll - with pytest.raises(CollectionError, match="not ready for reuse"): - driver.settle_case() - - -@pytest.mark.parametrize("status", ["SUCCEEDED", "RUNNING"]) -def test_stop_race_is_ignored_only_after_confirmed_completion(driver, status): - driver.invocations = [{"arn": "execution"}] - driver.lam.get_durable_execution = Mock( - side_effect=[{"Status": "RUNNING"}, {"Status": status}] - ) - driver.lam.stop_durable_execution = Mock( - side_effect=ClientError( - {"Error": {"Code": "InvalidParameterValueException"}}, - "StopDurableExecution", - ) - ) - driver.refresh = Mock(return_value=[]) - if status == "RUNNING": - with pytest.raises(ClientError): - driver.settle_case() - else: - driver.settle_case() - - -def test_missing_execution_arn_is_collection_error_not_pass(driver): - with Stubber(driver.lam) as stub: - stub.add_response("invoke", {"StatusCode": 202}) - with pytest.raises(CollectionError, match="API failed"): - driver.start("success") - - -@pytest.mark.parametrize( - "message", ["Runtime.Timeout", "Task timed out after 10 seconds"] -) -def test_requires_request_specific_service_timeout(driver, message): - good = { - "InvocationCompletedDetails": { - "RequestId": "request", - "Error": {"Payload": message}, - } - } - driver.history = Mock(return_value=[good]) - driver.poll = lambda predicate, **_: ( - predicate() or (_ for _ in ()).throw(CollectionError()) - ) - assert driver.platform_timeout({}, "request") == good - for events in ( - [{"EventType": "ExecutionTimedOut"}], - [ - { - "InvocationCompletedDetails": { - "RequestId": "different", - "Error": {"Payload": message}, - } - } - ], - ): - driver.history.return_value = events - with pytest.raises(CollectionError): - driver.platform_timeout({}, "request") - - -def test_summary_never_claims_unexecuted_or_skipped_cloud_success(tmp_path): - assert "did not run" in summarize(tmp_path) - (tmp_path / "manifest.json").write_text( - json.dumps( - { - "run": "current", - "commit": "sdk", - "runtime": "python3.14", - "concurrencies": {"c1": 1, "c2": 2}, - } - ) - ) - shared = tmp_path / "shared/standard" - shared.mkdir(parents=True) - (shared / "report.json").write_text( - json.dumps( - { - "run": {"name": "current"}, - "results": [ - {"id": "lmi-lifecycle-1", "status": "FAILED"}, - {"id": "lmi-lifecycle-2", "status": "UNCOVERED"}, - ], - "ci": {"exit_code": 1}, - } - ) - ) - summary = summarize(tmp_path) - assert "lmi-lifecycle-1: FAILED" in summary - assert "lmi-lifecycle-2: UNCOVERED" in summary - assert "PASS" not in summary - (tmp_path / "deploy-error.json").write_text( - json.dumps({"type": "ProvisioningError"}) - ) - assert "ProvisioningError" in summarize(tmp_path) - - -def test_runs_share_functions_but_never_controls_or_event_prefixes(driver): - other = module.Cloud({**driver.manifest, "run": "other"}, "c1") - other.s3 = driver.s3 - objects = {} - driver.s3.put_object.side_effect = lambda **kw: objects.update( - {kw["Key"]: kw["Body"]} - ) - driver.hold("same-gate") - other.hold("same-gate") - driver.release("same-gate") - assert objects == { - "runs/unit/control/same-gate": b"release", - "runs/other/control/same-gate": b"hold", - } - driver.s3.get_paginator.return_value.paginate.return_value = [] - driver.refresh(markers=[]) - other.refresh(markers=[]) - assert [ - c.kwargs["Prefix"] - for c in driver.s3.get_paginator.return_value.paginate.call_args_list - ] == ["runs/unit/events/", "runs/other/events/"] - - -def test_cleanup_discovery_ignores_other_runs_and_rejects_foreign_functions( - driver, tmp_path -): - arn = driver.manifest["functions"]["c1"] + "/durable-execution/case/id" - driver.refresh = Mock(return_value=[{"execution": arn, "marker": "case"}]) - records = tmp_path / "invocations" - records.mkdir() - (records / "old.json").write_text( - json.dumps({"payload": {"run": "previous", "marker": "old"}, "arn": "foreign"}) - ) - assert driver.run_invocations() == [{"arn": arn, "marker": "case"}] - driver.refresh.assert_called_once_with(markers=[], validate_controls=False) - driver.refresh.return_value = [{"execution": "foreign", "marker": "bad"}] - with pytest.raises(CollectionError, match="outside this deployment"): - driver.run_invocations() - - -def test_polling_reads_only_current_case_and_surfaces_control_errors(driver): - driver.invocations = [{"marker": "current"}] - event = { - "run": "unit", - "commit": "commit", - "marker": "current", - "request": "r", - "sequence": 1, - "time": 1, - "phase": "CONTROL_ERROR", - "gate": "g", - "error": "403 Forbidden", - } - key = "runs/unit/events/current/r/000001.json" - driver.s3.get_paginator.return_value.paginate.return_value = [ - {"Contents": [{"Key": key}]} - ] - driver.s3.get_object.side_effect = lambda **_: { - "Body": io.BytesIO(json.dumps(event).encode()) - } - with pytest.raises(CollectionError, match="403"): - driver.refresh() - driver.s3.get_paginator.return_value.paginate.assert_called_once_with( - Bucket="unit-bucket", Prefix="runs/unit/events/current/" - ) - driver.s3.get_object.assert_called_once_with(Bucket="unit-bucket", Key=key) - # Full diagnostic collection preserves errors without turning itself into a - # second failing test or omitting evidence from earlier cases. - assert driver.refresh(markers=[]) == [event] - driver.s3.get_paginator.return_value.paginate.assert_called_with( - Bucket="unit-bucket", Prefix="runs/unit/events/" - ) - - -class Clock: - def __init__(self): - self.now = 0.0 - - def monotonic(self): - return self.now - - def time(self): - return self.now - - def sleep(self, seconds): - self.now += seconds - - -def throttle_error(): - return ClientError( - {"Error": {"Code": "TooManyRequestsException", "Message": "Rate exceeded"}}, - "GetDurableExecutionHistory", - ) - - -def test_history_retries_preserve_pages_and_pace_requests(driver, monkeypatch): - clock = Clock() - monkeypatch.setattr(module, "time", clock) - driver.lam.get_durable_execution_history = Mock( - side_effect=[ - throttle_error(), - {"Events": [{"EventId": 1}], "NextMarker": "page2"}, - {"Events": [{"EventId": 2}]}, - ] - ) - assert driver.history({"arn": "execution", "marker": "case"}) == [ - {"EventId": 1}, - {"EventId": 2}, - ] - calls = driver.lam.get_durable_execution_history.call_args_list - assert calls[0] == calls[1] - assert calls[2].kwargs["Marker"] == "page2" - assert clock.now >= 2 - - -def test_history_throttling_has_a_finite_budget(driver, monkeypatch): - clock = Clock() - monkeypatch.setattr(module, "time", clock) - driver.lam.get_durable_execution_history = Mock(side_effect=throttle_error()) - with pytest.raises(CollectionError, match="throttle budget"): - driver.history_page(DurableExecutionArn="execution") - assert driver.lam.get_durable_execution_history.call_count == 5 - assert clock.now < 20 - - -def test_history_does_not_retry_permission_errors_or_invocations(driver, monkeypatch): - monkeypatch.setattr(module, "time", Clock()) - driver.lam.invoke = Mock( - return_value={"StatusCode": 202, "DurableExecutionArn": "execution"} - ) - item = driver.start("success") - error = ClientError( - {"Error": {"Code": "AccessDeniedException"}}, "GetDurableExecutionHistory" - ) - driver.lam.get_durable_execution_history = Mock(side_effect=error) - with pytest.raises(ClientError): - item["runner"].lambda_client.get_durable_execution_history( - DurableExecutionArn="execution" - ) - driver.lam.invoke.assert_called_once() - driver.lam.get_durable_execution_history.assert_called_once() - - -def test_case_collection_is_scoped_but_final_collection_is_complete(driver): - driver.invocations = [{"arn": "execution", "marker": "case"}] - driver.refresh = Mock(return_value=[{"execution": "execution", "marker": "case"}]) - driver.history = Mock(return_value=[]) - driver.lam.get_durable_execution = Mock(return_value={"Status": "SUCCEEDED"}) - driver.logs = Mock() - driver.collect(full_run=False) - driver.refresh.assert_called_once_with(markers=["case"], validate_controls=False) - driver.history.assert_called_once() - driver.logs.get_paginator.assert_not_called() - driver.manifest["created"] = 0 - driver.lam.get_function_configuration = Mock( - return_value={"LoggingConfig": {"LogGroup": "group"}} - ) - driver.logs.get_paginator.return_value.paginate.return_value = [] - driver.collect() - driver.refresh.assert_called_with(markers=[], validate_controls=False) - driver.logs.get_paginator.assert_called_once_with("filter_log_events") - - -def test_summary_rejects_other_run_reports_and_requires_matching_cleanup( - tmp_path, -): - config = { - "commit": "sdk", - "runtime": "python3.14", - "concurrencies": {"c1": 1, "c2": 2}, - "run": "current", - } - (tmp_path / "manifest.json").write_text(json.dumps(config)) - shared = tmp_path / "shared" - (shared / "standard").mkdir(parents=True) - (shared / "standard/report.json").write_text( - json.dumps( - { - "run": {"name": "previous"}, - "results": [{"id": "lmi-lifecycle-1", "status": "PASSED"}], - } - ) - ) - receipt = shared / "cleanup.json" - receipt.write_text( - json.dumps({"runId": "previous", "status": "CLEANUP_REQUESTED", "errors": []}) - ) - summary = summarize(tmp_path) - assert "another run" in summary and "PASSED" not in summary - assert "Run cleanup not confirmed" in summary - receipt.write_text( - json.dumps({"runId": "current", "status": "CLEANUP_REQUESTED", "errors": []}) - ) - assert "Run-scoped cleanup requests completed" in summarize(tmp_path) - - -def test_summary_does_not_accept_a_report_without_deployment_identity(tmp_path): - shared = tmp_path / "shared/standard" - shared.mkdir(parents=True) - (shared / "report.json").write_text( - json.dumps({"results": [{"id": "lmi-lifecycle-1", "status": "PASSED"}]}) - ) - summary = summarize(tmp_path) - assert "Deployment identity is missing" in summary - assert "PASSED" not in summary diff --git a/lmi_tests/tests/unit/test_deploy.py b/lmi_tests/tests/unit/test_deploy.py index 16c693ebc..57d1e8f92 100644 --- a/lmi_tests/tests/unit/test_deploy.py +++ b/lmi_tests/tests/unit/test_deploy.py @@ -1,425 +1,7 @@ -import copy -import json from pathlib import Path - -from botocore.exceptions import ClientError from unittest.mock import Mock - import pytest - from lmi_tests import deploy -from lmi_tests.evidence import ProvisioningError - - -@pytest.fixture -def manifest(): - return { - "run": "unit", - "stack": "python-lmi-e2e", - "bucket": "test-bucket", - "role": "role", - "provider": "arn:aws:lambda:us-west-2:123456789012:capacity-provider:provider", - "region": "us-west-2", - "account": "123456789012", - "runtime": "python3.14", - "concurrencies": deploy.CONCURRENCIES, - "functions": {}, - "commit": "sha", - "codeKey": "code/sha.zip", - "codeSha256": "hash", - "owned": True, - "invocationTimeout": 60, - "executionTimeout": 240, - } - - -def test_one_deployment_shares_two_functions_across_all_scenarios(manifest): - runtime = "python3.14" - template = deploy.template(manifest) - functions = { - key: resource - for key, resource in template["Resources"].items() - if resource["Type"] == "AWS::Lambda::Function" - } - assert set(functions) == {"c1Function", "c2Function"} - assert ( - functions["c1Function"]["Properties"]["Code"] - == functions["c2Function"]["Properties"]["Code"] - ) - for key, concurrency in deploy.CONCURRENCIES.items(): - config = template["Resources"][key + "Function"]["Properties"] - assert config["FunctionScalingConfig"] == deploy.SCALING - assert config["Runtime"] == runtime - assert ( - config["CapacityProviderConfig"][ - "LambdaManagedInstancesCapacityProviderConfig" - ]["PerExecutionEnvironmentMaxConcurrency"] - == concurrency - ) - assert config["DurableConfig"]["ExecutionTimeout"] == 240 - assert config["Timeout"] == 60 - assert config["Handler"] == "lmi_tests.fixture.handler" - assert ( - template["Outputs"][key]["Value"]["Fn::Join"][1][-1] == ":$LATEST.PUBLISHED" - ) - assert not any( - r["Type"] in {"AWS::Lambda::Version", "AWS::Lambda::CapacityProvider"} - for r in template["Resources"].values() - ) - - -def test_verify_rejects_standard_lambda_wrong_artifact_and_unbounded_placement( - manifest, -): - config = copy.deepcopy( - deploy.template(manifest)["Resources"]["c1Function"]["Properties"] - ) - config.update(State="Active", Version=deploy.QUALIFIER, CodeSha256="hash") - scaling = {"AppliedFunctionScalingConfig": deploy.SCALING} - deploy.verify(config, scaling, manifest, "c1") - with pytest.raises(ProvisioningError): - deploy.verify(config, scaling, manifest, "c2") - for field in ( - "CapacityProviderConfig", - "CodeSha256", - "Runtime", - "Timeout", - "DurableConfig", - "Environment", - "Role", - "Handler", - ): - bad = copy.deepcopy(config) - del bad[field] - with pytest.raises(ProvisioningError): - deploy.verify(bad, scaling, manifest, "c1") - with pytest.raises(ProvisioningError): - deploy.verify( - config, - {"AppliedFunctionScalingConfig": {"MaxExecutionEnvironments": 2}}, - manifest, - "c1", - ) - - -def test_cleanup_refuses_foreign_owner_and_failed_create(manifest, monkeypatch): - cfn = Mock() - cfn.describe_stacks.return_value = { - "Stacks": [{"Tags": [{"Key": "Suite", "Value": "other"}]}] - } - monkeypatch.setattr(deploy, "client", lambda _: cfn) - with pytest.raises(ProvisioningError, match="Refusing"): - deploy.cleanup(manifest) - cfn.delete_stack.assert_not_called() - deploy.cleanup({**manifest, "owned": False}) - cfn.delete_stack.assert_not_called() - - -def stack(manifest, status="UPDATE_COMPLETE"): - return { - "StackStatus": status, - "Tags": [ - {"Key": "Suite", "Value": deploy.OWNER}, - {"Key": "Stack", "Value": manifest["stack"]}, - {"Key": "Persistent", "Value": "true"}, - ], - "Outputs": [ - { - "OutputKey": key, - "OutputValue": f"arn:aws:lambda:us-west-2:123456789012:function:{manifest['stack']}-{key}:$LATEST.PUBLISHED", - } - for key in deploy.CONCURRENCIES - ], - } - - -@pytest.fixture -def apis(monkeypatch, tmp_path): - clients = {name: Mock() for name in ("cloudformation", "s3", "lambda")} - monkeypatch.setattr(deploy, "client", clients.__getitem__) - monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) - clients["s3"].get_public_access_block.return_value = { - "PublicAccessBlockConfiguration": { - "BlockPublicAcls": True, - "IgnorePublicAcls": True, - "BlockPublicPolicy": True, - "RestrictPublicBuckets": True, - } - } - clients["s3"].get_object.side_effect = AssertionError( - "Deployment must not read an S3 lock" - ) - return clients - - -def test_later_run_updates_fixed_functions_without_recreating_resources( - manifest, apis, monkeypatch -): - cfn, lam = apis["cloudformation"], apis["lambda"] - monkeypatch.setattr( - deploy, "owned_stack", Mock(side_effect=[None, stack(manifest)]) - ) - monkeypatch.setattr(deploy, "wait_stack", Mock(return_value=stack(manifest))) - idle = Mock() - monkeypatch.setattr(deploy, "wait_for_idle_functions", idle) - verify = Mock() - monkeypatch.setattr(deploy, "verify", verify) - deploy.deploy_stack(manifest, b"first-code") - second = { - **manifest, - "run": "second", - "codeKey": "code/new-digest.zip", - "functions": {}, - } - deploy.deploy_stack(second, b"second-code") - cfn.create_stack.assert_called_once() - assert cfn.update_stack.call_count == 2 - apis["s3"].get_object.assert_not_called() - assert all( - not call.kwargs["Key"].startswith("ownership/") - for call in apis["s3"].put_object.call_args_list - ) - templates = [ - json.loads(call.kwargs["TemplateBody"]) - for call in cfn.update_stack.call_args_list - ] - assert templates[0]["Outputs"] == templates[1]["Outputs"] - assert templates[0]["Resources"].keys() == templates[1]["Resources"].keys() - for key in deploy.CONCURRENCIES: - before = templates[0]["Resources"][key + "Function"]["Properties"] - after = templates[1]["Resources"][key + "Function"]["Properties"] - assert before["FunctionName"] == after["FunctionName"] - assert before["Code"]["S3Key"] != after["Code"]["S3Key"] - assert after["Environment"]["Variables"]["LMI_RUN_ID"] == "second" - idle.assert_called_once() - assert verify.call_count == 4 - assert lam.get_function_configuration.call_count == 4 - cfn.delete_stack.assert_not_called() - lam.delete_function.assert_not_called() - - -def test_unchanged_update_still_verifies_functions(manifest, apis, monkeypatch): - cfn = apis["cloudformation"] - cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} - cfn.update_stack.side_effect = ClientError( - { - "Error": { - "Code": "ValidationError", - "Message": "No updates are to be performed.", - } - }, - "UpdateStack", - ) - monkeypatch.setattr(deploy, "wait_for_idle_functions", Mock()) - wait = Mock() - monkeypatch.setattr(deploy, "wait_stack", wait) - verify = Mock() - monkeypatch.setattr(deploy, "verify", verify) - deploy.deploy_stack(manifest, b"code") - wait.assert_not_called() - assert verify.call_count == 2 - cfn.create_stack.assert_not_called() - - -@pytest.mark.parametrize( - "status", ["ROLLBACK_COMPLETE", "UPDATE_IN_PROGRESS", "UPDATE_ROLLBACK_FAILED"] -) -def test_failed_or_busy_persistent_stack_is_retained(manifest, apis, status): - cfn = apis["cloudformation"] - cfn.describe_stacks.return_value = {"Stacks": [stack(manifest, status)]} - with pytest.raises(ProvisioningError, match="retained"): - deploy.deploy_stack(manifest, b"code") - cfn.update_stack.assert_not_called() - cfn.delete_stack.assert_not_called() - apis["s3"].put_object.assert_not_called() - - -def test_foreign_stack_cannot_be_adopted(manifest, apis): - cfn = apis["cloudformation"] - cfn.describe_stacks.return_value = {"Stacks": [{"Tags": []}]} - with pytest.raises(ProvisioningError, match="not owned"): - deploy.deploy_stack(manifest, b"code") - cfn.update_stack.assert_not_called() - cfn.create_stack.assert_not_called() - - -def test_failed_creation_does_not_trigger_deletion(manifest, apis): - cfn = apis["cloudformation"] - cfn.describe_stacks.side_effect = ClientError( - {"Error": {"Code": "ValidationError", "Message": "Stack does not exist"}}, - "DescribeStacks", - ) - cfn.create_stack.side_effect = RuntimeError("create failed") - with pytest.raises(RuntimeError, match="create failed"): - deploy.deploy_stack(manifest, b"code") - cfn.delete_stack.assert_not_called() - - -def test_prior_executions_must_finish_before_code_updates(manifest, apis, monkeypatch): - lam = apis["lambda"] - one_function = {"Outputs": stack(manifest)["Outputs"][:1]} - lam.get_paginator.return_value.paginate.side_effect = [ - [{"DurableExecutions": [{"DurableExecutionArn": "old"}]}], - [{"DurableExecutions": []}], - ] - sleep = Mock() - monkeypatch.setattr(deploy.time, "sleep", sleep) - deploy.wait_for_idle_functions(one_function) - sleep.assert_called_once_with(2) - lam.stop_durable_execution.assert_not_called() - lam.get_paginator.return_value.paginate.assert_called_with( - FunctionName=one_function["Outputs"][0]["OutputValue"].rsplit(":", 1)[0], - Statuses=["RUNNING"], - ) - - -@pytest.mark.parametrize( - "page", - [ - {"DurableExecutions": [{"DurableExecutionArn": "old"}]}, - {"DurableExecutions": [], "NextMarker": "more"}, - ], -) -def test_busy_or_incompletely_listed_functions_block_updates( - manifest, apis, monkeypatch, page -): - apis["lambda"].get_paginator.return_value.paginate.return_value = [page] - cfn = apis["cloudformation"] - cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} - wait = deploy.wait_for_idle_functions - monkeypatch.setattr( - deploy, "wait_for_idle_functions", lambda value: wait(value, seconds=0) - ) - with pytest.raises(ProvisioningError, match="code was not updated"): - deploy.deploy_stack(manifest, b"code") - cfn.update_stack.assert_not_called() - assert not any( - call.kwargs["Key"] == manifest["codeKey"] - for call in apis["s3"].put_object.call_args_list - ) - apis["lambda"].stop_durable_execution.assert_not_called() - - -def test_only_run_data_expires_and_code_is_retained(manifest): - rules = deploy.template(manifest)["Resources"]["Bucket"]["Properties"][ - "LifecycleConfiguration" - ]["Rules"] - assert all(rule.get("Prefix") == "runs/" for rule in rules) - assert all(rule["ExpirationInDays"] == 1 for rule in rules) - - -def test_cleanup_releases_only_current_run_and_keeps_infrastructure( - manifest, apis, monkeypatch, tmp_path -): - from lmi_tests import cloud - - cfn, s3, lam = apis["cloudformation"], apis["s3"], apis["lambda"] - cfn.describe_stacks.return_value = {"Stacks": [stack(manifest)]} - cfn.describe_stack_resources.return_value = { - "StackResources": [ - { - "ResourceType": "AWS::S3::Bucket", - "PhysicalResourceId": manifest["bucket"], - "ResourceStatus": "CREATE_COMPLETE", - } - ] - } - driver = Mock() - driver.run_invocations.return_value = [{"arn": "this-run"}] - monkeypatch.setattr(cloud, "Cloud", Mock(return_value=driver)) - deploy.cleanup(manifest) - s3.put_object.assert_called_once_with( - Bucket=manifest["bucket"], Key="runs/unit/control/release-all", Body=b"release" - ) - assert driver.invocations == [{"arn": "this-run"}] - driver.settle_case.assert_called_once() - cfn.delete_stack.assert_not_called() - lam.delete_function.assert_not_called() - s3.delete_objects.assert_not_called() - assert json.loads((tmp_path / "cleanup.json").read_text())["status"] == "RETAINED" - - -@pytest.mark.parametrize( - "failure", - ["run-id", "stack-name", "environment", "account", "capacity", "build", "digest"], -) -def test_failed_preflight_cannot_reuse_prior_run_evidence( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path, failure: str -) -> None: - """An EXIT trap after any preflight error must not target the previous run.""" - import hashlib - from types import SimpleNamespace - - artifacts = tmp_path / "artifacts" - artifacts.mkdir() - package = tmp_path / "lmi_tests/build" - package.mkdir(parents=True) - data = b"current-build" - (package / "function.zip").write_bytes(data) - build = {"commit": "current", "sha256": hashlib.sha256(data).hexdigest()} - if failure == "digest": - build["sha256"] = "wrong" - deploy.save(artifacts / "build.json", build) - build_bytes = (artifacts / "build.json").read_bytes() - deploy.save(artifacts / "manifest.json", {"run": "previous", "owned": True}) - deploy.save(artifacts / "cleanup.json", {"run": "previous", "status": "RETAINED"}) - deploy.save(artifacts / "configuration/c1.json", {"run": "previous"}) - (artifacts / "cloud.xml").write_text( - '' - ) - clients = {name: Mock() for name in ("sts", "lambda", "s3")} - clients["sts"].get_caller_identity.return_value = {"Account": "123456789012"} - clients["lambda"].get_capacity_provider.return_value = { - "CapacityProvider": {"CapacityProviderScalingConfig": {"MaxVCpuCount": 2}} - } - monkeypatch.setattr(deploy, "ROOT", tmp_path) - monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) - monkeypatch.setattr(deploy, "client", clients.__getitem__) - deploy_stack = Mock() - monkeypatch.setattr(deploy, "deploy_stack", deploy_stack) - monkeypatch.setenv("AWS_REGION", "us-west-2") - monkeypatch.setenv("TEST_ACCOUNT_ID", "123456789012") - monkeypatch.setenv( - "CAPACITY_PROVIDER_ARN", - "arn:aws:lambda:us-west-2:123456789012:capacity-provider/test", - ) - monkeypatch.setenv("TEST_LAMBDA_EXECUTION_ROLE_ARN", "test-role") - args = SimpleNamespace( - run_id="current", stack_name="python-lmi-e2e", runtime="python3.14" - ) - if failure == "run-id": - args.run_id = "INVALID" - elif failure == "stack-name": - args.stack_name = "INVALID" - elif failure == "environment": - monkeypatch.delenv("CAPACITY_PROVIDER_ARN") - elif failure == "account": - clients["sts"].get_caller_identity.return_value = {"Account": "other"} - elif failure == "capacity": - clients["lambda"].get_capacity_provider.side_effect = RuntimeError( - "capacity unavailable" - ) - elif failure == "build": - (package / "function.zip").unlink() - with pytest.raises((ProvisioningError, KeyError, RuntimeError, FileNotFoundError)): - deploy.deploy(args) - assert (artifacts / "build.json").read_bytes() == build_bytes - assert not (artifacts / "manifest.json").exists() - assert not (artifacts / "cleanup.json").exists() - assert not (artifacts / "cloud.xml").exists() - assert not (artifacts / "configuration").exists() - deploy_stack.assert_not_called() - for command in ("collect", "cleanup"): - monkeypatch.setattr(deploy.sys, "argv", ["deploy", command]) - with pytest.raises(FileNotFoundError): - deploy.main() - clients["s3"].put_object.assert_not_called() - from lmi_tests.summary import summarize - - summary = summarize(artifacts) - assert "old-case" not in summary - assert "Cloud scenarios did not run" in summary - assert "Run cleanup not confirmed" in summary def test_failed_build_clears_prior_run_before_subprocess( @@ -437,32 +19,3 @@ def test_failed_build_clears_prior_run_before_subprocess( deploy.build() assert not (artifacts / "manifest.json").exists() assert not (artifacts / "cleanup.json").exists() - - -def test_artifact_cleanup_does_not_follow_symlinks( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - artifacts = tmp_path / "artifacts" - artifacts.mkdir() - outside = tmp_path / "outside" - outside.mkdir() - (outside / "evidence.json").write_text("keep") - (artifacts / "old-output").symlink_to(outside, target_is_directory=True) - monkeypatch.setattr(deploy, "ARTIFACTS", artifacts) - deploy.clear_run_artifacts() - assert (outside / "evidence.json").read_text() == "keep" - assert not (artifacts / "old-output").exists() - - -def test_manifest_is_invalidated_even_if_later_artifact_cleanup_fails( - monkeypatch: pytest.MonkeyPatch, tmp_path: Path -) -> None: - deploy.save(tmp_path / "manifest.json", {"run": "previous", "owned": True}) - deploy.save(tmp_path / "configuration/c1.json", {"run": "previous"}) - monkeypatch.setattr(deploy, "ARTIFACTS", tmp_path) - monkeypatch.setattr( - deploy.shutil, "rmtree", Mock(side_effect=PermissionError("blocked")) - ) - with pytest.raises(PermissionError, match="blocked"): - deploy.clear_run_artifacts() - assert not (tmp_path / "manifest.json").exists() diff --git a/lmi_tests/tests/unit/test_evidence.py b/lmi_tests/tests/unit/test_evidence.py deleted file mode 100644 index db1552ce5..000000000 --- a/lmi_tests/tests/unit/test_evidence.py +++ /dev/null @@ -1,65 +0,0 @@ -import copy - -import pytest - -from lmi_tests import evidence - - -def event(phase, time=1, **fields): - return { - "phase": phase, - "time": time, - "sequence": int(time * 10), - "marker": "a", - "request": "r1", - "execution": "exec1", - "environment": "env1", - "process": "p1", - "pid": 10, - "deadline": 10, - "gate": "gate", - **fields, - } - - -def alive(events): - return events + [ - {**e, "phase": "ALIVE", "time": max(v["time"] for v in events) + 1} - for e in events - if e["phase"] == "BLOCKED" - ] - - -def test_overlap_requires_distinct_processes_requests_and_executions(): - events = [ - event("BLOCKED"), - event( - "BLOCKED", - 2, - marker="b", - process="p2", - pid=11, - request="r2", - execution="exec2", - ), - ] - assert evidence.overlap(alive(events), {"a", "b"}, 2) == "env1" - with pytest.raises(evidence.PlacementError): - evidence.overlap(events, {"a", "b"}, 2) # starts alone do not prove overlap - for key in ("process", "pid", "request", "execution"): - invalid = copy.deepcopy(events) - invalid[1][key] = invalid[0][key] - with pytest.raises(evidence.PlacementError): - evidence.overlap(alive(invalid), {"a", "b"}, 2) - - -def test_different_environment_and_sequential_invocations_cannot_pass(): - a = event("BLOCKED") - b = event( - "BLOCKED", 3, marker="b", process="p2", pid=11, request="r2", execution="exec2" - ) - for events in ([a, {**b, "environment": "env2"}], [a, event("IO_EXIT", 2), b]): - with pytest.raises(evidence.PlacementError): - evidence.overlap(alive(events), {"a", "b"}, 2) - with pytest.raises(evidence.PlacementError): - evidence.overlap(alive([a, b]), {"a", "b"}, 2, environment="original-env") diff --git a/lmi_tests/tests/unit/test_lifecycle_mapping.py b/lmi_tests/tests/unit/test_lifecycle_mapping.py deleted file mode 100644 index c33b9fc9c..000000000 --- a/lmi_tests/tests/unit/test_lifecycle_mapping.py +++ /dev/null @@ -1,99 +0,0 @@ -"""All seven shared contracts reach their actual SDK fixture and preserve audit facts.""" - -from copy import deepcopy -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_conformance_tests.lmi.matrix import required_cases -from lmi_tests.shared_scenarios import ScenarioDriver, FIXTURES -from lmi_tests.shared_evidence import _decoded, assertion_events, retirement_snapshot -from lmi_tests.tests.unit.test_shared_adapter import ScriptCloud, context - - -@pytest.mark.parametrize( - "cell", required_cases("python", "lifecycle"), ids=lambda c: c.scenario -) -def test_each_shared_case_uses_its_required_original_roles(cell, monkeypatch): - cloud = ScriptCloud(cell.minimum_native_concurrency) - if cell.scenario == "peer-recovery": - from types import SimpleNamespace - from lmi_tests import shared_scenarios - from lmi_tests.tests.unit.test_peer_recovery import RecoveryCloud - - cloud = RecoveryCloud() - monkeypatch.setattr( - shared_scenarios, "time", SimpleNamespace(time=lambda: cloud.now) - ) - driver = ScenarioDriver( - context( - scenario=cell.scenario, - roles=cell.roles, - native=cell.minimum_native_concurrency, - ), - cloud, - ) - driver.wait_overlap = Mock(return_value=[]) - driver.start() - driver.place() - driver.execute() - launches = [action for action in cloud.actions if action[0] == "start"] - assert {item[2]["role"] for item in launches} == set(cell.roles) - {"resume"} - assert ( - next(item for item in launches if item[2]["role"] == "victim")[1] - == FIXTURES[cell.scenario] - ) - assert all( - "otel_view" not in item[2] and "export_mode" not in item[2] for item in launches - ) - if cell.scenario == "peer-recovery": - actions = cloud.actions - recovery_finished = next( - i for i, a in enumerate(actions) if a == ("finish", "recovery", "SUCCEEDED") - ) - peer_released = actions.index(("release", driver.gate("peer"))) - assert recovery_finished < peer_released - assert ("release", driver.gate("fault")) not in actions - if cell.scenario == "normal-failure": - assert ("finish", "victim", "FAILED") in cloud.actions - - -def test_scalar_and_wrapped_service_payloads_have_the_same_persisted_meaning(): - assert ( - _decoded({"Truncated": False, "Payload": '"value"'}) - == _decoded('"value"') - == "value" - ) - assert ( - _decoded({"Truncated": False, "Payload": "callback bytes"}) == "callback bytes" - ) - assert _decoded('{"Payload":"business field"}') == {"Payload": "business field"} - with pytest.raises(ValueError, match="Truncated"): - _decoded({"Truncated": True, "Payload": '"partial"'}) - - -@pytest.mark.parametrize( - "scenario,target", - [ - ("normal-return", "root"), - ("normal-failure", "root"), - ("warm-reuse", "late-callback"), - ], -) -def test_singular_assertion_boundary_does_not_discard_other_task_lifetimes( - scenario, target -): - original = [ - {"kind": kind, "taskId": task, "requestId": "victim", "atMs": time} - for task in ("root", "user-task-1", "late-callback") - for kind, time in (("task_start", 1), ("task_exit", 2)) - ] - retained = deepcopy(original) - events = assertion_events(original, scenario, {"victim": "victim"}) - assert [e["taskId"] for e in events if e["kind"] == "task_exit"] == [target] - assert len(events) == len(original) - assert original == retained - restored = retirement_snapshot({"events": events})["events"] - assert [ - {k: v for k, v in e.items() if k != "sourceKind"} for e in restored - ] == original diff --git a/lmi_tests/tests/unit/test_peer_recovery.py b/lmi_tests/tests/unit/test_peer_recovery.py deleted file mode 100644 index e62274ca5..000000000 --- a/lmi_tests/tests/unit/test_peer_recovery.py +++ /dev/null @@ -1,216 +0,0 @@ -"""Recovery scheduling uses original native deadlines, not late I/O or retries.""" - -from types import SimpleNamespace -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - Scenario, - ScenarioContext, -) -from lmi_tests import shared_scenarios -from lmi_tests.evidence import PlacementError -from lmi_tests.shared_scenarios import ScenarioDriver -from lmi_tests.tests.unit.test_shared_adapter import ScriptCloud - - -class RecoveryCloud(ScriptCloud): - def __init__(self): - super().__init__(2) - self.now = 0.0 - self.events = [] - self.retiring = False - self.configuration = "c2" - self.manifest["driverTimeout"] = 120 - - def emit(self, item, phase, **extra): - request = item["native_requests"][0] - value = { - "phase": phase, - "time": self.now, - "deadline": item["deadline"], - "marker": item["marker"], - "request": request, - "execution": item["arn"], - "environment": "environment", - "process": item["process"], - "pid": item["pid"], - "gate": item.get("held_gate"), - "sequence": 1 + sum(e["request"] == request for e in self.events), - **extra, - } - self.events.append(value) - return value - - def start(self, fixture, **options): - item = super().start(fixture, **options) - role = options["role"] - item.update( - native_requests=[f"request-{role}"], - deadline=self.now + 60, - process="peer-worker" if role == "peer" else "victim-worker", - pid=2 if role == "peer" else 1, - payload={"scenario": fixture, **options}, - held_gate=options.get("preparation_gate") or options.get("gate"), - ) - self.emit(item, "WRAPPER_ENTER") - self.now += 3 if role == "recovery" else 0.25 - self.emit(item, "BLOCKED") - return item - - def phase(self, item, phase, **options): - return [ - e - for e in self.events - if e["marker"] == item["marker"] and e["phase"] == phase - ] - - def for_item(self, item): - return [e for e in self.refresh() if e["marker"] == item["marker"]] - - def refresh(self, **options): - # Deliberately allow I/O liveness after a native deadline: this must not - # be accepted as proof that the original invocation is still healthy. - for item in self.invocations: - if not item.get("finished"): - self.emit(item, "ALIVE") - return list(self.events) - - def poll(self, predicate, seconds=30, **options): - deadline = self.now + seconds - while True: - value = predicate() - if value: - return value - if self.now >= deadline: - raise AssertionError(options.get("message", "poll timed out")) - self.now = min(deadline, self.now + 0.5) - - def release(self, gate): - super().release(gate) - for item in self.invocations: - if item["payload"].get("preparation_gate") == gate: - self.emit(item, "IO_EXIT") - item["held_gate"] = item["payload"]["gate"] - self.emit(item, "BLOCKED") - self.emit( - item, - "CONFORMANCE", - observation={"kind": "work_in_flight", "atMs": self.now * 1000}, - ) - - def finish(self, item, status="SUCCEEDED"): - assert self.now < item["deadline"], "The selected original invocation expired" - super().finish(item, status) - self.now += 0.25 - self.emit(item, "WRAPPER_RETURN", status=status) - item["finished"] = True - - def wait_terminal(self, item): - assert self.retiring, "An active victim must reach assertions before cleanup" - return super().wait_terminal(item) - - def wait(self, seconds): - self.now += seconds - - def collect(self, **options): - self.actions.append(("collect",)) - - def settle_case(self): - self.retiring = True - self.actions.append(("settle",)) - self.wait_terminal(self.invocations[0]) - - -def recovery_driver(monkeypatch): - cloud = RecoveryCloud() - monkeypatch.setattr( - shared_scenarios, "time", SimpleNamespace(time=lambda: cloud.now) - ) - profile = Profile("c2", "function", 2, 2, "sdk", "configuration") - scenario = Scenario( - "case", - "lmi-lifecycle-4", - "c2", - "peer-recovery", - roles=("victim", "peer", "recovery"), - ) - return ScenarioDriver(ScenarioContext("run", scenario, profile), cloud), cloud - - -def test_peer_has_recovery_headroom_and_fault_is_armed_after_cohort_placement( - monkeypatch, -): - driver, cloud = recovery_driver(monkeypatch) - driver.start() - assert [i["marker"] for i in cloud.invocations] == ["victim"] - driver.place() - victim, peer = cloud.invocations - assert peer["deadline"] - victim["deadline"] >= 30 - armed = next( - e - for e in cloud.events - if e.get("observation", {}).get("kind") == "work_in_flight" - ) - peer_ready = next( - e for e in cloud.events if e["phase"] == "BLOCKED" and e["marker"] == "peer" - ) - assert peer_ready["time"] <= armed["time"] < victim["deadline"] - driver.execute() - assert ("finish", "peer", "SUCCEEDED") in cloud.actions - assert not any(action[0] == "terminal" for action in cloud.actions) - assert ("release", driver.gate("fault")) not in cloud.actions - driver.collect() - driver.quiescent = Mock(return_value=True) - driver.retire() - assert cloud.actions.index(("collect",)) < cloud.actions.index(("settle",)) - - -def test_late_io_and_a_peer_retry_cannot_replace_the_expired_original(monkeypatch): - driver, cloud = recovery_driver(monkeypatch) - driver.start() - driver.place() - peer = driver.items["peer"] - cloud.now = peer["deadline"] + 1 - driver.launch("recovery", "barrier", gate=driver.gate("recovery")) - original = peer["native_requests"][0] - peer["native_requests"].append("peer-retry") - cloud.events.append( - { - **next(e for e in cloud.events if e["request"] == original), - "request": "peer-retry", - "time": cloud.now, - "deadline": cloud.now + 60, - } - ) - with pytest.raises(PlacementError, match="Original peer invocation expired"): - driver.wait_overlap(("peer", "recovery"), "environment") - - -def test_original_peer_cannot_expire_while_waiting_for_recovery_completion(monkeypatch): - driver, cloud = recovery_driver(monkeypatch) - driver.start() - driver.place() - finish = cloud.finish - - def delayed_finish(item, status="SUCCEEDED"): - finish(item, status) - if item["marker"] == "recovery": - cloud.now = driver.peer_boundary["deadline"] + 1 - - cloud.finish = delayed_finish - with pytest.raises(PlacementError, match="Original healthy peer expired"): - driver.execute() - assert ("release", driver.gate("peer")) not in cloud.actions - - -def test_expired_victim_does_not_start_a_peer_or_arm_the_fault(monkeypatch): - driver, cloud = recovery_driver(monkeypatch) - driver.start() - cloud.now = cloud.invocations[0]["deadline"] + 1 - with pytest.raises(PlacementError, match="Original victim expired"): - driver.place() - assert [item["marker"] for item in cloud.invocations] == ["victim"] - assert ("release", driver.gate("prepare")) not in cloud.actions diff --git a/lmi_tests/tests/unit/test_shared_adapter.py b/lmi_tests/tests/unit/test_shared_adapter.py deleted file mode 100644 index dcda6ca76..000000000 --- a/lmi_tests/tests/unit/test_shared_adapter.py +++ /dev/null @@ -1,547 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Ownership, native-request journaling and cancellation of the shared driver.""" - -import asyncio -import json -from pathlib import Path -from threading import Event -from unittest.mock import Mock - -import pytest - -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - Scenario, - ScenarioContext, -) -from lmi_tests import cloud as cloud_module -from lmi_tests import shared_adapter as adapter_module -from lmi_tests.evidence import CollectionError, ProvisioningError -from lmi_tests.shared_scenarios import ScenarioDriver - - -FUNCTION = "arn:aws:lambda:us-west-2:123456789012:function:lmi:$LATEST.PUBLISHED" -EXECUTION = FUNCTION + "/durable-execution/name/id" -PROFILE = Profile("native-1", FUNCTION, 1, 1, "sdk-sha", "configuration-sha") - - -def context( - case_id="case", - scenario="suspension-retry", - roles=("victim", "resume"), - *, - native=1, -): - profile = Profile( - f"native-{native}", FUNCTION, native, native, "sdk-sha", "configuration-sha" - ) - return ScenarioContext( - "run", - Scenario( - case_id, - "lmi-lifecycle-5", - profile.profile_id, - scenario, - roles=roles, - coverage_profile="standard", - ), - profile, - ) - - -def deployment(): - return { - "functions": {"c1": FUNCTION, "c2": FUNCTION + "-c2"}, - "concurrencies": {"c1": 1, "c2": 2}, - "region": "us-west-2", - "run": "run", - "bucket": "private-evidence", - "commit": "sdk-sha", - "runtime": "python3.14", - "role": "role", - "provider": "provider", - "codeSha256": "code", - "invocationTimeout": 60, - "executionTimeout": 240, - "driverTimeout": 1, - } - - -@pytest.fixture -def cloud(monkeypatch, tmp_path): - clients = {name: Mock() for name in ("lambda", "s3", "logs")} - clients["lambda"].invoke.return_value = { - "StatusCode": 202, - "DurableExecutionArn": EXECUTION, - } - clients["s3"].get_paginator.return_value.paginate.return_value = [] - monkeypatch.setattr(cloud_module, "client", lambda service: clients[service]) - monkeypatch.setattr( - "aws_durable_execution_sdk_python_testing.runner.boto3.client", - lambda *a, **kw: clients["lambda"], - ) - return cloud_module.Cloud( - deployment(), "c1", artifacts_dir=tmp_path / "case", case_context=context() - ) - - -def enter(item, request, at): - return { - "phase": "WRAPPER_ENTER", - "marker": item["marker"], - "request": request, - "execution": EXECUTION, - "time": at, - "sequence": 0, - "run": "run", - "commit": "sdk-sha", - "runtime": "python3.14", - "role": "role", - "provider": "provider", - "codeSha256": "code", - "invocationTimeout": 60, - "executionTimeout": 240, - } - - -def refresh_entries(cloud, *events): - cloud.events.update({f"events/{event['request']}": event for event in events}) - return cloud.refresh() - - -def test_direct_launch_is_journaled_before_public_runner_transport(cloud): - observed = [] - - def invoke(**kwargs): - observed.append(kwargs) - assert len(cloud.case_context.attempts) == 1 - assert "requestId" not in cloud.case_context.attempts[0] - return {"StatusCode": 202, "DurableExecutionArn": EXECUTION} - - cloud.lam.invoke.side_effect = invoke - item = cloud.start( - "replay", - role="victim", - resume_roles=("resume",), - dispatcher_payload=cloud.case_context.dispatcher("victim"), - ) - payload = json.loads(observed[0]["Payload"]) - assert observed[0]["FunctionName"] == FUNCTION - assert payload["scenario"] == "replay" - assert payload["sharedScenario"] == "suspension-retry" - assert payload["caseId"] == "case" - assert payload["controlPrefix"] == cloud.control_prefix - assert "/cases/" in cloud.control_prefix - assert item["native_requests"] == [] - refresh_entries(cloud, enter(item, "original", 1)) - assert cloud.case_context.attempts[0]["requestId"] == "original" - - -def test_service_replays_are_journaled_without_another_invoke(cloud): - item = cloud.start("replay", role="victim", resume_roles=("resume",)) - refresh_entries( - cloud, - enter(item, "first", 1), - enter(item, "second", 2), - enter(item, "third", 3), - ) - assert cloud.lam.invoke.call_count == 1 - assert [ - (attempt["role"], attempt["attempt"], attempt["requestId"]) - for attempt in cloud.case_context.attempts - ] == [("victim", 1, "first"), ("resume", 1, "second"), ("resume", 2, "third")] - assert cloud.request_roles == { - "first": "victim", - "second": "resume", - "third": "resume", - } - cloud.refresh() - assert len(cloud.case_context.attempts) == 3 - - -def test_lost_invoke_response_retains_launch_and_recovers_identity(cloud): - cloud.lam.invoke.side_effect = TimeoutError("response lost") - with pytest.raises(CollectionError, match="API failed"): - cloud.start("success", role="victim") - item = cloud.invocations[0] - assert item["arn"] is None - record = json.loads( - (cloud.artifacts_dir / f"invocations/{item['marker']}.json").read_text() - ) - assert "response lost" in record["error"] - refresh_entries(cloud, enter(item, "original", 1)) - assert item["arn"] == EXECUTION - assert cloud.case_context.attempts[0]["requestId"] == "original" - assert cloud.lam.invoke.call_count == 1 - - -def test_replacement_execution_cannot_rebind_a_launched_marker(cloud): - item = cloud.start("success", role="victim") - event = enter(item, "replacement", 1) - event["execution"] = EXECUTION + "-other" - with pytest.raises(CollectionError, match="original durable execution"): - refresh_entries(cloud, event) - - -def test_case_controls_and_collection_cannot_touch_another_case(cloud, tmp_path): - cloud.hold("own-gate") - with pytest.raises(CollectionError, match="not owned"): - cloud.release("another-gate") - with pytest.raises(CollectionError, match="another case"): - cloud.refresh(markers=["another-marker"]) - with pytest.raises(CollectionError, match="another case"): - cloud.refresh(markers=[]) - with pytest.raises(CollectionError, match="only its own"): - cloud.collect(full_run=True) - cloud.release_all() - keys = [call.kwargs["Key"] for call in cloud.s3.put_object.call_args_list] - assert all(key.startswith(cloud.control_prefix) for key in keys) - assert not (tmp_path / "events.json").exists() - - -def test_unlaunched_case_does_not_scan_shared_event_prefix(cloud): - assert cloud.refresh() == [] - cloud.s3.get_paginator.assert_not_called() - - -def test_cancellation_stops_waits_and_new_transports(cloud): - cloud.cancel() - with pytest.raises(CollectionError, match="cancelled"): - cloud.wait(30) - with pytest.raises(CollectionError, match="cancelled"): - cloud.start("success", role="victim") - cloud.lam.invoke.assert_not_called() - cloud.begin_retirement() - with pytest.raises(CollectionError, match="retiring"): - cloud.start("success", role="victim") - - -class FakeCloud: - instances = [] - - def __init__(self, manifest, configuration, *, artifacts_dir, case_context=None): - self.manifest, self.configuration = manifest, configuration - self.artifacts_dir, self.case_context = Path(artifacts_dir), case_context - self.cancel_event = Event() - self.actions = [] - self.instances.append(self) - - def verify(self): - self.actions.append("verify") - - def check_active(self): - if self.cancel_event.is_set(): - raise CollectionError("cancelled") - - def cancel(self): - self.cancel_event.set() - - def begin_retirement(self): - self.actions.append("begin-retirement") - self.cancel_event.clear() - - def release_all(self): - self.actions.append("release") - - -def adapter(tmp_path, *, factory=FakeCloud): - return adapter_module.SharedAdapter( - {"sdk": {"deployment": deployment(), "artifacts_dir": str(tmp_path)}}, - evidence_builder=lambda context, cloud, *, phase: { - "retirement": {"owner": context.scenario.case_id, "phase": phase} - }, - cloud_factory=factory, - ) - - -def test_prepare_checks_the_actual_deployment_profile(tmp_path): - value = adapter(tmp_path) - from lmi_tests.shared_contract import configuration_identity - - valid = Profile( - PROFILE.profile_id, - PROFILE.resource_id, - 1, - 1, - "sdk-sha", - configuration_identity(deployment(), "c1"), - ) - asyncio.run(value.prepare(valid)) - with pytest.raises(ProvisioningError, match="configuration identity"): - asyncio.run(value.prepare(PROFILE)) - assert FakeCloud.instances[-1].actions == ["verify"] - bad = Profile("bad", FUNCTION, 1, 2, "sdk-sha", "configuration-sha") - with pytest.raises(ProvisioningError, match="Native concurrency"): - asyncio.run(value.prepare(bad)) - - -def test_foreign_target_prevents_any_driver_launch(tmp_path): - value = adapter(tmp_path) - ctx = context() - ctx.profile = Profile( - "foreign", "arn:foreign", 1, 1, "sdk-sha", "configuration-sha" - ) - with pytest.raises(ProvisioningError, match="outside"): - asyncio.run(value.start(ctx)) - assert value.cases == {} - - -def test_cancelled_worker_is_joined_before_case_cleanup(monkeypatch, tmp_path): - entered, unblock, exited = Event(), Event(), Event() - - class Driver: - def __init__(self, context, cloud, **kwargs): - self.cloud = cloud - - def start(self): - entered.set() - unblock.wait(2) - try: - self.cloud.check_active() - finally: - exited.set() - - def retire(self): - assert exited.is_set() - return True - - monkeypatch.setattr(adapter_module, "ScenarioDriver", Driver) - - async def exercise(): - value = adapter(tmp_path) - ctx = context() - starting = asyncio.create_task(value.start(ctx)) - assert await asyncio.to_thread(entered.wait, 1) - starting.cancel() - with pytest.raises(asyncio.CancelledError): - await starting - releasing = asyncio.create_task(value.release(ctx)) - await asyncio.sleep(0) - assert "release" not in value.cases[ctx.scenario.case_id].driver.cloud.actions - unblock.set() - await releasing - assert exited.is_set() - assert await value.retire(ctx) - assert ctx.evidence["retirement"] == {"owner": "case", "phase": "final"} - - asyncio.run(exercise()) - - -class ScriptCloud: - def __init__(self, native=1): - self.concurrency = native - self.manifest = {"driverTimeout": 1, "invocationTimeout": 60} - self.actions = [] - self.callback_records = [] - self.invocations = [] - self.raw_histories = {} - self.source_uris = set() - - def record_deadline_measurement(self, snapshot): - self.actions.append(("measurement", snapshot)) - - def wait_terminal(self, item): - self.actions.append(("terminal", item["marker"])) - return {"Status": "FAILED"} - - def refresh(self, **kwargs): - return [event for item in self.invocations for event in self.for_item(item)] - - def start(self, fixture, **kwargs): - item = { - "marker": kwargs["role"], - "arn": f"execution:{kwargs['role']}", - "runner": Mock(), - } - item["runner"].wait_for_callback.return_value = "callback-id" - self.actions.append(("start", fixture, kwargs)) - self.invocations.append(item) - return item - - def phase(self, item, phase, **kwargs): - self.actions.append(("phase", item["marker"], phase)) - return [ - { - "phase": phase, - "time": 0, - "deadline": 0, - "environment": "environment", - "request": "request", - "status": "PENDING", - } - ] - - def poll(self, predicate, **kwargs): - self.actions.append(("poll", kwargs.get("message"))) - return predicate() or [ - {"phase": "PLACEMENT_ACCEPTED", "environment": "environment"} - ] - - def release(self, gate): - self.actions.append(("release", gate)) - - def finish(self, item, status="SUCCEEDED"): - self.actions.append(("finish", item["marker"], status)) - - def for_item(self, item): - return [ - { - "phase": "WRAPPER_RETURN", - "status": "PENDING", - "time": 1, - "request": "first", - }, - {"phase": "WRAPPER_ENTER", "request": "first"}, - {"phase": "WRAPPER_ENTER", "request": "second"}, - {"phase": "LATE_REJECTED"}, - {"phase": "ALIVE", "time": 2}, - ] - - def platform_timeout(self, item, request): - self.actions.append(("platform-timeout", item["arn"])) - - def history(self, item): - self.actions.append(("history", item["arn"])) - - def check_active(self): - pass - - -def test_callback_uses_same_execution_and_actual_pending_before_response(): - cloud = ScriptCloud() - driver = ScenarioDriver(context(), cloud) - driver.start() - driver.place() - driver.execute() - assert len(cloud.invocations) == 1 - runner = cloud.invocations[0]["runner"] - runner.wait_for_callback.assert_called_once_with( - "execution:victim", name="completion", timeout=1 - ) - runner.send_callback_success.assert_called_once_with( - "callback-id", result=b"victim" - ) - assert cloud.callback_records[0]["executionArn"] == "execution:victim" - - -def test_each_case_has_independent_artifact_namespace(monkeypatch, tmp_path): - class Driver: - def __init__(self, context, cloud, **kwargs): - self.cloud = cloud - - def start(self): - pass - - monkeypatch.setattr(adapter_module, "ScenarioDriver", Driver) - - async def exercise(): - value = adapter(tmp_path) - await asyncio.gather(value.start(context("one")), value.start(context("two"))) - first = value.cases["one"].driver.cloud - second = value.cases["two"].driver.cloud - assert first.artifacts_dir != second.artifacts_dir - assert first.case_context.scenario.case_id == "one" - assert second.case_context.scenario.case_id == "two" - with pytest.raises(CollectionError, match="replace"): - await value.start(context("one")) - - asyncio.run(exercise()) - - -def settled_cloud(*, view=None): - cloud = ScriptCloud() - cloud.invocations = [{"arn": "execution", "native_requests": ["native"]}] - events = [ - {"phase": "WRAPPER_ENTER", "request": "native"}, - {"phase": "WRAPPER_RETURN", "request": "native"}, - { - "phase": "CONFORMANCE", - "observation": { - "kind": "task_start", - "taskId": "root", - "requestId": "native", - }, - }, - { - "phase": "CONFORMANCE", - "observation": { - "kind": "task_exit", - "taskId": "root", - "requestId": "native", - }, - }, - ] - cloud.refresh = lambda **kwargs: events - driver = ScenarioDriver(context(), cloud) - return driver, events - - -@pytest.mark.parametrize( - "start,end,identity", - [ - ("task_start", "task_exit", "taskId"), - ("io_start", "io_exit", "taskId"), - ("checkpoint_call", "checkpoint_exit", "callId"), - ("poll_call", "poll_exit", "callId"), - ], -) -def test_retirement_requires_every_real_task_and_call_to_exit(start, end, identity): - driver, events = settled_cloud() - assert driver.quiescent() - event = {"requestId": "native", identity: "outstanding"} - events.append({"phase": "CONFORMANCE", "observation": {**event, "kind": start}}) - assert not driver.quiescent() - events.append({"phase": "CONFORMANCE", "observation": {**event, "kind": end}}) - assert driver.quiescent() - - -def test_unidentified_launch_or_missing_root_hook_cannot_claim_retirement(): - driver, events = settled_cloud() - driver.cloud.invocations[0]["native_requests"] = [] - assert not driver.quiescent() - driver.cloud.invocations[0]["native_requests"] = ["native"] - events[:] = [ - event - for event in events - if event.get("observation", {}).get("kind") != "task_exit" - ] - assert not driver.quiescent() - - -def test_wrong_function_response_is_never_adopted_for_case_cleanup(cloud): - cloud.lam.invoke.return_value = { - "StatusCode": 202, - "DurableExecutionArn": "arn:another-function/durable-execution/name/id", - } - with pytest.raises(CollectionError, match="API failed"): - cloud.start("success", role="victim") - assert cloud.invocations[0]["arn"] is None - - -def test_canonical_observations_keep_authoritative_outer_identity_and_hook_mismatch(): - from lmi_tests.shared_scenarios import canonical - - records = canonical( - [ - { - "phase": "CONFORMANCE", - "request": "outer", - "execution": "execution", - "observation": {"kind": "task_exit", "taskId": "root"}, - }, - { - "phase": "CONFORMANCE", - "request": "outer", - "execution": "execution", - "observation": { - "kind": "invocation_start", - "requestId": "wrong-hook-request", - }, - }, - ] - ) - assert records[0]["requestId"] == records[0]["ownerRequestId"] == "outer" - assert records[0]["executionArn"] == "execution" - assert records[1]["requestId"] == "wrong-hook-request" - assert records[1]["ownerRequestId"] == "outer" diff --git a/lmi_tests/tests/unit/test_shared_cleanup.py b/lmi_tests/tests/unit/test_shared_cleanup.py deleted file mode 100644 index 657055168..000000000 --- a/lmi_tests/tests/unit/test_shared_cleanup.py +++ /dev/null @@ -1,167 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Run-scoped cleanup needs no distributed lock and never touches foreign work.""" - -import hashlib -import io -import json -from unittest.mock import Mock - -import pytest - -from lmi_tests import shared -from lmi_tests.evidence import CollectionError - - -def test_no_recorded_cases_skip_aws_cleanup(monkeypatch, tmp_path): - client = Mock(side_effect=AssertionError("No case work was recorded")) - monkeypatch.setattr(shared, "client", client) - assert shared.cleanup({"run": "run", "functions": {}}, tmp_path) == 0 - assert json.loads((tmp_path / "cleanup.json").read_text())["status"] == "NO_CASES" - client.assert_not_called() - - -@pytest.mark.parametrize("foreign", [True, False]) -def test_case_identity_is_checked_before_control_release( - monkeypatch, tmp_path, foreign -): - target = "arn:aws:lambda:us-west-2:123:function:owned:$LATEST.PUBLISHED" - owned = target + "/durable-execution/name/id" - deployment = { - "run": "run", - "bucket": "private", - "region": "us-west-2", - "functions": {"c1": target}, - } - case = {"case_id": "case", "profile_id": "c1"} - plan = { - "runId": "run", - "profiles": [{"profile_id": "c1", "resource_id": target}], - "scenarios": [case], - } - output = tmp_path / "standard" - output.mkdir() - (output / "manifest.json").write_text(json.dumps(plan)) - identity = hashlib.sha256(b"case").hexdigest()[:20] - invocations = output / "source/cases" / identity / "invocations" - invocations.mkdir(parents=True) - (invocations / "intent.json").write_text( - json.dumps( - { - "payload": {"run": "run", "caseId": "case"}, - "arn": "arn:foreign" if foreign else owned, - } - ) - ) - storage, service = Mock(), Mock() - prefix = f"runs/run/cases/{identity}/control/" - storage.get_paginator.return_value.paginate.return_value = [ - {"Contents": [{"Key": prefix + "release-all"}]} - ] - storage.get_object.side_effect = lambda **kwargs: {"Body": io.BytesIO(b"hold")} - service.get_durable_execution.return_value = {"Status": "RUNNING"} - monkeypatch.setattr( - shared, "client", lambda name, region: storage if name == "s3" else service - ) - assert shared.cleanup(deployment, tmp_path) == int(foreign) - report = json.loads((tmp_path / "cleanup.json").read_text()) - assert report["status"] == ("FAILED" if foreign else "CLEANUP_REQUESTED") - if foreign: - storage.put_object.assert_not_called() - service.stop_durable_execution.assert_not_called() - else: - storage.put_object.assert_called_once_with( - Bucket="private", Key=prefix + "release-all", Body=b"release" - ) - service.stop_durable_execution.assert_called_once_with( - DurableExecutionArn=owned - ) - assert all( - call.kwargs["Key"].startswith(prefix) - for call in storage.get_object.call_args_list - ) - - -@pytest.mark.parametrize("wrong", ["run", "target"]) -def test_cleanup_rejects_foreign_manifest_before_aws_calls( - monkeypatch, tmp_path, wrong -): - deployment = {"run": "run", "functions": {"c1": "arn:owned"}} - output = tmp_path / "standard" - output.mkdir() - (output / "manifest.json").write_text( - json.dumps( - { - "runId": "other" if wrong == "run" else "run", - "profiles": [ - { - "profile_id": "c1", - "resource_id": "arn:foreign" - if wrong == "target" - else "arn:owned", - } - ], - "scenarios": [{"profile_id": "c1", "case_id": "case"}], - } - ) - ) - client = Mock(side_effect=AssertionError("Foreign targets must not be touched")) - monkeypatch.setattr(shared, "client", client) - with pytest.raises(CollectionError): - shared.cleanup(deployment, tmp_path) - client.assert_not_called() - - -def test_one_identified_execution_does_not_hide_another_unreconciled_invoke( - monkeypatch, tmp_path -): - from pathlib import Path - - target = "arn:aws:lambda:us-west-2:123:function:owned:$LATEST.PUBLISHED" - execution = target + "/durable-execution/name/id" - deployment = { - "run": "run", - "bucket": "private", - "region": "us-west-2", - "functions": {"c1": target}, - } - case = {"case_id": "case", "profile_id": "c1"} - output = tmp_path / "standard" - output.mkdir() - (output / "manifest.json").write_text( - json.dumps( - { - "runId": "run", - "profiles": [{"profile_id": "c1", "resource_id": target}], - "scenarios": [case], - } - ) - ) - owner = hashlib.sha256(b"case").hexdigest()[:20] - intents = output / "source/cases" / owner / "invocations" - intents.mkdir(parents=True) - known, missing = intents / "known.json", intents / "missing.json" - payload = {"run": "run", "caseId": "case"} - known.write_text( - json.dumps({"payload": {**payload, "marker": "known"}, "arn": execution}) - ) - missing.write_text( - json.dumps({"payload": {**payload, "marker": "missing"}, "arn": None}) - ) - original_glob = Path.glob - monkeypatch.setattr( - Path, - "glob", - lambda path, pattern: ( - iter([known, missing]) if path == intents else original_glob(path, pattern) - ), - ) - storage, service = Mock(), Mock() - storage.get_paginator.return_value.paginate.return_value = [] - service.get_durable_execution.return_value = {"Status": "SUCCEEDED"} - monkeypatch.setattr( - shared, "client", lambda name, region: storage if name == "s3" else service - ) - assert shared.cleanup(deployment, tmp_path) == 1 - errors = json.loads((tmp_path / "cleanup.json").read_text())["errors"] - assert errors == ["Unreconciled Invoke: case/missing"] diff --git a/lmi_tests/tests/unit/test_shared_control_receipts.py b/lmi_tests/tests/unit/test_shared_control_receipts.py deleted file mode 100644 index 553590f5e..000000000 --- a/lmi_tests/tests/unit/test_shared_control_receipts.py +++ /dev/null @@ -1,46 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Control intent and acknowledgement are separate, retained observations.""" - -import json -from threading import Event - -import pytest - -from lmi_tests.cloud import Cloud - - -def test_release_intent_is_saved_before_transport_and_ack_only_after_success(tmp_path): - cloud = object.__new__(Cloud) - cloud.manifest = {"bucket": "private"} - cloud.case_context = object() - cloud.cancel_event = Event() - cloud.gates = {"gate", "second"} - cloud.released_gates = set() - cloud.control_prefix = "runs/run/cases/case/control/" - cloud.control_records = [] - cloud.artifacts_dir = tmp_path - cloud.source_uris = set() - - class S3: - fail = False - - def put_object(self, **kwargs): - recorded = json.loads((tmp_path / "control-records.json").read_text()) - assert recorded[-1]["state"] == "PENDING" - assert "time" not in recorded[-1] - if self.fail: - raise OSError("ambiguous transport failure") - - cloud.s3 = S3() - cloud.release("gate") - successful = cloud.control_records[0] - assert successful["state"] == "ACKNOWLEDGED" - assert successful["started"] <= successful["time"] - cloud.s3.fail = True - cloud.release("gate") - assert len(cloud.control_records) == 1 - with pytest.raises(OSError): - cloud.release("second") - assert cloud.control_records[-1]["state"] == "PENDING" - assert "time" not in cloud.control_records[-1] diff --git a/lmi_tests/tests/unit/test_shared_manifest.py b/lmi_tests/tests/unit/test_shared_manifest.py index 8f6c0ab47..a9b562670 100644 --- a/lmi_tests/tests/unit/test_shared_manifest.py +++ b/lmi_tests/tests/unit/test_shared_manifest.py @@ -122,7 +122,9 @@ async def results(plan, adapter_name, *, audit_dir): client = Mock( side_effect=AssertionError("Run orchestration must not acquire an AWS lock") ) - monkeypatch.setattr(shared, "client", client) + from aws_durable_execution_conformance_tests.lmi import runner + + monkeypatch.setattr(runner, "client", client) monkeypatch.setattr(app, "live_results", results) output = tmp_path / "run" assert shared.run(deployment(), output) == 0 diff --git a/lmi_tests/tests/unit/test_shared_partial_evidence.py b/lmi_tests/tests/unit/test_shared_partial_evidence.py deleted file mode 100644 index 3e76a4a04..000000000 --- a/lmi_tests/tests/unit/test_shared_partial_evidence.py +++ /dev/null @@ -1,159 +0,0 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Partial observations remain separate from facts collected after cleanup.""" - -import asyncio -from copy import deepcopy -from types import SimpleNamespace -from unittest.mock import Mock, patch - - -from aws_durable_execution_conformance_tests.lmi.coordinator import ( - Profile, - Scenario, - ScenarioContext, -) -from lmi_tests.shared_adapter import SharedAdapter, _Case -from lmi_tests.shared_evidence import build_evidence - - -def context(): - profile = Profile("c1", "function", 1, 1, "sdk", "configuration") - return ScenarioContext( - "run", - Scenario( - "case", - "lmi-lifecycle-3", - "c1", - "deadline-during-step", - roles=("victim", "retry"), - coverage_profile="standard", - ), - profile, - ) - - -def adapter(tmp_path, *, read_fails=False): - calls = [] - cloud = SimpleNamespace(observation_ended_at=None) - - def read(**options): - calls.append(("read", options)) - if read_fails: - raise OSError("source unavailable") - - cloud.collect = read - cloud._save = lambda name, value: calls.append(("save", name, deepcopy(value))) - cloud.begin_retirement = lambda: calls.append(("retirement",)) - cloud.release_all = lambda: calls.append(("release",)) - driver = SimpleNamespace( - cloud=cloud, - collect=Mock(side_effect=AssertionError("must not wait for quiescence")), - retire=lambda: True, - ) - - def build(ctx, observed, *, phase): - calls.append(("build", phase)) - return { - "events": [{"kind": "effect"}] - if phase == "observations" - else [{"kind": "effect"}, {"kind": "task_exit"}], - "collection": {"watermarks": {"lifecycle": 1}}, - "collectionErrors": [], - **({"retirement": {"resources": []}} if phase == "final" else {}), - } - - result = SharedAdapter( - {"sdk": {"deployment": {}, "artifacts_dir": str(tmp_path)}}, - evidence_builder=build, - ) - result.cases["case"] = _Case(driver) - return result, cloud, calls - - -def test_partial_capture_reads_once_without_waiting_for_cleanup(tmp_path): - value, cloud, calls = adapter(tmp_path) - partial = asyncio.run(value.collect_partial(context())) - assert [call[0] for call in calls] == ["read", "build", "save"] - assert calls[0] == ("read", {"full_run": False}) - assert calls[-1][1] == "shared-partial-evidence.json" - assert cloud.observation_ended_at is not None - assert partial["events"] == [{"kind": "effect"}] - - -def test_partial_read_error_keeps_cached_observations_but_no_complete_watermark( - tmp_path, -): - value, _, _ = adapter(tmp_path, read_fails=True) - partial = asyncio.run(value.collect_partial(context())) - assert partial["events"] == [{"kind": "effect"}] - assert partial["collection"]["watermarks"] == {} - assert any("source unavailable" in error for error in partial["collectionErrors"]) - - -def test_sdk_retirement_does_not_replace_the_pre_release_events(tmp_path): - async def exercise(): - value, _, _ = adapter(tmp_path) - ctx = context() - ctx.evidence = deepcopy(await value.collect_partial(ctx)) - await value.release(ctx) - assert await value.retire(ctx) is True - assert ctx.evidence["events"] == [{"kind": "effect"}] - assert ctx.retirement_evidence["events"] == [ - {"kind": "effect"}, - {"kind": "task_exit"}, - ] - assert ctx.evidence["retirement"] == ctx.retirement_evidence["retirement"] - - asyncio.run(exercise()) - - -def test_final_read_bound_alone_does_not_invent_quiescence(tmp_path): - ctx = context() - ctx.scenario = Scenario( - "case", - "lmi-lifecycle-1", - "c1", - "native-overlap", - roles=("victim",), - coverage_profile="standard", - ) - cloud = SimpleNamespace( - manifest={"invocationTimeout": 60, "conformanceRevision": "conformance"}, - observation_ended_at=None, - retired_at=2, - source_uris={"memory://source"}, - control_records=[], - invocations=[], - artifacts_dir=tmp_path, - _save=lambda *args: None, - ) - event = { - "kind": "task_start", - "taskId": "root", - "requestId": "request", - "ownerRequestId": "request", - "executionArn": "execution", - "atMs": 1000, - } - sources = SimpleNamespace( - observations=lambda: ([event], []), - histories=lambda _: ([], {}, []), - attempts=lambda _: ( - [{"requestId": "request", "executionArn": "execution"}], - {"victim": "request"}, - [], - ), - ) - cloud.execution_results = {"execution": {"Status": "STOPPED"}} - with patch("lmi_tests.shared_evidence.CaseSources.capture", return_value=sources): - evidence = build_evidence(ctx, cloud, phase="final") - assert evidence["observationEndedAtMs"] == 2000 - assert evidence["collection"]["watermarks"] == {} - assert evidence["collectionErrors"] - invocation = next( - item - for item in evidence["retirement"]["resources"] - if item["kind"] == "invocation" - ) - assert invocation["outcome"] == "UNKNOWN" diff --git a/lmi_tests/tests/unit/test_shared_workflow.py b/lmi_tests/tests/unit/test_shared_workflow.py index b7751a67b..221d51701 100644 --- a/lmi_tests/tests/unit/test_shared_workflow.py +++ b/lmi_tests/tests/unit/test_shared_workflow.py @@ -1,107 +1,10 @@ -# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Cloud acceptance must use the same pinned runner as local adapter tests.""" - from pathlib import Path -import tomllib - import yaml - from lmi_tests.shared_contract import CONFORMANCE_REVISION ROOT = Path(__file__).resolve().parents[3] -def test_cloud_workflow_uses_pinned_shared_acceptance_and_final_cleanup(): - workflow = yaml.safe_load( - (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() - ) - assert workflow["env"]["CONFORMANCE_REVISION"] == CONFORMANCE_REVISION - assert "workflow_dispatch" in workflow["jobs"]["cloud"]["if"] - for job in workflow["jobs"].values(): - checkout = next( - step - for step in job["steps"] - if step.get("with", {}).get("repository") - == "aws/aws-durable-execution-conformance-tests" - ) - assert checkout["with"]["ref"] == "${{ env.CONFORMANCE_REVISION }}" - assert any( - step.get("run") == "hatch run lmi:shared-setup" for step in job["steps"] - ) - steps = workflow["jobs"]["cloud"]["steps"] - acceptance = next( - step for step in steps if step.get("run") == "hatch run lmi:shared" - ) - assert acceptance["timeout-minutes"] == 60 - cleanup = next( - step for step in steps if "lmi:shared-cleanup" in step.get("run", "") - ) - assert cleanup["if"] == "always()" - assert not any("lmi:cloud" in step.get("run", "") for step in steps) - scripts = tomllib.loads((ROOT / "pyproject.toml").read_text())["tool"]["hatch"][ - "envs" - ]["lmi"]["scripts"] - assert scripts["shared"].startswith("python -m lmi_tests.shared run") - assert scripts["shared-cleanup"].startswith("python -m lmi_tests.shared cleanup") - - -def test_codebuild_runner_has_isolated_python_and_keeps_untrusted_prs_hosted(): - workflow = yaml.safe_load( - (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() - ) - assert workflow["jobs"]["harness"]["runs-on"] == ( - "${{ github.event_name == 'pull_request' && " - "(github.event.pull_request.head.repo.full_name != github.repository || " - "github.actor == 'dependabot[bot]') && 'ubuntu-latest' || " - "format('codebuild-github-actions-runner-{0}-{1}', " - "github.run_id, github.run_attempt) }}" - ) - assert workflow["jobs"]["cloud"]["runs-on"] == ( - "codebuild-github-actions-runner-${{ github.run_id }}-${{ github.run_attempt }}" - ) - assert workflow["env"]["PIP_CONFIG_FILE"] == "/dev/null" - assert workflow["env"]["PYTHONPATH"] == "" - for job in workflow["jobs"].values(): - setup = next( - step - for step in job["steps"] - if step.get("uses", "").startswith("astral-sh/setup-uv@") - ) - assert setup["with"]["python-version"] == "3.14" - assert setup["with"]["activate-environment"] is True - assert any( - step.get("run") == "uv pip install hatch==1.16.5" for step in job["steps"] - ) - assert not any( - step.get("uses", "").startswith("actions/setup-python@") - for step in job["steps"] - ) - - -def test_cloud_runs_automatically_for_trusted_prs_and_supports_manual_dispatch(): - workflow = yaml.safe_load( - (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() - ) - triggers = workflow.get("on", workflow.get(True)) - assert set(triggers["pull_request"]["types"]) == { - "opened", - "synchronize", - "reopened", - } - assert "workflow_dispatch" in triggers - cloud = workflow["jobs"]["cloud"] - assert cloud["if"] == ( - "github.actor != 'dependabot[bot]' && " - "(github.event_name == 'workflow_dispatch' || " - "(github.event_name == 'pull_request' && " - "github.event.pull_request.head.repo.full_name == github.repository))" - ) - assert cloud["needs"] == "harness" - assert cloud["concurrency"]["group"] == "lmi-e2e-shared-capacity-provider" - assert cloud["concurrency"]["cancel-in-progress"] is False - - def test_shared_setup_bootstraps_before_conformance_is_installed(): import subprocess import sys @@ -134,3 +37,70 @@ def find_spec(self, fullname, path=None, target=None): timeout=20, ) assert completed.returncode == 0, completed.stdout + completed.stderr + + +def test_workflow_calls_the_pinned_shared_definition(): + workflow = yaml.safe_load( + (ROOT / ".github/workflows/lmi-e2e-tests.yml").read_text() + ) + caller = workflow["jobs"]["lifecycle"] + assert list(workflow["jobs"]) == ["lifecycle"] + assert ( + caller["uses"] + == f"aws/aws-durable-execution-conformance-tests/.github/workflows/lmi-lifecycle.yml@{CONFORMANCE_REVISION}" + ) + assert caller["with"]["conformance-ref"] == CONFORMANCE_REVISION + assert (ROOT / caller["with"]["sdk-script"]).is_file() + assert set(caller["secrets"]) == { + "TEST_ROLE_ARN", + "TEST_ACCOUNT_ID", + "TEST_LAMBDA_EXECUTION_ROLE_ARN", + "CAPACITY_PROVIDER_ARN", + } + triggers = workflow.get("on", workflow.get(True)) + assert set(triggers["pull_request"]["types"]) == { + "opened", + "synchronize", + "reopened", + } + assert "workflow_dispatch" in triggers + + +def test_stage_dispatch_propagates_failure_and_preserves_deployment_identity(tmp_path): + import os + import subprocess + + fake = tmp_path / "hatch" + fake.write_text( + '#!/bin/bash\nprintf "%s\\n" "$@" > "$CALL_LOG"\nexit "${HATCH_RESULT:-0}"\n' + ) + fake.chmod(0o755) + calls = tmp_path / "calls.txt" + env = { + **os.environ, + "PATH": str(tmp_path) + ":" + os.environ["PATH"], + "CALL_LOG": str(calls), + "LMI_SHARED_RUN_ID": "123-2", + "LMI_RUNTIME": "python3.14", + "LMI_STACK_NAME": "python-lmi-e2e", + } + script = ROOT / ".github/scripts/lmi-ci.sh" + subprocess.run(["bash", str(script), "deploy"], env=env, check=True) + assert calls.read_text().splitlines() == [ + "run", + "lmi:python", + "-m", + "lmi_tests.deploy", + "deploy", + "--run-id", + "123-2", + "--runtime", + "python3.14", + "--stack-name", + "python-lmi-e2e", + ] + failed = subprocess.run( + ["bash", str(script), "run"], env={**env, "HATCH_RESULT": "17"}, check=False + ) + assert failed.returncode == 17 + assert calls.read_text().splitlines() == ["run", "lmi:shared"]