diff --git a/.gitignore b/.gitignore index 258a8c07042..9a528c309bb 100644 --- a/.gitignore +++ b/.gitignore @@ -207,6 +207,10 @@ temp*/ # AI **/.checkpoints/ +# Local AgentServer file store + crash-recovery HOME roots used by hosted samples +**/.agentserver-state/ +**/.agentserver-state-*/ +**/.home-*/ .claude/ .omc/ .omx/ diff --git a/docs/decisions/0035-foundry-hosting-resilient-long-running-agents.md b/docs/decisions/0035-foundry-hosting-resilient-long-running-agents.md new file mode 100644 index 00000000000..2f57ac194f3 --- /dev/null +++ b/docs/decisions/0035-foundry-hosting-resilient-long-running-agents.md @@ -0,0 +1,191 @@ +--- +status: proposed +contact: rogerbarreto +date: 2026-08-21 +deciders: rogerbarreto +consulted: Tao Chen, Sergey M., Ben Thomas, Shanmukha +informed: Agent Framework .NET team +--- + +# Resilient long-running agents in Microsoft.Agents.AI.Foundry.Hosting + +## Context and Problem Statement + +The Foundry Hosted Agents platform can run a hosted agent as a long job that continues when no +client is connected, and that the platform restarts after the container crashes or is recycled. +On restart the platform re-invokes the handler with the same input, sets `ResponseContext.IsRecovery` +to true, and supplies the last durable `ResponseObject` snapshot as `PersistedResponse`. The +snapshot is not itself a workflow checkpoint. For workflow agents, hosting records the ID of the +matching workflow checkpoint inside AgentServer internal response metadata before it persists the +response snapshot. + +This applies only to **background** requests (`background=true`) whose `store` value is omitted or +true. Omitted `store` uses the Responses API default of true. Foreground requests and explicit +`store=false` requests have no crash-recovery contract. + +Python currently supports resilient background execution for workflow agents and steering for +single agents. .NET hosting must offer the same opt-in capabilities on top of the durable session +and checkpoint storage introduced for Foundry state stores (PR #7649). + +## Decision Drivers + +- Match the Python recovery contract. +- Pair each persisted workflow response snapshot with the exact workflow checkpoint it represents. +- Opt-in and off by default; non-resilient hosts pay nothing. +- Prefer workflows: they already checkpoint between supersteps. +- Keep a lean API on `FoundryResponsesOptions`, forwarded to `ResponsesServerOptions`. +- Persist agent sessions through the Foundry state store (or its local fallback), not a second disk layout. + +## Decision Outcome + +Chosen option: **turn resilience on through the existing handler and registration path**. + +### Public surface + +`FoundryResponsesOptions.ResilientBackground` and `FoundryResponsesOptions.SteerableConversations` +are forwarded to `ResponsesServerOptions` so the AgentServer SDK enables recovery and steering. +This forwarding must happen in the callback passed directly to `AddResponsesServer`. The SDK makes +two process-level choices during that registration call: whether local SSE replay uses durable +storage and whether the conversation task accepts steering. Configuring the options only through +the later `IOptions` pipeline is too late for those choices. + +The first `AddFoundryResponses` call owns this host-level configuration. Repeated calls do not +register another Responses server or redefine its resilience mode. Later calls can still configure +MAF-only options such as `AllowStoredOutputEnabled`; attempting to enable an AgentServer task +feature after the first call fails immediately instead of leaving AgentServer and MAF with +different settings. + +```csharp +builder.Services.AddFoundryResponses(agent, configure: o => o.ResilientBackground = true); +``` + +### Handler contract on recovery + +When `IsRecovery` is true: + +1. Seed `ResponseEventStream` from the `PersistedResponse` that AgentServer provides. This preserves + its response fields, completed output items, and internal metadata. +2. When the snapshot contains `_last_checkpoint_id` and a persisted workflow `AgentSession` was + restored, select that exact checkpoint as the workflow resume point. This prevents a newer + checkpoint already present in workflow storage from being combined with an older response + snapshot. Foundry Hosting obtains the experimental `WorkflowSessionCheckpointRecovery` service + from the restored `AgentSession`; the internal `WorkflowSession` remains hidden. The resumed run + continues the work already queued in that checkpoint without sending a new `TurnToken` to the + start executor. +3. When `_last_checkpoint_id` is absent, retain the checkpoint already referenced by the restored + session. This covers a crash after the workflow wrote its first checkpoint but before AgentServer + persisted the first paired response snapshot. If the process stopped before the first session + save, no resumable MAF state exists, so the handler re-injects the original input instead of + invoking a fresh session with no messages. A regular agent has no equivalent within-turn workflow + checkpoint, so recovery remains best-effort and depends on its serialized session state. +4. On graceful shutdown of a resilient turn, call `ExitForRecoveryAsync` instead of emitting + incomplete. The AgentServer shutdown token is linked to the token passed into the MAF agent so + long-running model, tool, and workflow operations stop promptly. The handler also checks + `IsShutdownRequested` after each agent update, because an agent may consume cancellation and + return normally instead of throwing. If shutdown becomes visible after the agent advanced but + before the corresponding event was emitted, the final session save is skipped. Recovery uses + the last session snapshot that corresponds to output already handed to AgentServer. +5. For non-workflow agents, best-effort save the agent session after each + `ResponseOutputItemDoneEvent`, with an authoritative end-of-turn save in `finally` (skipped when + the turn failed). Workflow agents use only the paired superstep path below for incremental saves, + so their persisted session cannot advance independently through ordinary output-item saves. + +### Workflow response checkpoint alignment + +When `OutputConverter` receives a `SuperStepCompletedEvent` with a new workflow checkpoint ID: + +1. Close any response output item still open for that superstep. +2. Compare the new ID with `_last_checkpoint_id` in `ResponseEventStream.InternalMetadata`. If they + match, do nothing. +3. Save the `AgentSession` that references the new workflow checkpoint. If this save fails, keep the + prior response snapshot and metadata. The turn continues, and a later workflow checkpoint or the + final save can try again. +4. Write the new ID to `_last_checkpoint_id`. +5. Emit `response.in_progress` with the updated response state. AgentServer beta.8 tracks a + separate authoritative response object, so this event copies the internal metadata into the + snapshot that its checkpoint operation persists. The reserved metadata remains stripped from + client payloads. +6. Yield `ResponseEventStream.Checkpoint()`. AgentServer persists the response snapshot before it + resumes the handler. + +The workflow checkpoint itself is already durable before `SuperStepCompletedEvent` is emitted. The +session save and response checkpoint therefore establish a recoverable boundary with three matching +parts: completed response output, serialized session state, and workflow checkpoint ID. + +If a crash occurs after the workflow creates a newer checkpoint but before the next response +checkpoint, recovery deliberately uses the older ID from `PersistedResponse`. The workflow may +repeat work after that older boundary, but it does not duplicate output already present in the +response snapshot or lose output by resuming ahead of it. + +### Handler contract on steering + +When a second input arrives for an active steerable conversation: + +1. AgentServer returns a response with `status=queued`, records the input, increments + `PendingInputCount` on the active handler context, and signals that handler's cancellation token. +2. The superseded handler invocation has `IsSteeredTurn=false`. If a cancellation-aware MAF + operation throws `OperationCanceledException`, Foundry Hosting uses `PendingInputCount > 0` to + distinguish steering from shutdown and client cancellation. +3. Foundry Hosting completes the superseded response cleanly and saves its `AgentSession` with a + non-cancelled save token. This gives the queued turn the latest committed MAF state. +4. AgentServer invokes the handler again with `IsSteeredTurn=true`. This is not crash recovery: + `IsRecovery=false`, so the new input is converted to MAF messages normally. The same + `conversation_id` resolves the same persisted `AgentSession`. + +No special MAF branch is required merely because `IsSteeredTurn=true`. The classification is +available for handlers that need different application behavior; the generic adapter treats the +drained input as the next normal turn on the same session. + +Steering does not create a response checkpoint merely because another input was queued. Completed +workflow supersteps have already been paired with response checkpoints. An interrupted superstep +has no new `SuperStepCompletedEvent`, so its partial output and session state do not advance the +paired recovery boundary. The superseded response still reaches a terminal `completed` event. + +### State ownership + +| State | Owner | Recovery purpose | +|---|---|---| +| Resilient task, SSE events, `ResponseObject` snapshots, `_last_checkpoint_id` | AgentServer | Re-invoke the handler and identify the workflow checkpoint represented by each response snapshot | +| Serialized `AgentSession` | Foundry Hosting | Restore agent-owned state and the workflow checkpoint reference | +| Workflow execution checkpoints | Workflow runtime through `FoundryJsonCheckpointStore` | Restore executors, queued messages, pending requests, and workflow state | + +The handler calls `ResponseEventStream.Checkpoint()` only after a workflow superstep supplies a new +checkpoint ID and the matching `AgentSession` save succeeds. `PersistedResponse.Output.Count` is not +the workflow cursor. `_last_checkpoint_id` is the explicit link between the response snapshot and +workflow storage. + +### Relationship to durable storage (PR #7649) + +Sessions and workflow checkpoints already go through `FoundryAgentSessionStore` / +`FoundryJsonCheckpointStore`. AgentServer separately owns resilient task records, response snapshots, +and SSE event replay. Resilience does not invent another store; it coordinates handler re-entry with +the existing session and workflow stores. + +## Consequences + +- Samples: `Hosted-Workflow-Resilient`, `Hosted-Workflow-Resilient-Long-Running`, and + `Hosted-Steering`. +- `Using-E2E-Resilience` runs the complete local crash-recovery demonstration in one console: + it consumes the server through a MAF agent created by `AIProjectClient`, force-kills the process, + restarts it, reconnects with a sequence-aware `ResponseContinuationToken`, then uses a third call + on the same agent and session without a sequence cursor to replay the full stream. It validates + the exact final countdown against the client accumulator and cursor-free replay. +- Handler-level tests cover recovery input skip, consumption of an available response snapshot, + response checkpoint deduplication by workflow checkpoint ID, and session-save failure that keeps + the prior paired boundary. +- A local two-lifetime integration test starts a real Responses host, persists a MAF + `AgentSession`, stops the host, starts a new host over the same local AgentServer state, and + verifies that the same response completes without re-injecting the original input. +- A deterministic countdown recovery test interrupts a workflow after outputs `6`, `5`, and `4`, + starts a new host, and verifies the final output is exactly `6`, `5`, `4`, `3`, `2`, `1`, + `Countdown complete.` with no missing or duplicated items. +- A local steering integration test sends two real HTTP turns through AgentServer and the MAF + adapter. It verifies `queued`, serial execution, delivery of the steering input, and reuse of the + persisted session. +- Live Foundry tests cover background continuation without client traffic, hard process + termination through `Environment.Exit`, recovery in a different process incarnation, transient + `404`/`424` polling responses during replacement, and long-running steering on the same + conversation. +- The checkpoint-index optimistic-concurrency retry count is configurable through + `FoundryJsonCheckpointStore`, with a default of eight attempts. +- Package floor: Azure.AI.AgentServer Core beta.28, Invocations beta.6, Responses beta.8. diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index b3b71c1dc71..403c61f54f5 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -377,11 +377,23 @@ + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.agentignore b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.agentignore new file mode 100644 index 00000000000..3a44251d6c5 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.agentignore @@ -0,0 +1,22 @@ +# azd tooling files +azure.yaml +.agentignore + +# Security / secrets +.env +.env.* +.azure/ +.git/ + +# .NET build output +bin/ +obj/ +*.user +*.suo +.vs/ + +# Local agent state +.checkpoints/ +.agentserver-state/ +.agentserver-state-*/ +.home-*/ diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example new file mode 100644 index 00000000000..7bc60bb435f --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/.env.example @@ -0,0 +1,9 @@ +# Foundry project endpoint +FOUNDRY_PROJECT_ENDPOINT= + +# Model deployment name +AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o + +# Local development only +ASPNETCORE_URLS=http://+:8088 +AZURE_TOKEN_CREDENTIALS=dev diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj new file mode 100644 index 00000000000..2ae94932bc4 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/HostedSteering.csproj @@ -0,0 +1,40 @@ + + + + false + + + + + + net10.0 + + enable + enable + HostedSteering + HostedSteering + 8197fe92-5ccf-45fd-ab1e-f45755ef3a48 + 1.18.0-preview.260818.1 + $(MSBuildThisFileDirectory)..\..\..\..\..\src + true + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs new file mode 100644 index 00000000000..aa5df92e3e8 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/Program.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Sample: a Foundry Hosted Agent that accepts steering input while a response is still running. +// It deploys directly from source, so Foundry builds and runs the uploaded project. + +using Azure.AI.Projects; +using Azure.Identity; +using DotNetEnv; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry.Hosting; + +Env.TraversePath().Load(); + +var projectEndpoint = new Uri(System.Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.")); +var deployment = FirstNonBlank( + System.Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME"), + System.Environment.GetEnvironmentVariable("FOUNDRY_MODEL"), + "gpt-4o"); +var agentName = System.Environment.GetEnvironmentVariable("AGENT_NAME") ?? "hosted-steering"; + +AIAgent agent = new AIProjectClient(projectEndpoint, new DefaultAzureCredential()) + .AsAIAgent( + model: deployment, + instructions: """ + You are a helpful AI assistant. When another message arrives while you are working, + treat it as a course correction and incorporate it into the answer. + """, + name: agentName, + description: "A steerable general-purpose AI assistant"); + +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddFoundryResponses(agent, configure: options => options.SteerableConversations = true); + +var app = builder.Build(); +app.MapFoundryResponses(); +app.Run(); + +static string FirstNonBlank(params string?[] candidates) => + Array.Find(candidates, candidate => !string.IsNullOrWhiteSpace(candidate))!; diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md new file mode 100644 index 00000000000..1dc0a3372a3 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/README.md @@ -0,0 +1,76 @@ +# Hosted-Steering + +A Foundry Hosted Agent with steerable conversations enabled. When a second input arrives while a +conversation turn is running, AgentServer queues it instead of returning `conversation_locked`. + +This sample deploys directly from source. Foundry uploads the project as a ZIP, restores its +packages, builds it, and runs `HostedSteering.dll`. No Dockerfile or container registry is needed. + +## Key setting + +```csharp +builder.Services.AddFoundryResponses( + agent, + configure: options => options.SteerableConversations = true); +``` + +Steering and resilient background execution are separate options. This sample enables only +steering. See [Hosted-Workflow-Resilient](../Hosted-Workflow-Resilient/README.md) for crash recovery. + +## Local development + +Copy `.env.example` to `.env`, set the project endpoint and model deployment, then run: + +```powershell +az login +dotnet run --tl:off +``` + +The in-repository project automatically uses ProjectReference to run the current framework source. + +## Deploy from source + +Create an empty working directory outside the repository: + +```powershell +$work = Join-Path $env:TEMP "hosted-steering-work" +New-Item -ItemType Directory -Path $work -Force | Out-Null +Set-Location $work + +$sample = "/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/azure.yaml" +azd auth login +azd ai agent init -m $sample -d +``` + +### Contributors testing framework changes + +**Skip this section unless you are testing an Agent Framework change from the current codebase that +has not been released yet.** The normal deployment uses the published packages. To test local +framework changes, pack the current repository source into the scaffolded upload before provisioning: + +```powershell +/dotnet/samples/04-hosting/FoundryHostedAgents/scripts/Add-LocalFrameworkFeed.ps1 ` + -Path ./hosted-steering +``` + +The helper creates `local-feed/`, writes `nuget.config`, and changes `AgentFrameworkVersion` in the +scaffolded project. Both generated artifacts are included in the source ZIP. + +```powershell +Set-Location hosted-steering +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME +azd provision +azd deploy +``` + +## Exercise steering + +Start a stored background response, keep its response or conversation identity, then submit a second +input to the same in-progress conversation. The second request should be queued instead of rejected. +Use the Responses API or an OpenAI-compatible client that exposes background and conversation fields. + +## Related samples + +- [Hosted-ChatClientAgent](../Hosted-ChatClientAgent/README.md): basic source-deployed agent. +- [Hosted-Workflow-Resilient](../Hosted-Workflow-Resilient/README.md): resilient background workflow. +- [Hosted-Workflow-Resilient-Long-Running](../Hosted-Workflow-Resilient-Long-Running/README.md): deterministic countdown recovery with exact output validation. diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/azure.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/azure.yaml new file mode 100644 index 00000000000..94a968272fd --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Steering/azure.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: hosted-steering +services: + ai-project: + host: azure.ai.project + hosted-steering: + project: . + host: azure.ai.agent + language: csharp + uses: + - ai-project + codeConfiguration: + dependencyResolution: remote_build + entryPoint: HostedSteering.dll + runtime: dotnet_10 + env: + ASPNETCORE_URLS: http://+:8088 + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${AZURE_AI_MODEL_DEPLOYMENT_NAME} + container: + resources: + cpu: "0.5" + memory: 1Gi + description: | + A Foundry Hosted Agent that accepts steering input while a response is still running. + kind: hosted + metadata: + tags: + - AI Agent Hosting + - Azure AI AgentServer + - Agent Framework + - Steering + name: hosted-steering + protocols: + - protocol: responses + version: 2.0.0 diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.agentignore b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.agentignore new file mode 100644 index 00000000000..1ab7f4e0225 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.agentignore @@ -0,0 +1,22 @@ +# azd tooling files +azure.yaml +.agentignore + +# Security / secrets +.env +.env.* +.azure/ +.git/ + +# .NET build output +bin/ +obj/ +*.user +*.suo +.vs/ + +# Local agent state +.checkpoints/ +.agentserver-state/ +.agentserver-state-*/ +.home-*/ diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.env.example b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.env.example new file mode 100644 index 00000000000..1104c2d65ba --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/.env.example @@ -0,0 +1,5 @@ +# Optional local countdown delay +COUNTDOWN_DELAY_SECONDS=1 + +# Local development only +ASPNETCORE_URLS=http://+:8088 diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/HostedWorkflowResilientLongRunning.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/HostedWorkflowResilientLongRunning.csproj new file mode 100644 index 00000000000..57d2acbe972 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/HostedWorkflowResilientLongRunning.csproj @@ -0,0 +1,38 @@ + + + + false + + + + + + net10.0 + + enable + enable + HostedWorkflowResilientLongRunning + HostedWorkflowResilientLongRunning + 440f42c9-f64e-441e-9d92-ea814203075e + 1.18.0-preview.260818.1 + $(MSBuildThisFileDirectory)..\..\..\..\..\src + true + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/Program.cs new file mode 100644 index 00000000000..0928eac8cb6 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/Program.cs @@ -0,0 +1,137 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Sample: a long-running countdown workflow hosted as a resilient background response. +// Each completed superstep is paired with an AgentServer response checkpoint so a restarted +// process resumes with ordered output and without losing or duplicating countdown items. + +using System.Globalization; +using System.Text.RegularExpressions; +using DotNetEnv; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry.Hosting; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +Env.TraversePath().Load(); + +var agentName = System.Environment.GetEnvironmentVariable("AGENT_NAME") + ?? "hosted-workflow-resilient-long-running"; +var delaySeconds = int.TryParse( + System.Environment.GetEnvironmentVariable("COUNTDOWN_DELAY_SECONDS"), + NumberStyles.None, + CultureInfo.InvariantCulture, + out int configuredDelaySeconds) + ? configuredDelaySeconds + : 1; +if (delaySeconds < 0) +{ + throw new InvalidOperationException("COUNTDOWN_DELAY_SECONDS must be zero or greater."); +} + +var start = new CountdownStartExecutor(); +var countdown = new CountdownExecutor(TimeSpan.FromSeconds(delaySeconds)); +var complete = new CountdownCompleteExecutor(); + +Workflow workflow = new WorkflowBuilder(start) + .AddEdge(start, countdown) + .AddEdge(countdown, countdown) + .AddEdge(countdown, complete) + .WithOutputFrom(start, countdown, complete) + .Build(); + +AIAgent agent = workflow.AsAIAgent( + id: agentName, + name: agentName, + includeExceptionDetails: true, + includeWorkflowOutputsInResponse: true); + +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddFoundryResponses( + agent, + configure: options => options.ResilientBackground = true); + +var app = builder.Build(); +app.MapFoundryResponses(); +if (app.Environment.IsDevelopment()) +{ + app.MapFoundryResponses("openai/v1"); +} + +Console.WriteLine($"Process ID: {System.Environment.ProcessId}"); +app.Run(); + +[SendsMessage(typeof(int))] +[YieldsOutput(typeof(string))] +internal sealed partial class CountdownStartExecutor() : ChatProtocolExecutor( + "start", + new ChatProtocolExecutorOptions { AutoSendTurnToken = false }) +{ + protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder) => + base.ConfigureProtocol(protocolBuilder).SendsMessage(); + + protected override async ValueTask TakeTurnAsync( + List messages, + IWorkflowContext context, + bool? emitEvents, + CancellationToken cancellationToken = default) + { + string input = string.Join( + System.Environment.NewLine, + messages.Select(message => message.Text).Where(text => !string.IsNullOrWhiteSpace(text))); + Match match = PositiveIntegerRegex().Match(input); + if (!match.Success + || !int.TryParse(match.Value, NumberStyles.None, CultureInfo.InvariantCulture, out int target) + || target <= 0) + { + await context.YieldOutputAsync( + "The message must contain a positive integer counter target.", + cancellationToken); + return; + } + + await context.SendMessageAsync(target, cancellationToken: cancellationToken); + } + + [GeneratedRegex(@"(?("countdown") +{ + public override async ValueTask HandleAsync( + int message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + if (message <= 0) + { + await context.SendMessageAsync( + "Countdown complete.", + targetId: "complete", + cancellationToken: cancellationToken); + return; + } + + await Task.Delay(delay, cancellationToken); + await context.YieldOutputAsync( + message.ToString(CultureInfo.InvariantCulture), + cancellationToken); + await context.SendMessageAsync( + message - 1, + targetId: "countdown", + cancellationToken: cancellationToken); + } +} + +[YieldsOutput(typeof(string))] +internal sealed class CountdownCompleteExecutor() : Executor("complete") +{ + public override ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) => + context.YieldOutputAsync(message, cancellationToken); +} diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/README.md new file mode 100644 index 00000000000..3b2b20498b1 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/README.md @@ -0,0 +1,117 @@ +# Hosted-Workflow-Resilient-Long-Running + +A deterministic countdown workflow that demonstrates resilient background execution. Each number is +one workflow output item. If the process stops, AgentServer restores the last response snapshot and +the workflow resumes from the exact workflow checkpoint ID recorded in that snapshot. + +For an input such as `Count down from 6`, the final message outputs are: + +```text +6 +5 +4 +3 +2 +1 +Countdown complete. +``` + +The exact list makes recovery errors visible. A missing item means response state advanced beyond +workflow state. A repeated item means the workflow resumed before the response snapshot boundary. + +## Workflow + +| Executor | Behavior | +| --- | --- | +| `start` | Reads the first positive integer from the request. | +| `countdown` | Waits, yields the current number, decrements it, and sends it back to itself. | +| `complete` | Yields `Countdown complete.` after the counter reaches zero. | + +All executor IDs and the workflow agent ID are stable so a replacement process reconstructs the same +workflow topology. + +## Recovery boundary + +At every completed workflow superstep, Foundry Hosting: + +1. Closes the response output item produced by that superstep. +2. Saves the matching AgentSession. +3. Writes the workflow checkpoint ID to AgentServer internal response metadata as + `_last_checkpoint_id`. +4. Emits the updated `response.in_progress` state so AgentServer's authoritative response includes + the internal metadata. +5. Yields `ResponseEventStream.Checkpoint()`. + +On recovery, the handler reads `_last_checkpoint_id` from `PersistedResponse` and selects that exact +workflow checkpoint before execution continues. + +## Local development + +The easiest local demonstration is the automated E2E console: + +```powershell +dotnet run --project dotnet\samples\04-hosting\FoundryHostedAgents\responses\Using-E2E-Resilience +``` + +It starts this server, prints countdown outputs, force-kills the process, restarts it, prints replay +and recovery outputs, and validates the final sequence. + +To run only the server, copy `.env.example` to `.env`, then run: + +```powershell +dotnet run --tl:off +``` + +Set `COUNTDOWN_DELAY_SECONDS=0` to make a normal run complete immediately. + +## Deploy from source + +Create an empty working directory outside the repository: + +```powershell +$work = Join-Path $env:TEMP "hosted-workflow-resilient-long-running-work" +New-Item -ItemType Directory -Path $work -Force | Out-Null +Set-Location $work + +$sample = "/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/azure.yaml" +azd auth login +azd ai agent init -m $sample +``` + +### Contributors testing framework changes + +Skip this section unless the current framework changes have not been released. Pack the repository +source into the scaffolded upload before provisioning: + +```powershell +/dotnet/samples/04-hosting/FoundryHostedAgents/scripts/Add-LocalFrameworkFeed.ps1 ` + -Path ./hosted-workflow-resilient-long-running +``` + +Then deploy: + +```powershell +Set-Location hosted-workflow-resilient-long-running +azd provision +azd deploy +``` + +Grant the hosted agent identity `Foundry User` on the Foundry project so it can write workflow +checkpoints and AgentSession state. + +## Automated coverage + +`ResilientTwoLifetimeIntegrationTests.StoppedHost_RecoversWorkflowWithCompleteOrderedOutputAsync` +starts the Responses host twice over shared durable state. It interrupts the first host while the +counter is processing `3`, then verifies that the recovered response contains exactly: + +```text +6, 5, 4, 3, 2, 1, Countdown complete. +``` + +## Related samples + +- [Using-E2E-Resilience](../Using-E2E-Resilience/README.md): automated local crash-recovery console. +- [Hosted-Workflow-Resilient](../Hosted-Workflow-Resilient/README.md): resilient model-backed translation workflow. +- [Hosted-Workflow-Simple](../Hosted-Workflow-Simple/README.md): workflow hosting without resilient background execution. +- [Hosted-Steering](../Hosted-Steering/README.md): mid-turn steering. diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/azure.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/azure.yaml new file mode 100644 index 00000000000..55281bc1c08 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient-Long-Running/azure.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: hosted-workflow-resilient-long-running +services: + ai-project: + host: azure.ai.project + hosted-workflow-resilient-long-running: + project: . + host: azure.ai.agent + language: csharp + uses: + - ai-project + codeConfiguration: + dependencyResolution: remote_build + entryPoint: HostedWorkflowResilientLongRunning.dll + runtime: dotnet_10 + env: + ASPNETCORE_URLS: http://+:8088 + COUNTDOWN_DELAY_SECONDS: "1" + container: + resources: + cpu: "0.5" + memory: 1Gi + description: | + A resilient long-running countdown workflow hosted with the Foundry Responses protocol. + kind: hosted + metadata: + tags: + - AI Agent Hosting + - Azure AI AgentServer + - Agent Framework + - Resilient Background + - Long Running + name: hosted-workflow-resilient-long-running + protocols: + - protocol: responses + version: 2.0.0 diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.agentignore b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.agentignore new file mode 100644 index 00000000000..3a44251d6c5 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.agentignore @@ -0,0 +1,22 @@ +# azd tooling files +azure.yaml +.agentignore + +# Security / secrets +.env +.env.* +.azure/ +.git/ + +# .NET build output +bin/ +obj/ +*.user +*.suo +.vs/ + +# Local agent state +.checkpoints/ +.agentserver-state/ +.agentserver-state-*/ +.home-*/ diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example new file mode 100644 index 00000000000..7bc60bb435f --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/.env.example @@ -0,0 +1,9 @@ +# Foundry project endpoint +FOUNDRY_PROJECT_ENDPOINT= + +# Model deployment name +AZURE_AI_MODEL_DEPLOYMENT_NAME=gpt-4o + +# Local development only +ASPNETCORE_URLS=http://+:8088 +AZURE_TOKEN_CREDENTIALS=dev diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj new file mode 100644 index 00000000000..8eaa39205ac --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/HostedWorkflowResilient.csproj @@ -0,0 +1,40 @@ + + + + false + + + + + + net10.0 + + enable + enable + HostedWorkflowResilient + HostedWorkflowResilient + b45eca04-a1ba-4c64-8318-a6051e83b485 + 1.18.0-preview.260818.1 + $(MSBuildThisFileDirectory)..\..\..\..\..\src + true + + + + + + + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs new file mode 100644 index 00000000000..fd67681f7aa --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/Program.cs @@ -0,0 +1,63 @@ +// Copyright (c) Microsoft. All rights reserved. + +// Sample: a resilient background workflow hosted with the Foundry Responses protocol. AgentServer +// re-invokes an interrupted response, while the workflow resumes from its durable checkpoint. +// It deploys directly from source, so Foundry builds and runs the uploaded project. + +using Azure.AI.Projects; +using Azure.Identity; +using DotNetEnv; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry.Hosting; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +Env.TraversePath().Load(); + +var projectEndpoint = new Uri(System.Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set.")); +var deployment = FirstNonBlank( + System.Environment.GetEnvironmentVariable("AZURE_AI_MODEL_DEPLOYMENT_NAME"), + System.Environment.GetEnvironmentVariable("FOUNDRY_MODEL"), + "gpt-4o"); +var agentName = System.Environment.GetEnvironmentVariable("AGENT_NAME") ?? "hosted-workflow-resilient"; + +IChatClient chatClient = new AIProjectClient(projectEndpoint, new DefaultAzureCredential()) + .GetProjectOpenAIClient() + .GetChatClient(deployment) + .AsIChatClient(); + +AIAgent frenchAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "french-translator", + Name = "French Translator", + ChatOptions = new() { Instructions = "Translate the provided text to French. Return only the translation." }, +}); +AIAgent spanishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "spanish-translator", + Name = "Spanish Translator", + ChatOptions = new() { Instructions = "Translate the provided text to Spanish. Return only the translation." }, +}); +AIAgent englishAgent = chatClient.AsAIAgent(new ChatClientAgentOptions +{ + Id = "english-translator", + Name = "English Translator", + ChatOptions = new() { Instructions = "Translate the provided text to English. Return only the translation." }, +}); + +AIAgent agent = new WorkflowBuilder(frenchAgent) + .AddEdge(frenchAgent, spanishAgent) + .AddEdge(spanishAgent, englishAgent) + .Build() + .AsAIAgent(name: agentName); + +var builder = WebApplication.CreateBuilder(args); +builder.Services.AddFoundryResponses(agent, configure: options => options.ResilientBackground = true); + +var app = builder.Build(); +app.MapFoundryResponses(); +app.Run(); + +static string FirstNonBlank(params string?[] candidates) => + Array.Find(candidates, candidate => !string.IsNullOrWhiteSpace(candidate))!; diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md new file mode 100644 index 00000000000..6022d77ded2 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/README.md @@ -0,0 +1,112 @@ +# Hosted-Workflow-Resilient + +A sequential translation workflow hosted with resilient background Responses enabled. AgentServer +re-invokes an interrupted background response, Foundry Hosting reloads the AgentSession, and the +workflow runtime continues from the checkpoint referenced by that session. + +This sample deploys directly from source. Foundry uploads the project as a ZIP, restores its +packages, builds it, and runs `HostedWorkflowResilient.dll`. No Dockerfile or container registry is +needed. + +## Key setting + +```csharp +builder.Services.AddFoundryResponses( + agent, + configure: options => options.ResilientBackground = true); +``` + +Each workflow agent has a fixed `Id` and `Name`. A restarted process must reconstruct the same +executor identities for a stored workflow checkpoint to match. + +## State ownership + +| State | Owner | +| --- | --- | +| Background task, response events, and selected response snapshots | AgentServer | +| AgentSession and workflow checkpoint reference | `FoundryAgentSessionStore` | +| Workflow execution checkpoints | `FoundryJsonCheckpointStore` | + +At each completed workflow superstep, the hosting adapter saves the AgentSession, records the +workflow checkpoint ID in AgentServer internal response metadata, and calls +`ResponseEventStream.Checkpoint()`. Recovery selects that exact workflow checkpoint ID. The response +output count is not used as the workflow cursor. + +## Local development + +Copy `.env.example` to `.env`, set the project endpoint and model deployment, then run: + +```powershell +az login +dotnet run --tl:off +``` + +The in-repository project automatically uses ProjectReference to run the current framework source. + +## Deploy from source + +Create an empty working directory outside the repository: + +```powershell +$work = Join-Path $env:TEMP "hosted-workflow-resilient-work" +New-Item -ItemType Directory -Path $work -Force | Out-Null +Set-Location $work + +$sample = "/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/azure.yaml" +azd auth login +azd ai agent init -m $sample -d +``` + +### Contributors testing framework changes + +**Skip this section unless you are testing an Agent Framework change from the current codebase that +has not been released yet.** The normal deployment uses the published packages. To test local +framework changes, pack the current repository source into the scaffolded upload before provisioning: + +```powershell +/dotnet/samples/04-hosting/FoundryHostedAgents/scripts/Add-LocalFrameworkFeed.ps1 ` + -Path ./hosted-workflow-resilient +``` + +The helper creates `local-feed/`, writes `nuget.config`, and changes `AgentFrameworkVersion` in the +scaffolded project. Both generated artifacts are included in the source ZIP. + +```powershell +Set-Location hosted-workflow-resilient +azd env set AZURE_AI_MODEL_DEPLOYMENT_NAME +azd provision +azd deploy +``` + +The workflow checkpoint store writes through the hosted agent's managed identity. Grant that +identity `Foundry User` on the existing Foundry project after the first deployment: + +```powershell +$agent = azd ai agent show hosted-workflow-resilient -o json | ConvertFrom-Json +az role assignment create ` + --assignee-object-id $agent.instance_identity.principal_id ` + --assignee-principal-type ServicePrincipal ` + --role "Foundry User" ` + --scope +``` + +Allow a few minutes for the role assignment to take effect before the first request. + +Submit the request with `store=true` and `background=true`. Poll the returned response id until it +reaches a terminal status. + +## Live integration coverage + +`Foundry.Hosting.IntegrationTests` contains a deterministic `resilient-workflow` scenario: + +- `long:` holds a background workflow without client traffic, then completes with the token. +- `crash:` writes a crash-once marker, terminates the container process, and completes only + after AgentServer reclaims the response and the workflow resumes in a replacement process. + +The test suite deploys that scenario to a real Foundry project and validates both behaviors. + +## Related samples + +- [Hosted-Workflow-Resilient-Long-Running](../Hosted-Workflow-Resilient-Long-Running/README.md): deterministic countdown recovery with exact output validation. +- [Hosted-Workflow-Simple](../Hosted-Workflow-Simple/README.md): workflow hosting without resilient background execution. +- [Hosted-Steering](../Hosted-Steering/README.md): mid-turn steering. diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/azure.yaml b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/azure.yaml new file mode 100644 index 00000000000..24d0e98928c --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Resilient/azure.yaml @@ -0,0 +1,36 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/Azure/azure-dev/main/schemas/v1.0/azure.yaml.json + +name: hosted-workflow-resilient +services: + ai-project: + host: azure.ai.project + hosted-workflow-resilient: + project: . + host: azure.ai.agent + language: csharp + uses: + - ai-project + codeConfiguration: + dependencyResolution: remote_build + entryPoint: HostedWorkflowResilient.dll + runtime: dotnet_10 + env: + ASPNETCORE_URLS: http://+:8088 + AZURE_AI_MODEL_DEPLOYMENT_NAME: ${AZURE_AI_MODEL_DEPLOYMENT_NAME} + container: + resources: + cpu: "0.5" + memory: 1Gi + description: | + A resilient background translation workflow hosted with the Foundry Responses protocol. + kind: hosted + metadata: + tags: + - AI Agent Hosting + - Azure AI AgentServer + - Agent Framework + - Resilient Background + name: hosted-workflow-resilient + protocols: + - protocol: responses + version: 2.0.0 diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Simple/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Simple/README.md index 032868050e7..a21c268cae8 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Simple/README.md +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted-Workflow-Simple/README.md @@ -205,4 +205,10 @@ that conversation no longer exists on the server. Start a fresh one: azd ai agent invoke --new-conversation "Hello!" ``` -For the full hosted-agent deployment guide, see the [official source-code deployment doc](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/deploy-hosted-agent-code). \ No newline at end of file +For the full hosted-agent deployment guide, see the [official source-code deployment doc](https://learn.microsoft.com/en-us/azure/foundry/agents/how-to/deploy-hosted-agent-code). + +## Related samples + +- [Hosted-Workflow-Resilient](../Hosted-Workflow-Resilient/README.md): adds resilient background execution to a model-backed workflow. +- [Hosted-Workflow-Resilient-Long-Running](../Hosted-Workflow-Resilient-Long-Running/README.md): deterministic countdown recovery with exact output validation. +- [Hosted-Workflow-Handoff](../Hosted-Workflow-Handoff/README.md): routes work between multiple specialized agents. \ No newline at end of file diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/Hosted_Shared_Contributor_Setup.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/Hosted_Shared_Contributor_Setup.csproj index 63cccf4613c..4ef51236487 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/Hosted_Shared_Contributor_Setup.csproj +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/Hosted_Shared_Contributor_Setup.csproj @@ -12,6 +12,7 @@ + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/LocalAIProjectClientDevelopmentSupport.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/LocalAIProjectClientDevelopmentSupport.cs new file mode 100644 index 00000000000..82fa27822e3 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Hosted_Shared_Contributor_Setup/LocalAIProjectClientDevelopmentSupport.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using Azure.Core; + +namespace Hosted_Shared_Contributor_Setup; + +/// +/// Rewrites an HTTPS request to a loopback HTTP endpoint immediately before transport. +/// +/// +/// Local development clients present an HTTPS endpoint to the bearer-token pipeline so it can +/// attach a token, then use this handler to reach a loopback HTTP server. +/// +public sealed class LocalHttpSchemeRewriteHandler : DelegatingHandler +{ + private readonly Uri _localEndpoint; + + /// + /// Initializes a new instance that routes requests to . + /// + /// The loopback HTTP endpoint hosting the local agent. + public LocalHttpSchemeRewriteHandler(Uri localEndpoint) + : base(new HttpClientHandler()) + { + ArgumentNullException.ThrowIfNull(localEndpoint); + if (!localEndpoint.IsLoopback + || localEndpoint.Scheme != Uri.UriSchemeHttp) + { + throw new ArgumentException( + "The local endpoint must be an HTTP loopback URI.", + nameof(localEndpoint)); + } + + this._localEndpoint = localEndpoint; + } + + /// + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + this.RewriteUri(request); + return base.SendAsync(request, cancellationToken); + } + + private void RewriteUri(HttpRequestMessage request) + { + Uri uri = request.RequestUri + ?? throw new InvalidOperationException("The local request URI is missing."); + if (!uri.IsLoopback) + { + throw new InvalidOperationException( + "The local HTTP rewrite policy can only target a loopback endpoint."); + } + + if (uri.Scheme == Uri.UriSchemeHttps) + { + request.RequestUri = + new UriBuilder(uri) + { + Scheme = Uri.UriSchemeHttp, + Host = this._localEndpoint.Host, + Port = this._localEndpoint.Port, + }.Uri; + } + } +} + +/// +/// Supplies a placeholder bearer token for a loopback server that does not validate authentication. +/// +/// +/// This credential is only for local sample development. It must not be used with remote services. +/// +public sealed class LocalDevelopmentTokenCredential : TokenCredential +{ + private static readonly AccessToken s_token = + new("local-development", DateTimeOffset.MaxValue); + + /// + public override AccessToken GetToken( + TokenRequestContext requestContext, + CancellationToken cancellationToken) => + s_token; + + /// + public override ValueTask GetTokenAsync( + TokenRequestContext requestContext, + CancellationToken cancellationToken) => + new(s_token); +} diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Program.cs b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Program.cs new file mode 100644 index 00000000000..65136cce510 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Program.cs @@ -0,0 +1,971 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.ClientModel; +using System.ClientModel.Primitives; +using System.Diagnostics; +using System.Globalization; +using System.Net; +using System.Net.Sockets; +using System.Text; +using System.Text.Json; +using Azure.AI.Projects; +using Hosted_Shared_Contributor_Setup; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; + +const string AgentName = "hosted-workflow-resilient-long-running"; +VerificationOptions options = VerificationOptions.Parse(args); +string repositoryRoot = FindRepositoryRoot(); +string serverProject = Path.Combine( + repositoryRoot, + "dotnet", + "samples", + "04-hosting", + "FoundryHostedAgents", + "responses", + "Hosted-Workflow-Resilient-Long-Running", + "HostedWorkflowResilientLongRunning.csproj"); +string workingRoot = Path.Combine( + Path.GetTempPath(), + $"maf-resilient-workflow-{Guid.NewGuid():N}"); +string serverOutput = Path.Combine(workingRoot, "server"); +string serverAssembly = Path.Combine( + serverOutput, + "HostedWorkflowResilientLongRunning.dll"); +string stateRoot = Path.Combine(workingRoot, "state"); +string logPath = Path.Combine( + Path.GetTempPath(), + $"maf-resilient-workflow-{Guid.NewGuid():N}.log"); +int port = GetAvailablePort(); +var baseAddress = new Uri($"http://127.0.0.1:{port}"); +bool succeeded = false; + +Directory.CreateDirectory(workingRoot); + +var cancellationSource = new CancellationTokenSource(TimeSpan.FromMinutes(3)); +var client = new HttpClient +{ + BaseAddress = baseAddress, + Timeout = Timeout.InfiniteTimeSpan, +}; +await using var logWriter = new StreamWriter(logPath, append: false, new UTF8Encoding(false)) +{ + AutoFlush = true, +}; +ServerProcess? server = null; +Task? createStream = null; +AgentStreamObserver? streamObserver = null; +LocalAgentClient? localAgentClient = null; +CancellationTokenSource? initialStreamCancellation = null; +try +{ + PrintHeader(options, stateRoot, logPath); + + Console.WriteLine("Preparing isolated Debug server binaries..."); + await BuildServerAsync( + serverProject, + serverOutput, + logWriter, + cancellationSource.Token); + Console.WriteLine(" server build complete"); + Console.WriteLine(); + + Console.WriteLine("[1/7] Starting the first server process..."); + Console.WriteLine($" endpoint: {baseAddress}"); + server = StartServer( + serverAssembly, + stateRoot, + port, + options.DelaySeconds, + logWriter); + Console.WriteLine($" process tree root: {server.Id}"); + await WaitForReadinessAsync(client, cancellationSource.Token); + Console.WriteLine(" server ready"); + Console.WriteLine(); + + localAgentClient = CreateClientAgent(baseAddress, AgentName); + AIAgent agent = localAgentClient.Agent; + AgentSession session = await agent.CreateSessionAsync(cancellationSource.Token); + AgentRunOptions runOptions = new() { AllowBackgroundResponses = true }; + + Console.WriteLine("[2/7] Starting the background countdown..."); + streamObserver = new AgentStreamObserver(options.CrashAfterCount); + initialStreamCancellation = + CancellationTokenSource.CreateLinkedTokenSource(cancellationSource.Token); +#pragma warning disable CA2025 // The stream must run concurrently until the server is killed; finally awaits it before disposing resources. + createStream = WatchInitialAgentStreamAsync( + agent, + session, + runOptions, + options.Target, + streamObserver, + initialStreamCancellation.Token); +#pragma warning restore CA2025 + + string responseId = await WaitForResponseIdAsync( + streamObserver, + createStream, + cancellationSource.Token); + Console.WriteLine($" response id: {responseId}"); + Console.WriteLine(); + + Console.WriteLine( + $"[3/7] Waiting for {options.CrashAfterCount} countdown items and their response checkpoint..."); + await streamObserver.CrashPointReached.Task.WaitAsync(cancellationSource.Token); + await WaitForPersistedResponseCheckpointAsync( + stateRoot, + responseId, + streamObserver.CompletedTexts, + cancellationSource.Token); + Console.WriteLine(" checkpoint persisted"); + Console.WriteLine(); + + Console.WriteLine("[4/7] Force-killing the first server process..."); + initialStreamCancellation.Cancel(); + await IgnoreExpectedDisconnectAsync(createStream); + createStream = null; + initialStreamCancellation.Dispose(); + initialStreamCancellation = null; + await server.KillAsync(); + server = null; + DeleteStaleStreamLocks(stateRoot); + Console.WriteLine(" process terminated"); + Console.WriteLine(); + + Console.WriteLine("[5/7] Starting a replacement server over the same durable state..."); + server = StartServer( + serverAssembly, + stateRoot, + port, + options.DelaySeconds, + logWriter); + Console.WriteLine($" process tree root: {server.Id}"); + await WaitForReadinessAsync(client, cancellationSource.Token); + Console.WriteLine(" recovery scan completed"); + Console.WriteLine(); + Console.WriteLine("[6/7] Reconnecting with the sequence-aware continuation token..."); + streamObserver.BeginRecovery(); + runOptions.ContinuationToken = streamObserver.ContinuationToken + ?? throw new InvalidOperationException( + "The initial stream did not provide a continuation token."); + await WatchRecoveredAgentStreamAsync( + agent, + session, + runOptions, + streamObserver, + cancellationSource.Token); + + List actual = streamObserver.CompletedTexts; + List expected = + [ + .. Enumerable.Range(1, options.Target) + .Reverse() + .Select(value => value.ToString(CultureInfo.InvariantCulture)), + "Countdown complete.", + ]; + + if (!streamObserver.ResponseCompleted) + { + throw new InvalidOperationException( + "The recovered stream ended without response.completed."); + } + + if (!actual.SequenceEqual(expected)) + { + throw new InvalidOperationException( + "Recovered output did not match the expected countdown." + + $"{Environment.NewLine}Expected: {string.Join(", ", expected)}" + + $"{Environment.NewLine}Actual: {string.Join(", ", actual)}"); + } + + Console.WriteLine(); + Console.WriteLine("[7/7] Replaying from the start without a sequence cursor..."); + AgentRunOptions replayOptions = new() + { + AllowBackgroundResponses = true, + ContinuationToken = CreateReplayFromStartToken(responseId), + }; + var replayObserver = new AgentStreamObserver(int.MaxValue); + await WatchReplayedAgentStreamAsync( + agent, + session, + replayOptions, + replayObserver, + cancellationSource.Token); + if (!replayObserver.ResponseCompleted + || !replayObserver.CompletedTexts.SequenceEqual(expected)) + { + throw new InvalidOperationException( + "The cursor-free replay did not return the complete countdown."); + } + int retainedCountdownUpdates = + actual.Count(text => text != "Countdown complete."); + int replayedCountdownUpdates = + replayObserver.CompletedTexts.Count( + text => text != "Countdown complete."); + Console.WriteLine(); + Console.WriteLine( + $"Client retained countdown updates: {retainedCountdownUpdates}"); + Console.WriteLine( + $"Replay countdown updates: {replayedCountdownUpdates}"); + + Console.WriteLine(); + Console.ForegroundColor = ConsoleColor.Green; + Console.WriteLine( + "PASS: crash recovery completed with ordered output and no missing or duplicated items."); + Console.ResetColor(); + succeeded = true; +} +catch (Exception exception) +{ + Console.WriteLine(); + Console.ForegroundColor = ConsoleColor.Red; + Console.Error.WriteLine($"FAIL: {exception.Message}"); + Console.ResetColor(); + Console.Error.WriteLine($"Server log: {logPath}"); + System.Environment.ExitCode = 1; +} +finally +{ + if (server is not null) + { + await server.KillAsync(); + } + + if (createStream is not null) + { + initialStreamCancellation?.Cancel(); + await IgnoreExpectedDisconnectAsync(createStream); + } + + initialStreamCancellation?.Dispose(); + client.Dispose(); + localAgentClient?.Dispose(); + cancellationSource.Dispose(); + + if (succeeded) + { + TryDeleteDirectory(workingRoot); + } + else + { + Console.Error.WriteLine($"E2E working directory retained at: {workingRoot}"); + } +} + +static void PrintHeader( + VerificationOptions options, + string stateRoot, + string logPath) +{ + Console.ForegroundColor = ConsoleColor.Cyan; + Console.WriteLine("============================================================"); + Console.WriteLine("Resilient long-running workflow E2E demonstration"); + Console.WriteLine("============================================================"); + Console.ResetColor(); + Console.WriteLine($"Countdown target: {options.Target}"); + Console.WriteLine($"Crash after: {options.CrashAfterCount} message items"); + Console.WriteLine($"Step delay: {options.DelaySeconds} second(s)"); + Console.WriteLine($"Durable state: {stateRoot}"); + Console.WriteLine($"Server log: {logPath}"); + Console.WriteLine(); +} + +static ServerProcess StartServer( + string serverAssembly, + string stateRoot, + int port, + int delaySeconds, + TextWriter logWriter) +{ + var startInfo = new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = Path.GetDirectoryName(serverAssembly)!, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("exec"); + startInfo.ArgumentList.Add(serverAssembly); + startInfo.Environment["AGENTSERVER_STATE_ROOT"] = stateRoot; + startInfo.Environment["FOUNDRY_AGENT_SESSION_ID"] = "using-e2e-resilience"; + startInfo.Environment["AGENT_NAME"] = AgentName; + startInfo.Environment["ASPNETCORE_URLS"] = $"http://127.0.0.1:{port}"; + startInfo.Environment["ASPNETCORE_ENVIRONMENT"] = "Development"; + startInfo.Environment["COUNTDOWN_DELAY_SECONDS"] = + delaySeconds.ToString(CultureInfo.InvariantCulture); + startInfo.Environment["DOTNET_NOLOGO"] = "true"; + startInfo.Environment.Remove("FOUNDRY_HOSTING_ENVIRONMENT"); + + return ServerProcess.Start(startInfo, logWriter); +} + +static LocalAgentClient CreateClientAgent(Uri baseAddress, string agentName) +{ + Uri httpsProjectEndpoint = new UriBuilder(baseAddress) + { + Scheme = Uri.UriSchemeHttps, + Port = baseAddress.Port, + }.Uri; + + var transportClient = new HttpClient( + new LocalHttpSchemeRewriteHandler(baseAddress)); + var clientOptions = new AIProjectClientOptions + { + Transport = new HttpClientPipelineTransport(transportClient), + }; + + AIAgent agent = new AIProjectClient( + httpsProjectEndpoint, + new LocalDevelopmentTokenCredential(), + clientOptions) + .AsAIAgent( + model: agentName, + instructions: "Invoke the local hosted countdown workflow."); + return new LocalAgentClient(agent, transportClient); +} + +static ResponseContinuationToken CreateReplayFromStartToken( + string responseId) +{ + ResponseContinuationToken innerToken = + ResponseContinuationToken.FromBytes( + JsonSerializer.SerializeToUtf8Bytes( + new { responseId })); + string serializedInnerToken = JsonSerializer.Serialize( + innerToken, + AgentAbstractionsJsonUtilities.DefaultOptions.GetTypeInfo( + typeof(ResponseContinuationToken))); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes( + new + { + type = "chatClientAgentContinuationToken", + innerToken = serializedInnerToken, + }); + return ResponseContinuationToken.FromBytes(bytes); +} + +static async Task BuildServerAsync( + string serverProject, + string serverOutput, + TextWriter logWriter, + CancellationToken cancellationToken) +{ + var startInfo = new ProcessStartInfo + { + FileName = "dotnet", + WorkingDirectory = Path.GetDirectoryName(serverProject)!, + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + startInfo.ArgumentList.Add("build"); + startInfo.ArgumentList.Add(serverProject); + startInfo.ArgumentList.Add("--configuration"); + startInfo.ArgumentList.Add("Debug"); + startInfo.ArgumentList.Add("--output"); + startInfo.ArgumentList.Add(serverOutput); + startInfo.ArgumentList.Add("--tl:off"); + startInfo.Environment["DOTNET_NOLOGO"] = "true"; + + using Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Could not start the server build."); + TextWriter synchronizedLogWriter = TextWriter.Synchronized(logWriter); + process.OutputDataReceived += (_, eventArgs) => + { + if (eventArgs.Data is not null) + { + synchronizedLogWriter.WriteLine($"[build stdout] {eventArgs.Data}"); + } + }; + process.ErrorDataReceived += (_, eventArgs) => + { + if (eventArgs.Data is not null) + { + synchronizedLogWriter.WriteLine($"[build stderr] {eventArgs.Data}"); + } + }; + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + await process.WaitForExitAsync(cancellationToken); + process.WaitForExit(); + if (process.ExitCode != 0) + { + throw new InvalidOperationException( + $"Server build failed with exit code {process.ExitCode}."); + } +} + +static async Task WaitForReadinessAsync( + HttpClient client, + CancellationToken cancellationToken) +{ + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(30); + while (DateTimeOffset.UtcNow < deadline) + { + try + { + using var requestCancellation = + CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + requestCancellation.CancelAfter(TimeSpan.FromSeconds(2)); + using HttpResponseMessage response = await client.GetAsync( + new Uri("readiness", UriKind.Relative), + requestCancellation.Token); + if (response.StatusCode == HttpStatusCode.OK) + { + return; + } + } + catch (Exception exception) + when (exception is HttpRequestException or TaskCanceledException) + { + } + + await Task.Delay(TimeSpan.FromMilliseconds(250), cancellationToken); + } + + throw new TimeoutException("Server did not become ready within 30 seconds."); +} + +static async Task WatchInitialAgentStreamAsync( + AIAgent agent, + AgentSession session, + AgentRunOptions options, + int target, + AgentStreamObserver observer, + CancellationToken cancellationToken) +{ + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync( + $"Count down from {target}", + session, + options, + cancellationToken)) + { + observer.ObserveInitial(update); + } +} + +static async Task WatchRecoveredAgentStreamAsync( + AIAgent agent, + AgentSession session, + AgentRunOptions options, + AgentStreamObserver observer, + CancellationToken cancellationToken) +{ + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync( + session, + options, + cancellationToken)) + { + observer.ObserveRecovered(update); + } +} + +static async Task WatchReplayedAgentStreamAsync( + AIAgent agent, + AgentSession session, + AgentRunOptions options, + AgentStreamObserver observer, + CancellationToken cancellationToken) +{ + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync( + session, + options, + cancellationToken)) + { + observer.ObserveReplayed(update); + } +} + +static async Task WaitForResponseIdAsync( + AgentStreamObserver observer, + Task createStream, + CancellationToken cancellationToken) +{ + Task completed = await Task.WhenAny( + observer.ResponseId.Task, + createStream, + Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken)); + if (completed == createStream) + { + await createStream; + throw new InvalidOperationException( + "The initial stream ended before returning a response ID."); + } + + cancellationToken.ThrowIfCancellationRequested(); + return await observer.ResponseId.Task; +} + +static async Task WaitForPersistedResponseCheckpointAsync( + string stateRoot, + string responseId, + IReadOnlyList expectedPrefix, + CancellationToken cancellationToken) +{ + string path = Path.Combine( + stateRoot, + "responses", + "envelopes", + $"{responseId}.json"); + var deadline = DateTimeOffset.UtcNow + TimeSpan.FromSeconds(15); + while (DateTimeOffset.UtcNow < deadline) + { + try + { + using FileStream file = new( + path, + FileMode.Open, + FileAccess.Read, + FileShare.ReadWrite | FileShare.Delete); + using JsonDocument document = await JsonDocument.ParseAsync( + file, + cancellationToken: cancellationToken); + JsonElement response = + document.RootElement.GetProperty("envelope"); + List persistedTexts = GetPersistedMessageTexts(response); + bool hasCheckpointMetadata = + response.TryGetProperty("metadata", out JsonElement metadata) + && metadata.TryGetProperty("_internal_metadata", out JsonElement internalMetadata) + && !string.IsNullOrWhiteSpace(internalMetadata.GetString()); + if (hasCheckpointMetadata + && persistedTexts.Count >= expectedPrefix.Count + && persistedTexts + .Take(expectedPrefix.Count) + .SequenceEqual(expectedPrefix)) + { + return; + } + } + catch (Exception exception) + when (exception is IOException + or JsonException + or KeyNotFoundException) + { + } + + await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); + } + + throw new TimeoutException( + "The response checkpoint was not persisted within 15 seconds."); +} + +static List GetPersistedMessageTexts(JsonElement response) +{ + List texts = []; + foreach (JsonElement item in response.GetProperty("output").EnumerateArray()) + { + if (item.GetProperty("type").GetString() != "message") + { + continue; + } + + foreach (JsonElement content in item.GetProperty("content").EnumerateArray()) + { + if (content.GetProperty("type").GetString() == "output_text") + { + texts.Add(content.GetProperty("text").GetString() ?? string.Empty); + } + } + } + + return texts; +} + +static async Task IgnoreExpectedDisconnectAsync(Task streamTask) +{ + try + { + await streamTask; + } + catch (Exception exception) + when (IsExpectedDisconnect(exception)) + { + } + + static bool IsExpectedDisconnect(Exception exception) + { + if (exception is AggregateException aggregate) + { + return aggregate + .Flatten() + .InnerExceptions + .All(IsExpectedDisconnect); + } + + return exception is ClientResultException + or HttpRequestException + or IOException + or OperationCanceledException; + } +} + +static void DeleteStaleStreamLocks(string stateRoot) +{ + string streamsPath = Path.Combine(stateRoot, "streams"); + if (!Directory.Exists(streamsPath)) + { + return; + } + + foreach (string lockPath in Directory.EnumerateFiles( + streamsPath, + "*.jsonl.lock", + SearchOption.TopDirectoryOnly)) + { + for (int attempt = 1; attempt <= 10; attempt++) + { + try + { + File.Delete(lockPath); + break; + } + catch (UnauthorizedAccessException) when (attempt < 10) + { + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + } + catch (IOException) when (attempt < 10) + { + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + } + } + } +} + +static int GetAvailablePort() +{ + var listener = new TcpListener(IPAddress.Loopback, 0); + listener.Start(); + int port = ((IPEndPoint)listener.LocalEndpoint).Port; + listener.Stop(); + return port; +} + +static string FindRepositoryRoot() +{ + foreach (string start in new[] { Environment.CurrentDirectory, AppContext.BaseDirectory }) + { + DirectoryInfo? directory = new(start); + while (directory is not null) + { + if (File.Exists(Path.Combine( + directory.FullName, + "dotnet", + "agent-framework-dotnet.slnx"))) + { + return directory.FullName; + } + + directory = directory.Parent; + } + } + + throw new InvalidOperationException( + "Could not find the Agent Framework repository root."); +} + +static void TryDeleteDirectory(string path) +{ + try + { + Directory.Delete(path, recursive: true); + } + catch (IOException) + { + } + catch (UnauthorizedAccessException) + { + } +} + +internal sealed class AgentStreamObserver(int crashAfterCount) +{ + private readonly Dictionary _messageBuffers = + new(StringComparer.Ordinal); + private readonly HashSet _completedMessageIds = + new(StringComparer.Ordinal); + private List? _preCrashTexts; + private bool? _recoveryIncludesSnapshot; + private int _recoverySnapshotIndex; + private int _messageCount; + + public TaskCompletionSource ResponseId { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource CrashPointReached { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public List CompletedTexts { get; } = []; + + public ResponseContinuationToken? ContinuationToken { get; private set; } + + public bool ResponseCompleted { get; private set; } + + public void BeginRecovery() + { + this._preCrashTexts = [.. this.CompletedTexts]; + this._recoveryIncludesSnapshot = null; + this._recoverySnapshotIndex = 0; + } + + public void ObserveInitial(AgentResponseUpdate update) => + this.Observe(update, "before", trackCheckpoint: true); + + public void ObserveRecovered(AgentResponseUpdate update) => + this.Observe(update, "recovered", trackCheckpoint: false); + + public void ObserveReplayed(AgentResponseUpdate update) => + this.Observe(update, "replayed", trackCheckpoint: false); + + private void Observe( + AgentResponseUpdate update, + string phase, + bool trackCheckpoint) + { + object? rawRepresentation = + update.RawRepresentation is ChatResponseUpdate chatResponseUpdate + ? chatResponseUpdate.RawRepresentation + : update.RawRepresentation; + + if (update.ContinuationToken is { } continuationToken) + { + this.ContinuationToken = continuationToken; + } + + if (!string.IsNullOrWhiteSpace(update.ResponseId)) + { + this.ResponseId.TrySetResult(update.ResponseId); + } + + if (!string.IsNullOrWhiteSpace(update.MessageId) + && !string.IsNullOrEmpty(update.Text)) + { + if (!this._messageBuffers.TryGetValue( + update.MessageId, + out StringBuilder? buffer)) + { + buffer = new StringBuilder(); + this._messageBuffers[update.MessageId] = buffer; + } + + buffer.Append(update.Text); + } + + if (rawRepresentation is StreamingResponseOutputItemDoneUpdate + { + Item: MessageResponseItem message + } + && this._completedMessageIds.Add(message.Id)) + { + string text = this._messageBuffers.TryGetValue( + message.Id, + out StringBuilder? buffer) + ? buffer.ToString() + : string.Empty; + if (phase != "before" && text.Length == 0) + { + return; + } + + if (phase == "recovered" + && this.TryHandleRecoverySnapshot(text)) + { + return; + } + + this.CompletedTexts.Add(text); + WriteOutput(phase, text); + + if (trackCheckpoint && ++this._messageCount >= crashAfterCount) + { + this.CrashPointReached.TrySetResult(); + } + } + + if (rawRepresentation is StreamingResponseCompletedUpdate) + { + this.ResponseCompleted = true; + } + } + + private bool TryHandleRecoverySnapshot(string text) + { + if (this._preCrashTexts is not { Count: > 0 } preCrashTexts) + { + return false; + } + + this._recoveryIncludesSnapshot ??= + string.Equals(text, preCrashTexts[0], StringComparison.Ordinal); + if (this._recoveryIncludesSnapshot is not true) + { + return false; + } + + if (this._recoverySnapshotIndex >= preCrashTexts.Count) + { + return false; + } + + if (!string.Equals( + text, + preCrashTexts[this._recoverySnapshotIndex], + StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "The response snapshot returned during reconnection did not match the pre-crash output."); + } + + this._recoverySnapshotIndex++; + WriteOutput("restored", text); + return true; + } + + private static void WriteOutput(string phase, string text) + { + Console.ForegroundColor = phase == "recovered" + ? ConsoleColor.Green + : ConsoleColor.DarkGray; + Console.WriteLine($" {phase,-9} > {text}"); + Console.ResetColor(); + } +} + +internal sealed class ServerProcess +{ + private readonly Process _process; + private readonly Task _outputPump; + private readonly Task _errorPump; + + private ServerProcess(Process process, TextWriter logWriter) + { + this._process = process; + this._outputPump = PumpAsync(process.StandardOutput, logWriter, "stdout"); + this._errorPump = PumpAsync(process.StandardError, logWriter, "stderr"); + } + + public int Id => this._process.Id; + + public static ServerProcess Start( + ProcessStartInfo startInfo, + TextWriter logWriter) + { + Process process = Process.Start(startInfo) + ?? throw new InvalidOperationException("Could not start the server process."); + return new ServerProcess(process, TextWriter.Synchronized(logWriter)); + } + + public async Task KillAsync() + { + if (!this._process.HasExited) + { + this._process.Kill(entireProcessTree: true); + } + + await this._process.WaitForExitAsync(); + await Task.WhenAll(this._outputPump, this._errorPump) + .WaitAsync(TimeSpan.FromSeconds(5)); + this._process.Dispose(); + } + + private static async Task PumpAsync( + StreamReader reader, + TextWriter writer, + string source) + { + while (await reader.ReadLineAsync() is { } line) + { + await writer.WriteLineAsync($"[{source}] {line}"); + } + } +} + +internal sealed class LocalAgentClient( + AIAgent agent, + HttpClient transportClient) : IDisposable +{ + public AIAgent Agent { get; } = agent; + + public void Dispose() => transportClient.Dispose(); +} + +internal sealed record VerificationOptions( + int Target, + int CrashAfterCount, + int DelaySeconds) +{ + public static VerificationOptions Parse(string[] args) + { + int target = 20; + int? crashAfterCount = null; + int delaySeconds = 1; + + for (int index = 0; index < args.Length; index++) + { + string argument = args[index]; + switch (argument) + { + case "--target": + target = ReadInteger(args, ref index, argument); + break; + case "--crash-after-count": + crashAfterCount = ReadInteger(args, ref index, argument); + break; + case "--delay-seconds": + delaySeconds = ReadInteger(args, ref index, argument); + break; + default: + throw new ArgumentException($"Unknown argument '{argument}'."); + } + } + + int resolvedCrashAfterCount = crashAfterCount ?? Math.Max(1, target / 2); + if (target < 2) + { + throw new ArgumentOutOfRangeException( + nameof(args), + "Target must be at least 2."); + } + + if (resolvedCrashAfterCount < 1 || resolvedCrashAfterCount >= target) + { + throw new ArgumentOutOfRangeException( + nameof(args), + "Crash count must be greater than zero and less than the target."); + } + + if (delaySeconds < 0) + { + throw new ArgumentOutOfRangeException( + nameof(args), + "Delay seconds must be zero or greater."); + } + + return new(target, resolvedCrashAfterCount, delaySeconds); + } + + private static int ReadInteger( + string[] args, + ref int index, + string argument) + { + if (++index >= args.Length + || !int.TryParse( + args[index], + NumberStyles.None, + CultureInfo.InvariantCulture, + out int value)) + { + throw new ArgumentException( + $"Argument '{argument}' requires an integer value."); + } + + return value; + } +} diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/README.md new file mode 100644 index 00000000000..1741878c4db --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/README.md @@ -0,0 +1,103 @@ +# Using-E2E-Resilience + +A self-contained local E2E demonstration for +[`Hosted-Workflow-Resilient-Long-Running`](../Hosted-Workflow-Resilient-Long-Running/). +It owns both server process lifetimes, consumes their response stream, and prints every countdown +output in one console. + +The E2E creates a MAF client agent through `AIProjectClient.AsAIAgent(model, instructions)`. It +enables `AgentRunOptions.AllowBackgroundResponses`, consumes `AgentResponseUpdate` values, saves the +latest non-null `ResponseContinuationToken`, and supplies that token after the replacement server +starts. It does not implement the Responses HTTP or SSE protocol itself. + +The demonstration uses one MAF client agent and one agent session for three calls: + +1. Starts the hosted workflow server as a child process. +2. Creates a stored background streaming response through the MAF agent. +3. Prints countdown messages as MAF streaming updates arrive. +4. Waits until the matching workflow and response checkpoint is durable. +5. Force-kills the server process tree. +6. Starts a replacement server over the same AgentServer state. +7. The second call reconnects with the sequence-aware continuation token and prints only newly + recovered messages. +8. The third call uses the same agent and session with the same response ID but no sequence cursor, + replaying the entire stream from the start. +9. The E2E verifies that the client accumulator and cursor-free replay contain the same complete + countdown. + +## Run + +Run from the repository root: + +```powershell +dotnet run --project dotnet\samples\04-hosting\FoundryHostedAgents\responses\Using-E2E-Resilience +``` + +The E2E program starts the first server, ends it abruptly, starts the replacement server with the +same durable state, and ends the replacement when the verification completes. A separately running +local server may remain open: the E2E uses a random port, isolated Debug binaries, and an isolated +AgentServer state directory. + +No Azure project, model deployment, credentials, or second terminal is required. +The E2E builds the server in Debug into an isolated temporary directory, so it does not reuse or +overwrite the binaries of a separately running local server. + +`AIProjectClient` requires an HTTPS endpoint before its bearer-token policy will run. The shared +`LocalHttpSchemeRewriteHandler` presents HTTPS to that pipeline, then routes the request to the +random loopback HTTP port at transport time. The handler rejects non-loopback targets. + +Example: + +```text +[1/7] Starting the first server process... +[2/7] Starting the background countdown... + before > 20 + before > 19 + before > 18 +... +[4/7] Force-killing the first server process... +[5/7] Starting a replacement server over the same durable state... +[6/7] Reconnecting to the response stream... + recovered > 10 + recovered > 9 +... + recovered > Countdown complete. + +[7/7] Replaying from the start without a sequence cursor... + replayed > 20 + replayed > 19 +... + replayed > Countdown complete. + +Client retained countdown updates: 20 +Replay countdown updates: 20 + +PASS: crash recovery completed with ordered output and no missing or duplicated items. +``` + +## Options + +```powershell +dotnet run --project dotnet\samples\04-hosting\FoundryHostedAgents\responses\Using-E2E-Resilience -- ` + --target 30 ` + --crash-after-count 12 ` + --delay-seconds 1 +``` + +| Option | Default | Meaning | +| --- | --- | --- | +| `--target` | `20` | First countdown value. Must be at least 2. | +| `--crash-after-count` | Half the target | Number of completed countdown messages before the crash. | +| `--delay-seconds` | `1` | Delay between countdown steps. | + +Server output is redirected to a temporary log whose path is printed at startup. Each run uses a +random local port and an isolated AgentServer state directory. Successful runs delete their durable +state. Failed runs retain state and print its path for investigation. + +The second call's continuation token resumes after the last update consumed before the crash. +Previously consumed countdown messages are retained in the client accumulator and are not streamed +again. Only work after the durable checkpoint appears as `recovered`. + +For the third call, the E2E derives another valid `ChatClientAgent` continuation token whose inner +Responses token contains the same response ID without a sequence number. That call prints every +persisted stream item as `replayed`. diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Using-E2E-Resilience.csproj b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Using-E2E-Resilience.csproj new file mode 100644 index 00000000000..f450479e8e5 --- /dev/null +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-E2E-Resilience/Using-E2E-Resilience.csproj @@ -0,0 +1,26 @@ + + + + Exe + net10.0 + enable + enable + false + UsingE2EResilience + using-e2e-resilience + $(NoWarn);MEAI001;OPENAI001 + + + + + + + + + + + + + + + diff --git a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-Samples/README.md b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-Samples/README.md index ee7a318672f..543132c0c5b 100644 --- a/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-Samples/README.md +++ b/dotnet/samples/04-hosting/FoundryHostedAgents/responses/Using-Samples/README.md @@ -48,6 +48,9 @@ never hits the TLS check. | [`Hosted-Toolbox-AuthPaths-Client/`](./Hosted-Toolbox-AuthPaths-Client/) | Hosted toolbox agents | Handles OAuth consent, function-tool approvals, and native MCP approvals. Use it with `Hosted-Toolbox-AuthPaths` or `Hosted-ToolboxMcpSkills`. | | [`SessionFilesClient/`](./SessionFilesClient/) | [`Hosted-Files`](../Hosted-Files/) | Same shape as `SimpleAgent`, framed around the bundled-files demo. | +For a self-contained crash-recovery demonstration that starts, interrupts, and restarts its own +local server, see [`Using-E2E-Resilience`](../Using-E2E-Resilience/). + ## Configuration (common to all clients) ```env diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs index 1b946d95611..725fd28f03b 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/AgentFrameworkResponseHandler.cs @@ -6,8 +6,10 @@ using System.Runtime.CompilerServices; using System.Security.Cryptography; using System.Threading; +using System.Threading.Tasks; using Azure.AI.AgentServer.Responses; using Azure.AI.AgentServer.Responses.Models; +using Microsoft.Agents.AI.Workflows; using Microsoft.Extensions.AI; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -21,6 +23,7 @@ using ResponseCompletedEvent = Azure.AI.AgentServer.Responses.Models.ResponseCompletedEvent; using ResponseFailedEvent = Azure.AI.AgentServer.Responses.Models.ResponseFailedEvent; using ResponseIncompleteEvent = Azure.AI.AgentServer.Responses.Models.ResponseIncompleteEvent; +using ResponseOutputItemDoneEvent = Azure.AI.AgentServer.Responses.Models.ResponseOutputItemDoneEvent; namespace Microsoft.Agents.AI.Foundry.Hosting; @@ -32,10 +35,19 @@ namespace Microsoft.Agents.AI.Foundry.Hosting; [Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] public class AgentFrameworkResponseHandler : ResponseHandler { + private const string LatestWorkflowCheckpointIdMetadataKey = "_last_checkpoint_id"; + private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly FoundryToolboxService? _toolboxService; + /// + /// Whether the host was configured for durable long-running (resilient) background responses + /// (). When the + /// handler never does mid-turn session saves or recovery exit and behaves exactly as a non-resilient host. + /// + private readonly bool _resilientBackground; + /// /// Cached fallback used when no is registered in DI. /// Avoids a per-request allocation on the request hot path. @@ -53,13 +65,39 @@ public AgentFrameworkResponseHandler( IServiceProvider serviceProvider, ILogger logger, FoundryToolboxService? toolboxService = null) + : this( + serviceProvider, + logger, + Options.Create(new FoundryResponsesOptions()), + toolboxService) + { + } + + /// + /// Initializes a new instance of the class + /// that resolves agents from keyed DI services. + /// + /// The service provider for resolving agents. + /// The logger instance. + /// + /// Hosting options, used to read whether resilient background responses are enabled. + /// + /// Optional Foundry Toolbox service providing MCP tools. + [ActivatorUtilitiesConstructor] + public AgentFrameworkResponseHandler( + IServiceProvider serviceProvider, + ILogger logger, + IOptions foundryResponsesOptions, + FoundryToolboxService? toolboxService = null) { _ = Throw.IfNull(serviceProvider); _ = Throw.IfNull(logger); + _ = Throw.IfNull(foundryResponsesOptions); this._serviceProvider = serviceProvider; this._logger = logger; this._toolboxService = toolboxService; + this._resilientBackground = foundryResponsesOptions.Value.ResilientBackground; } /// @@ -118,6 +156,7 @@ public override async IAsyncEnumerable CreateAsync( // nothing is persisted for the key, so a fresh conversation and a resumed one both end up with // a session to run against. AgentSession? session; + bool sessionRestoredFromStore = false; if (string.IsNullOrWhiteSpace(agentSessionId)) { session = await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); @@ -130,6 +169,7 @@ public override async IAsyncEnumerable CreateAsync( resolvedUserId, cancellationToken).ConfigureAwait(false); + sessionRestoredFromStore = session is not null; session ??= await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); } @@ -163,8 +203,31 @@ public override async IAsyncEnumerable CreateAsync( } } - // 3. Create the SDK event stream builder - var stream = new ResponseEventStream(context, request); + // 3. Create the SDK event stream builder. + // On recovery, AgentServer supplies the last ResponseObject snapshot that it persisted. + // Workflow response checkpoints carry the exact workflow checkpoint id represented by that + // snapshot, so recovery can select the matching workflow boundary rather than a newer + // checkpoint that may already exist in workflow storage. + var stream = context.IsRecovery && context.PersistedResponse is { } persistedResponse + ? new ResponseEventStream(context, persistedResponse) + : new ResponseEventStream(context, request); + + WorkflowSessionCheckpointRecovery? workflowCheckpointRecovery = + session?.GetService(); + if (context.IsRecovery + && sessionRestoredFromStore + && workflowCheckpointRecovery is not null) + { + string? checkpointId = + stream.InternalMetadata.TryGetValue(LatestWorkflowCheckpointIdMetadataKey, out string? persistedCheckpointId) + && !string.IsNullOrWhiteSpace(persistedCheckpointId) + ? persistedCheckpointId + : null; + + // When metadata is absent, TryPrepare keeps the checkpoint already referenced by the + // restored session. Either path continues queued work without starting a new turn. + workflowCheckpointRecovery.TryPrepare(checkpointId); + } // 3. Emit lifecycle events yield return stream.EmitCreated(); @@ -172,18 +235,26 @@ public override async IAsyncEnumerable CreateAsync( // 4. Convert input: the current input items become the run's messages. Earlier turns are not // added here; whatever holds the history for this agent supplies them, see step 5. + // + // On recovery the platform re-delivers the original input. When a persisted AgentSession was + // restored, this adapter leaves the message list empty and lets that session define re-entry. + // If no session was ever saved, there is no resumable MAF state, so recovery restarts from the + // original input instead of invoking a fresh session with no messages. + bool shouldInjectRequestInput = !context.IsRecovery || !sessionRestoredFromStore; var messages = new List(); - - // Load and convert current input items - var inputItems = await context.GetInputItemsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); - if (inputItems.Count > 0) + if (shouldInjectRequestInput) { - messages.AddRange(InputConverter.ConvertItemsToMessages(inputItems, session?.StateBag)); - } - else - { - // Fall back to raw request input - messages.AddRange(InputConverter.ConvertInputToMessages(request, session?.StateBag)); + // Load and convert current input items + var inputItems = await context.GetInputItemsAsync(cancellationToken: cancellationToken).ConfigureAwait(false); + if (inputItems.Count > 0) + { + messages.AddRange(InputConverter.ConvertItemsToMessages(inputItems, session?.StateBag)); + } + else + { + // Fall back to raw request input + messages.AddRange(InputConverter.ConvertInputToMessages(request, session?.StateBag)); + } } // 5. Build chat options @@ -337,9 +408,13 @@ await this._toolboxService var options = new ChatClientAgentRunOptions(chatOptions); - // We only use a volatile provider for the conversation history if the agent is a ChatClientAgent and the allow setting is not intentionally set or not custom chat history provider is intentionally supplied. + // We only use a volatile provider for the conversation history if the agent is a + // ChatClientAgent, stored output is not allowed, and no custom history provider was supplied. + // Recovery does not reload platform history because the restored AgentSession owns re-entry + // state. For workflows, that includes the workflow checkpoint reference. var useVolatileChatHistoryProvider = - !allowStoredOutputEnabled + shouldInjectRequestInput + && !allowStoredOutputEnabled && agent.GetService() is not null && agentOptions?.ChatHistoryProvider is null; @@ -358,13 +433,22 @@ await this._toolboxService // We create a linked CTS so the consent-aware tool wrapper can cancel the agent // run mid-loop when a -32006 error is returned by the proxy. The RequestConsentState // is a shared mutable object that flows via AsyncLocal to the tool wrapper. - using var consentCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + using var consentCts = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + context.Shutdown); var consentState = new RequestConsentState { CancellationSource = consentCts }; McpConsentContext.Current.Value = consentState; // 7. Run the agent and convert output // NOTE: C# forbids 'yield return' inside a try block that has a catch clause, // and inside catch blocks. We use a flag to defer the yield to outside the try/catch. + // + // On a resilient turn, save the AgentSession after completed response output items so a + // process crash can reload a recent session snapshot. Workflow supersteps use a stronger + // boundary below: save the session, record its workflow checkpoint id in internal response + // metadata, then ask AgentServer to persist the matching ResponseObject snapshot. + bool isResilientTurn = this.ShouldPersistForResilience(request) || context.IsRecovery; + bool emittedTerminal = false; bool notAllowedStoreUsageDetected = false; @@ -375,6 +459,49 @@ await this._toolboxService // A successful terminal event, held until the run is wound up and the session can be checked. ResponseStreamEvent? completedEvent = null; + bool steeringDetected = false; + bool deferredForRecovery = false; + + async ValueTask PersistWorkflowCheckpointAsync( + CheckpointInfo checkpoint, + CancellationToken checkpointCancellationToken) + { + if (!isResilientTurn + || workflowCheckpointRecovery is null + || session is null + || string.IsNullOrWhiteSpace(agentSessionId) + || (stream.InternalMetadata.TryGetValue(LatestWorkflowCheckpointIdMetadataKey, out string? lastCheckpointId) + && string.Equals(lastCheckpointId, checkpoint.CheckpointId, StringComparison.Ordinal))) + { + return null; + } + + try + { + await sessionStore.SaveSessionAsync( + agent, + agentSessionId, + session, + resolvedUserId, + checkpointCancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + if (this._logger.IsEnabled(LogLevel.Debug)) + { + this._logger.LogDebug( + ex, + "Workflow checkpoint {CheckpointId} was not paired with response {ResponseId} because its AgentSession could not be saved.", + checkpoint.CheckpointId, + context.ResponseId); + } + + return null; + } + + stream.InternalMetadata[LatestWorkflowCheckpointIdMetadataKey] = checkpoint.CheckpointId; + return stream.EmitInProgress(); + } // Check whenever the agent is storing messages when it should not. bool CheckNotAllowedStoreUsage() => @@ -386,7 +513,8 @@ bool CheckNotAllowedStoreUsage() => agent.RunStreamingAsync(messages, session, options: options, cancellationToken: consentCts.Token), stream, session?.StateBag, - cancellationToken).GetAsyncEnumerator(cancellationToken); + persistWorkflowCheckpointHandler: PersistWorkflowCheckpointAsync, + cancellationToken: cancellationToken).GetAsyncEnumerator(cancellationToken); try { while (true) @@ -408,6 +536,8 @@ bool CheckNotAllowedStoreUsage() => } evt = enumerator.Current; + shutdownDetected = + context.IsShutdownRequested && !emittedTerminal; } catch (OperationCanceledException) when (!emittedTerminal && consentState.Pending is not null) { @@ -418,6 +548,10 @@ bool CheckNotAllowedStoreUsage() => { shutdownDetected = true; } + catch (OperationCanceledException) when (context.PendingInputCount > 0 && !emittedTerminal) + { + steeringDetected = true; + } catch (Exception ex) when (ex is not OperationCanceledException && !emittedTerminal) { // Catch agent execution errors and emit a proper failed event @@ -467,12 +601,40 @@ bool CheckNotAllowedStoreUsage() => if (shutdownDetected) { - // Server is shutting down — emit incomplete so clients can resume + // Server is shutting down. On a resilient turn, leave the response in_progress + // so AgentServer can re-invoke this handler in a later process. The restored + // AgentSession determines how the agent continues. On a non-resilient turn, + // preserve the existing behavior and emit incomplete. + if (isResilientTurn) + { + this._logger.LogInformation("Shutdown detected on a resilient turn; deferring for recovery."); + deferredForRecovery = true; + await context.ExitForRecoveryAsync(cancellationToken).ConfigureAwait(false); + yield break; + } + this._logger.LogInformation("Shutdown detected, emitting incomplete response."); yield return stream.EmitIncomplete(); yield break; } + if (steeringDetected) + { + // AgentServer cancelled this active turn because another input is queued for the + // same conversation. Finish the current response cleanly so Core can drain the + // queued input as a new handler invocation. The MAF AgentSession is saved in the + // outer finally block and becomes the starting state for that invocation. + if (this._logger.IsEnabled(LogLevel.Information)) + { + this._logger.LogInformation( + "Steering input detected for response {ResponseId}; completing the active turn.", + context.ResponseId); + } + emittedTerminal = true; + yield return stream.EmitCompleted(); + yield break; + } + // A completed event is held back rather than sent straight out. The id of any // conversation the agent's own service kept only lands on the session once the run is // fully wound up, which is after this point, so sending the event now could tell the @@ -484,9 +646,41 @@ bool CheckNotAllowedStoreUsage() => continue; } + // Emit the output boundary before saving the matching MAF session. AgentServer + // persists the event while this iterator is suspended. Reversing this order could + // advance the session past output that the caller never received. A crash after the + // event but before the save can replay work, which is the deliberate at-least-once + // side of this cross-store boundary. // yield is in the outer try (finally-only) — allowed by C# yield return evt!; + // Best-effort session snapshot for a non-workflow agent after a response output item + // closes. Workflow agents save only at the paired superstep boundary so their session + // cursor cannot advance independently of the response snapshot. The final save below + // remains authoritative for a turn that reaches normal completion. + if (isResilientTurn + && evt is ResponseOutputItemDoneEvent + && workflowCheckpointRecovery is null + && session is not null + && !string.IsNullOrWhiteSpace(agentSessionId) + && !turnFailed) + { + try + { + await sessionStore.SaveSessionAsync(agent, agentSessionId!, session, resolvedUserId, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + if (this._logger.IsEnabled(LogLevel.Debug)) + { + this._logger.LogDebug( + ex, + "Incremental session save was skipped for response {ResponseId}; the end-of-turn save will persist the final state.", + context.ResponseId); + } + } + } + if (evt is ResponseFailedEvent or ResponseIncompleteEvent) { emittedTerminal = true; @@ -504,15 +698,16 @@ bool CheckNotAllowedStoreUsage() => turnFailed = true; } - // Persist the session for the next turn of this conversation, unless this one is being failed. - if (session is not null && !turnFailed) + // Persist the session for the next turn unless this turn failed or deferred after the + // agent advanced beyond the last event emitted to AgentServer. + if (session is not null && !turnFailed && !deferredForRecovery) { await sessionStore.SaveSessionAsync( agent, agentSessionId!, session, resolvedUserId, - cancellationToken).ConfigureAwait(false); + steeringDetected ? CancellationToken.None : cancellationToken).ConfigureAwait(false); } } @@ -573,6 +768,16 @@ private static string NewOAuthConsentItemId() return "oacr_" + Convert.ToHexString(bytes); } + /// + /// The resilience gate for the mid-turn session-save path: saving is worthwhile only when the + /// host enabled resilient background responses and this specific request is a background response + /// that did not explicitly disable storage. A null store value means the Responses API + /// default of true. When any part is false, the request runs exactly as it does on a non-resilient + /// host (the recovery path is gated separately on ResponseContext.IsRecovery). + /// + private bool ShouldPersistForResilience(CreateResponse request) + => this._resilientBackground && request.Background == true && request.Store != false; + /// /// Resolves an from the request. /// Tries agent.name first, then falls back to metadata["entity_id"]. diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryJsonCheckpointStore.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryJsonCheckpointStore.cs index 968c6634897..32af078b656 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryJsonCheckpointStore.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryJsonCheckpointStore.cs @@ -63,10 +63,10 @@ public sealed class FoundryJsonCheckpointStore : JsonCheckpointStore public const string DefaultStoreName = "agent-framework/checkpoints"; /// - /// How many times a losing index update is retried before giving up. Each attempt re-reads the - /// index, so a retry only happens when another writer committed a checkpoint in between. + /// The default number of attempts to update a workflow checkpoint index after concurrent writers + /// modify it. /// - private const int MaxIndexUpdateAttempts = 8; + public const int DefaultMaxIndexUpdateAttempts = 8; /// The item-body field holding the serialized checkpoint JSON. private const string CheckpointField = "checkpoint"; @@ -83,6 +83,7 @@ public sealed class FoundryJsonCheckpointStore : JsonCheckpointStore private readonly FoundryStateStoreBinding _binding; private readonly ILogger? _logger; + private readonly int _maxIndexUpdateAttempts; /// /// Initializes a new instance of the class. @@ -114,10 +115,43 @@ public FoundryJsonCheckpointStore( string storeName = DefaultStoreName, int itemTtlSeconds = FoundryStateStore.DefaultItemTtlSeconds, ILoggerFactory? loggerFactory = null) + : this( + DefaultMaxIndexUpdateAttempts, + endpoint, + credential, + storeName, + itemTtlSeconds, + loggerFactory) + { + } + + /// + /// Initializes a new instance of the class with a + /// configurable checkpoint-index update limit. + /// + /// + /// The maximum number of attempts to update a workflow checkpoint index when another writer + /// modifies it concurrently. Each retry re-reads the index before writing. Must be greater than + /// zero. + /// + /// The Foundry project endpoint, or to resolve it from the environment. + /// The credential used for hosted state storage. May be outside Foundry. + /// The state-store name to hold the checkpoints. + /// How long a checkpoint survives without being written, in seconds. + /// Creates the logger this store reports through. + public FoundryJsonCheckpointStore( + int maxIndexUpdateAttempts, + Uri? endpoint = null, + TokenCredential? credential = null, + string storeName = DefaultStoreName, + int itemTtlSeconds = FoundryStateStore.DefaultItemTtlSeconds, + ILoggerFactory? loggerFactory = null) { _ = Throw.IfNullOrWhitespace(storeName); + ArgumentOutOfRangeException.ThrowIfLessThan(maxIndexUpdateAttempts, 1); this.StoreName = storeName; + this._maxIndexUpdateAttempts = maxIndexUpdateAttempts; this._logger = loggerFactory?.CreateLogger(); this._binding = new(cancellationToken => FoundryStateStore.GetOrCreateAsync( storeName, @@ -135,15 +169,22 @@ public FoundryJsonCheckpointStore( /// Resolves the bound state store on first use. /// The state-store name, for diagnostics. /// Creates the logger this store reports through. + /// + /// The maximum number of attempts to update a workflow checkpoint index after a concurrent + /// modification. + /// internal FoundryJsonCheckpointStore( Func> storeFactory, string storeName = DefaultStoreName, - ILoggerFactory? loggerFactory = null) + ILoggerFactory? loggerFactory = null, + int maxIndexUpdateAttempts = DefaultMaxIndexUpdateAttempts) { _ = Throw.IfNull(storeFactory); + ArgumentOutOfRangeException.ThrowIfLessThan(maxIndexUpdateAttempts, 1); this._binding = new(storeFactory); this.StoreName = storeName; + this._maxIndexUpdateAttempts = maxIndexUpdateAttempts; this._logger = loggerFactory?.CreateLogger(); } @@ -176,7 +217,7 @@ await store.SetItemAsync( // Announce the stored checkpoint by appending its identifier to the session's index, giving // way and reading again whenever another instance updated that same index first. - for (int attempt = 0; attempt < MaxIndexUpdateAttempts; attempt++) + for (int attempt = 0; attempt < this._maxIndexUpdateAttempts; attempt++) { StateStoreItem? indexItem = await store.GetItemAsync(sessionIndexKey, CancellationToken.None).ConfigureAwait(false); List entries = ReadEntries(indexItem); @@ -207,7 +248,7 @@ await store.SetItemAsync( ex, "Attempt {Attempt} of {MaxAttempts} to index checkpoint '{CheckpointId}' for session '{SessionId}' lost to another writer. Retrying.", attempt + 1, - MaxIndexUpdateAttempts, + this._maxIndexUpdateAttempts, checkpointInfo.CheckpointId, sessionId); } @@ -217,7 +258,7 @@ await store.SetItemAsync( } throw new InvalidOperationException( - $"Could not add a checkpoint for session '{sessionId}' to the Foundry state store after {MaxIndexUpdateAttempts} attempts because other writers kept updating the same session index."); + $"Could not add a checkpoint for session '{sessionId}' to the Foundry state store after {this._maxIndexUpdateAttempts} attempts because other writers kept updating the same session index."); } /// diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryResponsesOptions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryResponsesOptions.cs index d326ce230ac..984a624c945 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryResponsesOptions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/FoundryResponsesOptions.cs @@ -50,4 +50,50 @@ public sealed class FoundryResponsesOptions /// Default is . /// public bool IncludeReasoningEncryptedContent { get; set; } = true; + + /// + /// Gets or sets a value indicating whether background responses are resilient to process crashes + /// and graceful shutdown. + /// + /// + /// + /// When , accepted background responses (background=true and + /// store omitted or ) are registered with the durable task subsystem + /// so a handler interrupted by a crash or shutdown is + /// re-invoked in a subsequent process lifetime with the original request context restored + /// (ResponseContext.IsRecovery is ). AgentServer supplies its last durable + /// response snapshot. For workflow agents, the hosting handler pairs completed supersteps with response + /// checkpoints and records the matching workflow checkpoint ID in AgentServer internal response metadata. + /// Recovery restores the AgentSession, selects that exact workflow checkpoint, skips re-injecting the + /// original input, and defers on shutdown instead of ending the response as incomplete. Regular agents + /// continue to depend on their serialized AgentSession state. + /// + /// + /// When (the default), an interrupted background response transitions to a + /// failed terminal state and is not re-invoked. The hosting handler does not perform resilient + /// mid-turn session saves or shutdown deferral. + /// + /// + /// This value is forwarded to + /// . + /// + /// + /// + /// Default is . + /// + public bool ResilientBackground { get; set; } + + /// + /// Gets or sets a value indicating whether in-flight conversations accept steering (mid-turn + /// additional input) sharing a single resilient task. + /// + /// + /// Forwarded to + /// . + /// When (the default), steering is disabled. + /// + /// + /// Default is . + /// + public bool SteerableConversations { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs index 75ae37e67b5..27be4dcdea4 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/OutputConverter.cs @@ -32,6 +32,9 @@ internal static class OutputConverter /// The agent response updates to convert. /// The SDK event stream builder. /// Optional session state bag used to persist tool-approval id mappings across turns. + /// + /// Optional callback invoked after all output from a completed workflow superstep has been closed. + /// /// Cancellation token. /// An async enumerable of SDK response stream events (excluding lifecycle events). [UnconditionalSuppressMessage("Trimming", "IL2026", Justification = "Serializing function call arguments dictionary.")] @@ -40,6 +43,7 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents IAsyncEnumerable updates, ResponseEventStream stream, AgentSessionStateBag? stateBag = null, + Func>? persistWorkflowCheckpointHandler = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { ResponseUsage? accumulatedUsage = null; @@ -78,6 +82,21 @@ public static async IAsyncEnumerable ConvertUpdatesToEvents yield return evt; } + if (workflowEvent is SuperStepCompletedEvent { CompletionInfo.Checkpoint: { } checkpoint } + && persistWorkflowCheckpointHandler is not null) + { + ResponseStreamEvent? checkpointStateEvent = + await persistWorkflowCheckpointHandler(checkpoint, cancellationToken).ConfigureAwait(false); + if (checkpointStateEvent is not null) + { + // AgentServer persists its orchestrator-owned response snapshot. Emit the + // updated response state first so internal metadata becomes part of that + // authoritative snapshot, then persist it with the control event. + yield return checkpointStateEvent; + yield return stream.Checkpoint(); + } + } + continue; } diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs index 08f9fdc57ed..45c332c0895 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry.Hosting/ServiceCollectionExtensions.cs @@ -58,18 +58,27 @@ public static class FoundryHostingExtensions /// The service collection. /// /// Optional callback to configure , for example to allow the - /// agent's own service to store the responses it produces. + /// agent's own service to store the responses it produces, or to opt in to durable long-running + /// (resilient) background responses via . /// /// The service collection for chaining. public static IServiceCollection AddFoundryResponses(this IServiceCollection services, Action? configure = null) { _ = Throw.IfNull(services); - AddResponsesServerOnce(services); + FoundryResponsesOptions configuredOptions = CreateFoundryResponsesOptions(configure); + bool serverAdded = AddResponsesServerOnce( + services, + configuredOptions, + configure is not null); services.AddHealthChecks(); ConfigureFoundryListenPort(services); - ConfigureFoundryResponsesOptions(services, configure); + ConfigureFoundryResponsesOptions( + services, + configuredOptions, + includeServerOptions: serverAdded, + applyOptions: serverAdded || configure is not null); services.TryAddSingleton(_ => CreateDefaultAgentSessionStore()); - services.TryAddSingleton(); + RegisterResponseHandler(services); MarkFeatureUsed(); return services; } @@ -99,7 +108,8 @@ public static IServiceCollection AddFoundryResponses(this IServiceCollection ser /// The agent session store to use for managing agent sessions server-side. If null, is used: the Foundry durable state store when hosted, and the AgentServer SDK's local state-store fallback otherwise. /// /// Optional callback to configure , for example to allow the - /// agent's own service to store the responses it produces. + /// agent's own service to store the responses it produces, or to opt in to durable long-running + /// (resilient) background responses via . /// /// The service collection for chaining. public static IServiceCollection AddFoundryResponses( @@ -111,10 +121,18 @@ public static IServiceCollection AddFoundryResponses( _ = Throw.IfNull(services); _ = Throw.IfNull(agent); - AddResponsesServerOnce(services); + FoundryResponsesOptions configuredOptions = CreateFoundryResponsesOptions(configure); + bool serverAdded = AddResponsesServerOnce( + services, + configuredOptions, + configure is not null); services.AddHealthChecks(); ConfigureFoundryListenPort(services); - ConfigureFoundryResponsesOptions(services, configure); + ConfigureFoundryResponsesOptions( + services, + configuredOptions, + includeServerOptions: serverAdded, + applyOptions: serverAdded || configure is not null); agentSessionStore ??= CreateDefaultAgentSessionStore(); if (!string.IsNullOrWhiteSpace(agent.Name)) @@ -128,7 +146,7 @@ public static IServiceCollection AddFoundryResponses( services.TryAddSingleton(agent); services.TryAddSingleton(agentSessionStore); - services.TryAddSingleton(); + RegisterResponseHandler(services); MarkFeatureUsed(); return services; } @@ -142,12 +160,44 @@ public static IServiceCollection AddFoundryResponses( /// The checks are registered on the same /readiness pipeline that /// maps, so such a container never takes traffic. /// AddCheck does not dedupe by name, so a repeated registration is guarded here. + /// Resilience flags on are forwarded to + /// so the AgentServer SDK enables recovery for the same host. /// - private static void ConfigureFoundryResponsesOptions(IServiceCollection services, Action? configure) + private static FoundryResponsesOptions CreateFoundryResponsesOptions(Action? configure) { - if (configure is not null) + FoundryResponsesOptions options = new(); + configure?.Invoke(options); + return options; + } + + private static void RegisterResponseHandler(IServiceCollection services) + { + services.TryAddSingleton(serviceProvider => + new AgentFrameworkResponseHandler( + serviceProvider, + serviceProvider.GetRequiredService>(), + serviceProvider.GetRequiredService>(), + serviceProvider.GetService())); + } + + private static void ConfigureFoundryResponsesOptions( + IServiceCollection services, + FoundryResponsesOptions configuredOptions, + bool includeServerOptions, + bool applyOptions) + { + if (applyOptions) { - services.Configure(configure); + services.Configure(options => + { + options.AllowStoredOutputEnabled = configuredOptions.AllowStoredOutputEnabled; + options.IncludeReasoningEncryptedContent = configuredOptions.IncludeReasoningEncryptedContent; + if (includeServerOptions) + { + options.ResilientBackground = configuredOptions.ResilientBackground; + options.SteerableConversations = configuredOptions.SteerableConversations; + } + }); } AddReadinessCheckOnce(services, "foundry-stored-output", sp => ActivatorUtilities.CreateInstance(sp)); @@ -358,15 +408,37 @@ private static void MarkFeatureUsed() /// a host that registers several agents naturally does, so the second and later calls are /// skipped here. /// - private static void AddResponsesServerOnce(IServiceCollection services) + private static bool AddResponsesServerOnce( + IServiceCollection services, + FoundryResponsesOptions configuredOptions, + bool hasConfigureCallback) { - if (services.Any(static d => d.ServiceType == typeof(FoundryResponsesServerMarker))) + FoundryResponsesServerMarker? marker = services + .LastOrDefault(static descriptor => + descriptor.ServiceType == typeof(FoundryResponsesServerMarker)) + ?.ImplementationInstance as FoundryResponsesServerMarker; + if (marker is not null) { - return; + if (hasConfigureCallback + && ((!marker.ResilientBackground && configuredOptions.ResilientBackground) + || (!marker.SteerableConversations && configuredOptions.SteerableConversations))) + { + throw new InvalidOperationException( + "ResilientBackground and SteerableConversations must be configured on the first AddFoundryResponses call because AgentServer registers its durable tasks during that call."); + } + + return false; } - services.AddSingleton(); - services.AddResponsesServer(); + services.AddSingleton(new FoundryResponsesServerMarker( + configuredOptions.ResilientBackground, + configuredOptions.SteerableConversations)); + services.AddResponsesServer(options => + { + options.ResilientBackground = configuredOptions.ResilientBackground; + options.SteerableConversations = configuredOptions.SteerableConversations; + }); + return true; } /// @@ -407,7 +479,14 @@ private sealed class FoundryListenPortMarker; /// Marker registered once per so the Responses Server SDK is /// registered at most once, even across multiple AddFoundryResponses calls. /// - private sealed class FoundryResponsesServerMarker; + private sealed class FoundryResponsesServerMarker( + bool resilientBackground, + bool steerableConversations) + { + public bool ResilientBackground { get; } = resilientBackground; + + public bool SteerableConversations { get; } = steerableConversations; + } /// /// Binds Kestrel to the port the Foundry hosted runtime probes and routes to, so a plain diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowAgentMetadata.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowAgentMetadata.cs index f6ea2135a42..60bc9a4f4ee 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowAgentMetadata.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowAgentMetadata.cs @@ -10,7 +10,7 @@ namespace Microsoft.Agents.AI.Workflows; /// /// Retrieve it with agent.GetService<WorkflowAgentMetadata>(). Getting an instance back /// is what identifies the agent as running a workflow; means it does not. -/// Going through means the answer is still +/// Going through means the answer is still /// found when the agent has been wrapped, by middleware for example, which a test on the type of the /// agent would miss. /// diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSession.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSession.cs index 4e262a6f319..97134a94073 100644 --- a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSession.cs +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSession.cs @@ -31,6 +31,8 @@ internal sealed class WorkflowSession : AgentSession private readonly bool _includeWorkflowOutputsInResponse; private InMemoryCheckpointManager? _inMemoryCheckpointManager; + private bool _resumeWithoutNewTurn; + private WorkflowSessionCheckpointRecovery? _checkpointRecovery; /// /// Tracks pending external requests by their workflow-facing request ID. @@ -132,6 +134,31 @@ public WorkflowSession(Workflow workflow, JsonElement serializedSession, IWorkfl public CheckpointInfo? LastCheckpoint { get; set; } + /// + public override object? GetService(Type serviceType, object? serviceKey = null) + { + return base.GetService(serviceType, serviceKey) + ?? (serviceKey is null && serviceType == typeof(WorkflowSessionCheckpointRecovery) + ? this._checkpointRecovery ??= new(this) + : null); + } + + internal bool TryPrepareCheckpointRecovery(string? checkpointId) + { + if (checkpointId is not null) + { + _ = Throw.IfNullOrWhitespace(checkpointId); + this.LastCheckpoint = new CheckpointInfo(this.SessionId, checkpointId); + } + else if (this.LastCheckpoint is null) + { + return false; + } + + this._resumeWithoutNewTurn = true; + return true; + } + internal JsonElement Serialize(JsonSerializerOptions? jsonSerializerOptions = null) { JsonMarshaller marshaller = new(jsonSerializerOptions); @@ -449,6 +476,8 @@ IAsyncEnumerable InvokeStageAsync( ResumeRunResult resumeResult = await this.CreateOrResumeRunAsync(messages, cancellationToken).ConfigureAwait(false); + bool resumeWithoutNewTurn = this._resumeWithoutNewTurn; + this._resumeWithoutNewTurn = false; #pragma warning disable CA2007 // Analyzer misfiring. await using StreamingRun run = resumeResult.Run; @@ -462,8 +491,9 @@ IAsyncEnumerable InvokeStageAsync( // ContinueTurnAsync). Non-start executors (e.g., RequestInfoExecutor) do not emit // TurnTokens after processing responses, so the session must always provide one. bool shouldSendTurnToken = - !dispatchInfo.HasMatchedExternalResponses - || !dispatchInfo.HasMatchedResponseForStartExecutor; + !resumeWithoutNewTurn + && (!dispatchInfo.HasMatchedExternalResponses + || !dispatchInfo.HasMatchedResponseForStartExecutor); if (shouldSendTurnToken) { await run.TrySendMessageAsync(new TurnToken(emitEvents: true)).ConfigureAwait(false); diff --git a/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSessionCheckpointRecovery.cs b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSessionCheckpointRecovery.cs new file mode 100644 index 00000000000..99df9a5b22a --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Workflows/WorkflowSessionCheckpointRecovery.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; +using Microsoft.Shared.DiagnosticIds; + +namespace Microsoft.Agents.AI.Workflows; + +/// +/// Prepares a workflow-backed to continue from a workflow checkpoint. +/// +/// +/// +/// Retrieve this service from a workflow-backed session with +/// session.GetService<WorkflowSessionCheckpointRecovery>(). Other session types return +/// . +/// +/// +/// This service prepares recovery of an interrupted run. It is not a general rollback mechanism. +/// The selected checkpoint must belong to the same serialized session state, workflow definition, +/// and checkpoint store. Selecting an older checkpoint can repeat external effects. +/// +/// +[Experimental(DiagnosticIds.Experiments.AgentsAIExperiments)] +public sealed class WorkflowSessionCheckpointRecovery +{ + private readonly WorkflowSession _session; + + internal WorkflowSessionCheckpointRecovery(WorkflowSession session) + { + this._session = session; + } + + /// + /// Gets the checkpoint currently selected by the workflow session. + /// + public CheckpointInfo? CurrentCheckpoint => this._session.LastCheckpoint; + + /// + /// Prepares the session to continue the work queued in a workflow checkpoint without starting + /// a new user turn. + /// + /// + /// The checkpoint identifier to select. When , the session keeps its + /// current checkpoint. + /// + /// + /// when a checkpoint is available for recovery; otherwise + /// . + /// + public bool TryPrepare(string? checkpointId = null) => + this._session.TryPrepareCheckpointRecovery(checkpointId); +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Foundry.Hosting.IntegrationTests.TestContainer.csproj b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Foundry.Hosting.IntegrationTests.TestContainer.csproj index fff625f782f..21492c77b22 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Foundry.Hosting.IntegrationTests.TestContainer.csproj +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Foundry.Hosting.IntegrationTests.TestContainer.csproj @@ -30,6 +30,7 @@ + diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs index 34e74a04a7c..323495c29e4 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/Program.cs @@ -46,6 +46,8 @@ "session-files" => CreateSessionFilesAgent(projectClient, deployment), "agent-skills" => CreateAgentSkillsAgent(projectClient, deployment), "user-identity" => CreateUserIdentityAgent(projectClient, deployment), + "resilient-workflow" => ResilientWorkflowAgent.Create(), + "steerable-long-running" => new SteerableLongRunningAgent(), _ => throw new InvalidOperationException($"Unknown IT_SCENARIO '{scenario}'.") }; @@ -57,7 +59,12 @@ builder.WebHost.UseUrls($"http://+:{port}"); } -builder.Services.AddFoundryResponses(agent); +builder.Services.AddFoundryResponses(agent, configure: options => +{ + options.ResilientBackground = + scenario is "resilient-workflow" or "steerable-long-running"; + options.SteerableConversations = scenario == "steerable-long-running"; +}); // toolbox-oauth-consent scenario: pre-register a Foundry toolbox whose tool source is fronted by a // per-user OAuth connection. IT_TOOLBOX_NAME names that toolbox (the fixture sets it). With the diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/ResilientWorkflowAgent.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/ResilientWorkflowAgent.cs new file mode 100644 index 00000000000..949b2ed8f68 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/ResilientWorkflowAgent.cs @@ -0,0 +1,372 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Globalization; +using System.Security.Cryptography; +using System.Text; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; + +namespace Foundry.Hosting.IntegrationTests.TestContainer; + +internal static class ResilientWorkflowAgent +{ + // Agent Administration tracks the durable session, not individual process lifetimes. + // Persist our own incarnation so recovery can prove that a different process continued the work. + private static readonly string s_processIncarnation = Guid.NewGuid().ToString("N"); + + public static AIAgent Create() + { + ResilientInputExecutor input = new(); + ResilientWorkExecutor work = new(); + ResilientOutputExecutor output = new(); + ResilientCountdownExecutor countdown = new(); + ResilientCountdownCrashExecutor countdownCrash = new(); + ResilientCountdownCompleteExecutor countdownComplete = new(); + + return new WorkflowBuilder(input) + .AddEdge(input, work) + .AddEdge(input, countdown) + .AddEdge(work, output) + .AddEdge(countdown, countdown) + .AddEdge(countdown, countdownCrash) + .AddEdge(countdown, countdownComplete) + .AddEdge(countdownCrash, countdown) + .WithOutputFrom(output, countdown, countdownComplete) + .Build() + .AsAIAgent( + name: "resilient-workflow-agent", + includeExceptionDetails: true, + includeWorkflowOutputsInResponse: true); + } + + private sealed class ResilientInputExecutor() + : ChatProtocolExecutor("resilient-input", new() { AutoSendTurnToken = false }) + { + protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder) => + base.ConfigureProtocol(protocolBuilder).SendsMessage(); + + protected override ValueTask TakeTurnAsync( + List messages, + IWorkflowContext context, + bool? emitEvents, + CancellationToken cancellationToken = default) + { + string request = messages.LastOrDefault()?.Text + ?? throw new InvalidOperationException("The resilient workflow requires an input message."); + string targetId = request.StartsWith("countdown:", StringComparison.Ordinal) + ? "resilient-countdown" + : "resilient-work"; + return context.SendMessageAsync( + request, + targetId: targetId, + cancellationToken: cancellationToken); + } + } + + private sealed class ResilientWorkExecutor() + : Executor("resilient-work") + { + public override async ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + string[] parts = message.Split(':', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2 || string.IsNullOrWhiteSpace(parts[1])) + { + throw new InvalidOperationException("Expected ':'."); + } + + string mode = parts[0]; + string token = parts[1]; + + if (string.Equals(mode, "long", StringComparison.Ordinal)) + { + int delaySeconds = GetLongRunningDelaySeconds(); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds), cancellationToken).ConfigureAwait(false); + return $"LONG-RUN-COMPLETE:{token}"; + } + + if (string.Equals(mode, "crash", StringComparison.Ordinal)) + { + if (TryCreateCrashMarker(token, out string crashedProcessIncarnation)) + { + Console.Out.Flush(); + Console.Error.Flush(); + Environment.Exit(70); + throw new InvalidOperationException("Process termination did not stop execution."); + } + + if (string.Equals(crashedProcessIncarnation, s_processIncarnation, StringComparison.Ordinal)) + { + throw new InvalidOperationException("The crash recovery stage resumed in the original process."); + } + + return $"CRASH-RECOVERED:{token}:PROCESS-CHANGED"; + } + + throw new InvalidOperationException($"Unknown resilient workflow mode '{mode}'."); + } + + private static int GetLongRunningDelaySeconds() + { + const int DefaultDelaySeconds = 20; + string? value = Environment.GetEnvironmentVariable("IT_LONG_RUNNING_DELAY_SECONDS"); + return int.TryParse(value, out int seconds) && seconds > 0 ? seconds : DefaultDelaySeconds; + } + } + + [SendsMessage(typeof(string))] + [YieldsOutput(typeof(string))] + private sealed class ResilientCountdownExecutor() + : Executor("resilient-countdown") + { + public override async ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + CountdownState state = CountdownState.Parse(message); + if (state.Current <= 0) + { + await context.SendMessageAsync( + "Countdown complete.", + targetId: "resilient-countdown-complete", + cancellationToken: cancellationToken).ConfigureAwait(false); + return; + } + + await Task.Delay( + TimeSpan.FromMilliseconds(GetCountdownDelayMilliseconds()), + cancellationToken).ConfigureAwait(false); + await context.YieldOutputAsync( + state.Current.ToString(CultureInfo.InvariantCulture), + cancellationToken).ConfigureAwait(false); + + CountdownState next = state with { Current = state.Current - 1 }; + string targetId = state.Current == state.CrashAtValue + ? "resilient-countdown-crash" + : "resilient-countdown"; + await context.SendMessageAsync( + next.ToString(), + targetId: targetId, + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static int GetCountdownDelayMilliseconds() + { + const int DefaultDelayMilliseconds = 250; + string? value = Environment.GetEnvironmentVariable( + "IT_COUNTDOWN_DELAY_MILLISECONDS"); + return int.TryParse( + value, + NumberStyles.None, + CultureInfo.InvariantCulture, + out int milliseconds) + && milliseconds >= 0 + ? milliseconds + : DefaultDelayMilliseconds; + } + } + + [SendsMessage(typeof(string))] + private sealed class ResilientCountdownCrashExecutor() + : Executor("resilient-countdown-crash") + { + public override async ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + CountdownState state = CountdownState.Parse(message); + if (TryCreateCrashMarker( + state.Token, + out string crashedProcessIncarnation)) + { + await Task.Delay( + TimeSpan.FromSeconds(GetCountdownCrashDelaySeconds()), + cancellationToken).ConfigureAwait(false); + Console.Out.Flush(); + Console.Error.Flush(); + Environment.Exit(70); + throw new InvalidOperationException( + "Process termination did not stop execution."); + } + + if (string.Equals( + crashedProcessIncarnation, + s_processIncarnation, + StringComparison.Ordinal)) + { + throw new InvalidOperationException( + "The countdown resumed in the original process."); + } + + await context.SendMessageAsync( + state.ToString(), + targetId: "resilient-countdown", + cancellationToken: cancellationToken).ConfigureAwait(false); + } + + private static int GetCountdownCrashDelaySeconds() + { + const int DefaultDelaySeconds = 5; + string? value = Environment.GetEnvironmentVariable( + "IT_COUNTDOWN_CRASH_DELAY_SECONDS"); + return int.TryParse( + value, + NumberStyles.None, + CultureInfo.InvariantCulture, + out int seconds) + && seconds >= 0 + ? seconds + : DefaultDelaySeconds; + } + } + + [YieldsOutput(typeof(string))] + private sealed class ResilientCountdownCompleteExecutor() + : Executor("resilient-countdown-complete") + { + public override ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) => + context.YieldOutputAsync(message, cancellationToken); + } + + [YieldsOutput(typeof(string))] + private sealed class ResilientOutputExecutor() + : Executor("resilient-output") + { + public override async ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + await context.YieldOutputAsync(message, cancellationToken).ConfigureAwait(false); + } + } + + private static bool TryCreateCrashMarker( + string token, + out string crashedProcessIncarnation) + { + string home = Environment.GetEnvironmentVariable("HOME") + ?? throw new InvalidOperationException("HOME is not set."); + string markerDirectory = Path.Combine( + home, + ".foundry-hosting-it", + "resilient-workflow"); + Directory.CreateDirectory(markerDirectory); + + string markerName = + Convert.ToHexString( + SHA256.HashData(Encoding.UTF8.GetBytes(token))) + + ".crashed"; + string markerPath = Path.Combine(markerDirectory, markerName); + + try + { + using FileStream marker = new( + markerPath, + FileMode.CreateNew, + FileAccess.Write, + FileShare.None, + bufferSize: 1, + FileOptions.WriteThrough); + byte[] incarnation = Encoding.UTF8.GetBytes( + s_processIncarnation); + marker.Write(incarnation); + marker.Flush(flushToDisk: true); + crashedProcessIncarnation = s_processIncarnation; + return true; + } + catch (IOException) when (File.Exists(markerPath)) + { + crashedProcessIncarnation = + File.ReadAllText(markerPath, Encoding.UTF8).Trim(); + if (string.IsNullOrWhiteSpace(crashedProcessIncarnation)) + { + throw new InvalidOperationException( + "The crash marker does not contain a process incarnation."); + } + + return false; + } + } + + private sealed record CountdownState( + int Current, + int CrashAtValue, + string Token) + { + private const string InitialPrefix = "countdown"; + private const string StatePrefix = "countdown-state"; + + public static CountdownState Parse(string value) + { + string[] parts = value.Split( + ':', + 4, + StringSplitOptions.TrimEntries); + if (parts.Length != 4 + || !int.TryParse( + parts[1], + NumberStyles.None, + CultureInfo.InvariantCulture, + out int first) + || !int.TryParse( + parts[2], + NumberStyles.None, + CultureInfo.InvariantCulture, + out int second) + || string.IsNullOrWhiteSpace(parts[3])) + { + throw new InvalidOperationException( + "Expected 'countdown:::' " + + "or a valid countdown state."); + } + + if (string.Equals( + parts[0], + InitialPrefix, + StringComparison.Ordinal)) + { + if (first < 2 || second < 1 || second >= first) + { + throw new InvalidOperationException( + "Countdown target must be at least 2 and the crash count " + + "must be between 1 and target minus 1."); + } + + return new( + Current: first, + CrashAtValue: first - second + 1, + Token: parts[3]); + } + + if (string.Equals( + parts[0], + StatePrefix, + StringComparison.Ordinal) + && first >= 0 + && second > 0) + { + return new( + Current: first, + CrashAtValue: second, + Token: parts[3]); + } + + throw new InvalidOperationException( + "The countdown state prefix or values are invalid."); + } + + public override string ToString() => + string.Create( + CultureInfo.InvariantCulture, + $"{StatePrefix}:{this.Current}:{this.CrashAtValue}:{this.Token}"); + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/SteerableLongRunningAgent.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/SteerableLongRunningAgent.cs new file mode 100644 index 00000000000..ad21dfb43b4 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests.TestContainer/SteerableLongRunningAgent.cs @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; + +namespace Foundry.Hosting.IntegrationTests.TestContainer; + +internal sealed class SteerableLongRunningAgent : AIAgent +{ + private int _activeRuns; + private int _maxConcurrentRuns; + + public override string? Name => "steerable-long-running-agent"; + + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var steeringSession = session as SteeringSession + ?? throw new InvalidOperationException("The steering agent requires a SteeringSession."); + int activeRuns = Interlocked.Increment(ref this._activeRuns); + UpdateMaximum(ref this._maxConcurrentRuns, activeRuns); + + try + { + int sessionTurn = ++steeringSession.Turn; + string input = string.Join( + "\n", + messages.Select(message => message.Text).Where(text => text is not null)); + string[] parts = input.Split(':', 2, StringSplitOptions.TrimEntries); + if (parts.Length != 2 || string.IsNullOrWhiteSpace(parts[1])) + { + throw new InvalidOperationException("Expected ':'."); + } + + string mode = parts[0]; + string token = parts[1]; + if (string.Equals(mode, "first", StringComparison.Ordinal)) + { + yield return NewUpdate( + $"FIRST-STARTED:{token}:SESSION-TURN-{sessionTurn}"); + + int delaySeconds = GetLongRunningDelaySeconds(); + await Task.Delay( + TimeSpan.FromSeconds(delaySeconds), + cancellationToken).ConfigureAwait(false); + + yield return NewUpdate( + $"FIRST-NATURAL-COMPLETE:{token}:SESSION-TURN-{sessionTurn}"); + yield break; + } + + if (string.Equals(mode, "steer", StringComparison.Ordinal)) + { + yield return NewUpdate( + $"STEERED-COMPLETE:{token}:SESSION-TURN-{sessionTurn}:" + + $"MAX-CONCURRENCY-{this.MaxConcurrentRuns}"); + yield break; + } + + throw new InvalidOperationException( + $"Unknown steerable long-running mode '{mode}'."); + } + finally + { + Interlocked.Decrement(ref this._activeRuns); + } + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(new SteeringSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + var steeringSession = session as SteeringSession + ?? throw new InvalidOperationException("The steering agent requires a SteeringSession."); + return new(JsonSerializer.SerializeToElement( + new SerializedSession(steeringSession.Turn), + jsonSerializerOptions)); + } + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + SerializedSession state = serializedState.Deserialize( + jsonSerializerOptions) + ?? throw new InvalidOperationException( + "Could not deserialize the steering session."); + return new(new SteeringSession { Turn = state.Turn }); + } + + private int MaxConcurrentRuns => Volatile.Read(ref this._maxConcurrentRuns); + + private static AgentResponseUpdate NewUpdate(string text) => + new() + { + MessageId = Guid.NewGuid().ToString("N"), + Contents = [new TextContent(text)], + }; + + private static int GetLongRunningDelaySeconds() + { + const int DefaultDelaySeconds = 30; + string? value = Environment.GetEnvironmentVariable( + "IT_STEERING_LONG_RUNNING_DELAY_SECONDS"); + return int.TryParse(value, out int seconds) && seconds > 0 + ? seconds + : DefaultDelaySeconds; + } + + private static void UpdateMaximum(ref int maximum, int candidate) + { + int current; + do + { + current = Volatile.Read(ref maximum); + if (candidate <= current) + { + return; + } + } + while (Interlocked.CompareExchange(ref maximum, candidate, current) != current); + } + + private sealed class SteeringSession : AgentSession + { + public int Turn { get; set; } + } + + private sealed record SerializedSession(int Turn); +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/ResilientWorkflowHostedAgentFixture.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/ResilientWorkflowHostedAgentFixture.cs new file mode 100644 index 00000000000..6281f8f23ba --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/ResilientWorkflowHostedAgentFixture.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Foundry.Hosting.IntegrationTests.Fixtures; + +/// +/// Provisions a hosted agent that runs the test container in +/// IT_SCENARIO=resilient-workflow mode. +/// +public sealed class ResilientWorkflowHostedAgentFixture : HostedAgentFixture +{ + protected override string ScenarioName => "resilient-workflow"; + + protected override TimeSpan ProvisioningTimeout => TimeSpan.FromMinutes(8); + + protected override void ConfigureEnvironment(IDictionary environment) + { + environment["IT_LONG_RUNNING_DELAY_SECONDS"] = "20"; + environment["IT_COUNTDOWN_DELAY_MILLISECONDS"] = "250"; + environment["IT_COUNTDOWN_CRASH_DELAY_SECONDS"] = "5"; + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/SteerableLongRunningHostedAgentFixture.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/SteerableLongRunningHostedAgentFixture.cs new file mode 100644 index 00000000000..1ca4cccb53d --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/Fixtures/SteerableLongRunningHostedAgentFixture.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; + +namespace Foundry.Hosting.IntegrationTests.Fixtures; + +/// +/// Provisions a hosted agent that runs the test container in +/// IT_SCENARIO=steerable-long-running mode. +/// +public sealed class SteerableLongRunningHostedAgentFixture : HostedAgentFixture +{ + protected override string ScenarioName => "steerable-long-running"; + + protected override TimeSpan ProvisioningTimeout => TimeSpan.FromMinutes(8); + + protected override void ConfigureEnvironment( + IDictionary environment) + { + environment["IT_STEERING_LONG_RUNNING_DELAY_SECONDS"] = "30"; + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md b/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md index 9828f0350cc..c20ba717ff4 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/README.md @@ -46,6 +46,19 @@ The container scenario injects `USER-ID:` via `x-agent-user-id`). The caller credential must be allowed to delegate via `x-ms-user-identity` or those tests fail with HTTP 403. +### Resilience and steering scenarios + +- `ResilientWorkflowHostedAgentTests` uses `IT_SCENARIO=resilient-workflow` to verify that a + background MAF workflow continues without client traffic and that a different process resumes it + after `Environment.Exit(70)`. Its countdown test receives `20` through `11`, ends the container + process, reconnects with the sequence-aware MAF continuation token, and verifies the recovered + accumulator contains exactly `20` through `1`. A third call uses the same agent and session with + the same response ID but no sequence cursor, and verifies the complete 20-item replay. +- `SteerableLongRunningHostedAgentTests` uses `IT_SCENARIO=steerable-long-running` to start a + background MAF turn, wait for its first streamed update, submit a second input on the same + conversation, assert `queued`, and verify that the persisted `AgentSession` advances to turn 2 + without concurrent MAF executions. + ## Required environment variables | Variable | Source | Purpose | @@ -61,7 +74,7 @@ The container scenario injects `USER-ID:` via Hosted agent invocation requires the agent's own managed identity to hold the `Azure AI User` role on the project scope. Because each agent's MI is created when the agent is first provisioned (and recycled on agent delete), the bootstrap creates the -eleven stable scenario agents once and grants the role to each MI. The fixture then only +stable scenario agents once and grants the role to each MI. The fixture then only manages versions under those existing agents, so the role grants survive across runs. ```powershell @@ -233,6 +246,7 @@ human-only operation; CI only adds and deletes versions under existing agents. | `AzureSearchRagHostedAgentFixture` | `azure-search-rag` | `it-azure-search-rag` | RAG against a real Azure AI Search index seeded with Contoso Outdoors documents; verifies the model cites the retrieved sources. | | `SessionFilesHostedAgentFixture` | `session-files` | `it-session-files` | End-to-end: upload via `AgentSessionFiles` (alpha) into a pinned `agent_session_id`, invoke the agent, assert it reads the file via the container's `ReadFile` tool. | | `AgentSkillsHostedAgentFixture` | `agent-skills` | `it-agent-skills` | Agent skills via `AgentSkillsProvider`: advertises two Contoso Outdoors skills (support-style, escalation-policy) in the system prompt, loads them on demand via `load_skill`, verifies canary tokens prove the skill was loaded. | +| `ResilientWorkflowHostedAgentFixture` | `resilient-workflow` | `it-resilient-workflow` | Stored background workflow remains active without client traffic, completes after an intentional container process crash, and replays a complete 20-item countdown without a sequence cursor. | The scenarios marked (placeholder) are already wired into the test container `Program.cs`, but their assertions stay skipped pending live validation and stabilization of the relevant diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/ResilientWorkflowHostedAgentTests.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/ResilientWorkflowHostedAgentTests.cs new file mode 100644 index 00000000000..570167aea6b --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/ResilientWorkflowHostedAgentTests.cs @@ -0,0 +1,428 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Foundry.Hosting.IntegrationTests.Fixtures; +using Microsoft.Agents.AI; +using Microsoft.Extensions.AI; +using OpenAI.Responses; + +#pragma warning disable OPENAI001 // Experimental Responses API surfaces + +namespace Foundry.Hosting.IntegrationTests; + +/// +/// Live long-running and crash-recovery tests for resilient background Responses hosting. +/// +[Trait("Category", "FoundryHostedAgents")] +public sealed class ResilientWorkflowHostedAgentTests(ResilientWorkflowHostedAgentFixture fixture) + : IClassFixture +{ + private static readonly TimeSpan s_completionTimeout = TimeSpan.FromMinutes(6); + private readonly ResilientWorkflowHostedAgentFixture _fixture = fixture; + + [Fact] + public async Task BackgroundResponse_ContinuesWithoutClientConnectionAsync() + { + // Arrange + string token = Guid.NewGuid().ToString("N"); + CreateResponseOptions options = CreateBackgroundRequest($"long:{token}"); + var responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + Stopwatch stopwatch = Stopwatch.StartNew(); + + // Act + ResponseResult accepted = (await responses.CreateResponseAsync(options)).Value; + TimeSpan acceptanceTime = stopwatch.Elapsed; + + // Leave the response alone while its deterministic delay runs. + await Task.Delay(TimeSpan.FromSeconds(25)); + ResponseWaitResult waitResult = await WaitForTerminalAsync(responses, accepted.Id, s_completionTimeout); + + // Assert + Assert.True(accepted.Status is ResponseStatus.Queued or ResponseStatus.InProgress); + Assert.True(acceptanceTime < TimeSpan.FromSeconds(10), $"Background acceptance took {acceptanceTime}."); + Assert.Equal(ResponseStatus.Completed, waitResult.Response.Status); + Assert.Contains($"LONG-RUN-COMPLETE:{token}", waitResult.Response.GetOutputText(), StringComparison.Ordinal); + } + + [Fact] + public async Task BackgroundResponse_ProcessCrash_RecoversAndCompletesAsync() + { + // Arrange + string token = Guid.NewGuid().ToString("N"); + CreateResponseOptions options = CreateBackgroundRequest($"crash:{token}"); + var responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + Stopwatch stopwatch = Stopwatch.StartNew(); + + // Act + ResponseResult accepted = (await responses.CreateResponseAsync(options)).Value; + TimeSpan acceptanceTime = stopwatch.Elapsed; + ResponseWaitResult waitResult = await WaitForTerminalAsync(responses, accepted.Id, s_completionTimeout); + + // Assert: this token is emitted only after a new process observes the crash marker written + // immediately before Environment.Exit. + Assert.True(accepted.Status is ResponseStatus.Queued or ResponseStatus.InProgress); + Assert.True( + waitResult.SawSessionNotReady + || waitResult.SawResponseNotFound + || waitResult.LongestPollDuration > acceptanceTime, + "Expected recovery to return transient HTTP 424/404 or take longer than background acceptance. " + + $"Acceptance: {acceptanceTime}; longest poll: {waitResult.LongestPollDuration}."); + Assert.Equal(ResponseStatus.Completed, waitResult.Response.Status); + Assert.Contains( + $"CRASH-RECOVERED:{token}:PROCESS-CHANGED", + waitResult.Response.GetOutputText(), + StringComparison.Ordinal); + } + + [Fact] + public async Task BackgroundCountdown_ProcessCrash_RecoversAndReplaysAllUpdatesAsync() + { + // Arrange + const int Target = 20; + const int CrashAfterCount = 10; + string token = Guid.NewGuid().ToString("N"); + List expected = + [ + .. Enumerable.Range(1, Target) + .Reverse() + .Select(value => value.ToString(System.Globalization.CultureInfo.InvariantCulture)), + "Countdown complete.", + ]; + AIAgent agent = this._fixture.Agent; + AgentSession session = await agent.CreateSessionAsync(); + AgentRunOptions initialOptions = new() { AllowBackgroundResponses = true }; + using CancellationTokenSource timeoutSource = new(s_completionTimeout); + + // Act + StreamCapture before = await CaptureUntilDisconnectAsync( + agent, + session, + $"countdown:{Target}:{CrashAfterCount}:{token}", + initialOptions, + "before", + timeoutSource.Token); + + ResponseContinuationToken continuationToken = before.ContinuationToken + ?? throw new InvalidOperationException( + "The interrupted stream did not provide a continuation token."); + AgentRunOptions recoveryOptions = new() + { + AllowBackgroundResponses = true, + ContinuationToken = continuationToken, + }; + StreamCapture recovered = await CaptureToCompletionWithRetryAsync( + agent, + session, + recoveryOptions, + "recovered", + before.CompletedMessageIds, + timeoutSource.Token); + + List recoveredCountdown = [.. before.Texts, .. recovered.Texts]; + string responseId = before.ResponseId + ?? recovered.ResponseId + ?? throw new InvalidOperationException( + "The countdown stream did not provide a response ID."); + AgentRunOptions replayOptions = new() + { + AllowBackgroundResponses = true, + ContinuationToken = CreateReplayFromStartToken(responseId), + }; + StreamCapture replayed = await CaptureToCompletionWithRetryAsync( + agent, + session, + replayOptions, + "replayed", + existingMessageIds: null, + timeoutSource.Token); + + // Assert + Assert.Equal(expected.Take(CrashAfterCount), before.Texts); + Assert.Equal(expected, recoveredCountdown); + Assert.Equal(Target, CountCountdownUpdates(recoveredCountdown)); + Assert.Equal(expected, replayed.Texts); + Assert.Equal(Target, CountCountdownUpdates(replayed.Texts)); + } + + private static CreateResponseOptions CreateBackgroundRequest(string input) + { + CreateResponseOptions options = new() + { + BackgroundModeEnabled = true, + StoredOutputEnabled = true, + }; + options.InputItems.Add(ResponseItem.CreateUserMessageItem(input)); + return options; + } + + private static async Task CaptureUntilDisconnectAsync( + AIAgent agent, + AgentSession session, + string input, + AgentRunOptions options, + string phase, + CancellationToken cancellationToken) + { + StreamCapture capture = new(); + + try + { + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync( + input, + session, + options, + cancellationToken)) + { + capture.Observe(update, phase); + } + } + catch (ClientResultException exception) + when (IsTransientRecoveryStatus(exception.Status)) + { + } + catch (HttpRequestException) + { + } + + return capture; + } + + private static async Task CaptureToCompletionWithRetryAsync( + AIAgent agent, + AgentSession session, + AgentRunOptions options, + string phase, + IEnumerable? existingMessageIds, + CancellationToken cancellationToken) + { + StreamCapture capture = new(existingMessageIds); + + while (!capture.ResponseCompleted) + { + if (capture.ContinuationToken is not null) + { + options.ContinuationToken = capture.ContinuationToken; + } + + try + { + await foreach (AgentResponseUpdate update in agent.RunStreamingAsync( + session, + options, + cancellationToken)) + { + capture.Observe(update, phase); + } + } + catch (ClientResultException exception) + when (IsTransientRecoveryStatus(exception.Status)) + { + } + catch (HttpRequestException) + { + } + + if (!capture.ResponseCompleted) + { + await Task.Delay(TimeSpan.FromSeconds(2), cancellationToken); + } + } + + return capture; + } + + private static ResponseContinuationToken CreateReplayFromStartToken( + string responseId) + { + ResponseContinuationToken innerToken = + ResponseContinuationToken.FromBytes( + JsonSerializer.SerializeToUtf8Bytes( + new { responseId })); + string serializedInnerToken = JsonSerializer.Serialize( + innerToken, + AgentAbstractionsJsonUtilities.DefaultOptions.GetTypeInfo( + typeof(ResponseContinuationToken))); + byte[] bytes = JsonSerializer.SerializeToUtf8Bytes( + new + { + type = "chatClientAgentContinuationToken", + innerToken = serializedInnerToken, + }); + return ResponseContinuationToken.FromBytes(bytes); + } + + private static bool IsTransientRecoveryStatus(int status) => + status is 404 or 424 or 500 or 502 or 503; + + private static int CountCountdownUpdates(IEnumerable texts) => + texts.Count(text => text != "Countdown complete."); + + private static async Task WaitForTerminalAsync( + ResponsesClient responses, + string responseId, + TimeSpan timeout) + { + bool sawSessionNotReady = false; + bool sawResponseNotFound = false; + TimeSpan longestPollDuration = TimeSpan.Zero; + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + ResponseResult response; + Stopwatch pollStopwatch = Stopwatch.StartNew(); + try + { + response = (await responses.GetResponseAsync(responseId)).Value; + } + catch (ClientResultException ex) when (ex.Status == 424) + { + longestPollDuration = Max(longestPollDuration, pollStopwatch.Elapsed); + sawSessionNotReady = true; + await Task.Delay(TimeSpan.FromSeconds(2)); + continue; + } + catch (ClientResultException ex) when (ex.Status == 404) + { + longestPollDuration = Max(longestPollDuration, pollStopwatch.Elapsed); + sawResponseNotFound = true; + await Task.Delay(TimeSpan.FromSeconds(2)); + continue; + } + + longestPollDuration = Max(longestPollDuration, pollStopwatch.Elapsed); + if (response.Status is ResponseStatus.Completed) + { + return new( + response, + sawSessionNotReady, + sawResponseNotFound, + longestPollDuration); + } + + if (response.Status is ResponseStatus.Cancelled or ResponseStatus.Failed or ResponseStatus.Incomplete) + { + throw new InvalidOperationException( + $"Response '{responseId}' terminated with status '{response.Status}': {response.Error?.Message}"); + } + + await Task.Delay(TimeSpan.FromSeconds(2)); + } + + throw new TimeoutException($"Response '{responseId}' did not complete within {timeout}."); + + static TimeSpan Max(TimeSpan left, TimeSpan right) => left >= right ? left : right; + } + + private sealed record ResponseWaitResult( + ResponseResult Response, + bool SawSessionNotReady, + bool SawResponseNotFound, + TimeSpan LongestPollDuration); + + private sealed class StreamCapture + { + private readonly HashSet _completedMessageIds; + + public StreamCapture( + IEnumerable? existingMessageIds = null) + { + this._completedMessageIds = new( + existingMessageIds ?? [], + StringComparer.Ordinal); + } + + public List Texts { get; } = []; + + public IReadOnlyCollection CompletedMessageIds => + this._completedMessageIds; + + public string? ResponseId { get; private set; } + + public ResponseContinuationToken? ContinuationToken { get; private set; } + + public bool ResponseCompleted { get; private set; } + + public void Observe(AgentResponseUpdate update, string phase) + { + object? rawRepresentation = + update.RawRepresentation is ChatResponseUpdate chatResponseUpdate + ? chatResponseUpdate.RawRepresentation + : update.RawRepresentation; + + if (update.ContinuationToken is { } continuationToken) + { + this.ContinuationToken = continuationToken; + } + + if (!string.IsNullOrWhiteSpace(update.ResponseId)) + { + this.ResponseId = update.ResponseId; + } + + if (rawRepresentation is StreamingResponseOutputItemDoneUpdate + { + Item: MessageResponseItem message + } + && this._completedMessageIds.Add(message.Id)) + { + this.AddMessage(message, phase); + } + + ResponseResult? responseSnapshot = rawRepresentation switch + { + StreamingResponseCreatedUpdate created => created.Response, + StreamingResponseInProgressUpdate inProgress => + inProgress.Response, + StreamingResponseCompletedUpdate completed => + completed.Response, + _ => null, + }; + if (responseSnapshot is not null) + { + foreach (MessageResponseItem snapshotMessage in + responseSnapshot.OutputItems.OfType()) + { + if (this._completedMessageIds.Add(snapshotMessage.Id)) + { + this.AddMessage(snapshotMessage, phase); + } + } + } + + if (rawRepresentation is StreamingResponseCompletedUpdate) + { + this.ResponseCompleted = true; + } + else if (rawRepresentation is StreamingResponseFailedUpdate failed) + { + throw new InvalidOperationException( + $"Response '{failed.Response.Id}' failed: " + + failed.Response.Error?.Message); + } + } + + private void AddMessage( + MessageResponseItem message, + string phase) + { + string text = string.Concat( + message.Content + .Where(content => + content.Kind is ResponseContentPartKind.OutputText) + .Select(content => content.Text)); + if (!string.IsNullOrEmpty(text)) + { + this.Texts.Add(text); + Console.WriteLine($"{phase} > {text}"); + } + } + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/SteerableLongRunningHostedAgentTests.cs b/dotnet/tests/Foundry.Hosting.IntegrationTests/SteerableLongRunningHostedAgentTests.cs new file mode 100644 index 00000000000..0d56ff71439 --- /dev/null +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/SteerableLongRunningHostedAgentTests.cs @@ -0,0 +1,182 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.ClientModel; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.Extensions.OpenAI; +using Foundry.Hosting.IntegrationTests.Fixtures; +using OpenAI.Responses; + +#pragma warning disable OPENAI001 // Experimental Responses API surfaces + +namespace Foundry.Hosting.IntegrationTests; + +/// +/// Live steering tests for an active long-running MAF turn. +/// +[Trait("Category", "FoundryHostedAgents")] +public sealed class SteerableLongRunningHostedAgentTests( + SteerableLongRunningHostedAgentFixture fixture) + : IClassFixture +{ + private static readonly TimeSpan s_completionTimeout = TimeSpan.FromMinutes(6); + private readonly SteerableLongRunningHostedAgentFixture _fixture = fixture; + + [Fact] + public async Task ActiveTurn_QueuesSteeringThenRunsItOnTheSameSessionAsync() + { + // Arrange + string token = Guid.NewGuid().ToString("N"); + string conversationId = await this._fixture.CreateConversationAsync(); + var responses = this._fixture.AgentOpenAIClient.GetProjectResponsesClient(); + + try + { + CreateResponseOptions firstOptions = + CreateBackgroundRequest(conversationId, $"first:{token}"); + string firstResponseId = await StartStreamingAndWaitForOutputAsync( + responses, + firstOptions, + $"FIRST-STARTED:{token}", + s_completionTimeout); + + // Act + CreateResponseOptions steeringOptions = + CreateBackgroundRequest(conversationId, $"steer:{token}"); + ResponseResult steering = (await responses.CreateResponseAsync(steeringOptions)).Value; + + // Assert + Assert.Equal(ResponseStatus.Queued, steering.Status); + + ResponseResult firstCompleted = + await WaitForTerminalAsync( + responses, + firstResponseId, + s_completionTimeout); + ResponseResult steeringCompleted = + await WaitForTerminalAsync(responses, steering.Id, s_completionTimeout); + + Assert.Equal(ResponseStatus.Completed, firstCompleted.Status); + Assert.Equal(ResponseStatus.Completed, steeringCompleted.Status); + Assert.Contains( + $"STEERED-COMPLETE:{token}:SESSION-TURN-2:MAX-CONCURRENCY-1", + steeringCompleted.GetOutputText(), + StringComparison.Ordinal); + } + finally + { + await this._fixture.DeleteConversationAsync(conversationId); + } + } + + private static CreateResponseOptions CreateBackgroundRequest( + string conversationId, + string input) + { + CreateResponseOptions options = new() + { + AgentConversationId = conversationId, + BackgroundModeEnabled = true, + StoredOutputEnabled = true, + }; + options.InputItems.Add(ResponseItem.CreateUserMessageItem(input)); + return options; + } + + private static async Task StartStreamingAndWaitForOutputAsync( + ResponsesClient responses, + CreateResponseOptions options, + string expected, + TimeSpan timeout) + { + using CancellationTokenSource timeoutSource = new(timeout); + string? responseId = null; + StringBuilder text = new(); + + await foreach (StreamingResponseUpdate update in responses + .CreateResponseStreamingAsync(options, timeoutSource.Token) + .WithCancellation(timeoutSource.Token)) + { + switch (update) + { + case StreamingResponseCreatedUpdate created: + responseId = created.Response.Id; + break; + + case StreamingResponseOutputTextDeltaUpdate delta: + text.Append(delta.Delta); + if (text.ToString().Contains(expected, StringComparison.Ordinal)) + { + return responseId + ?? throw new InvalidOperationException( + "The stream emitted text before response.created."); + } + break; + + case StreamingResponseFailedUpdate failed: + throw new InvalidOperationException( + $"Response '{failed.Response.Id}' failed: " + + failed.Response.Error?.Message); + } + } + + throw new InvalidOperationException( + $"The response stream ended before emitting '{expected}'."); + } + + private static async Task WaitForTerminalAsync( + ResponsesClient responses, + string responseId, + TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + while (DateTimeOffset.UtcNow < deadline) + { + ResponseResult? response = await TryGetResponseAsync(responses, responseId); + if (response?.Status is ResponseStatus.Completed) + { + return response; + } + + if (response is not null) + { + ThrowIfTerminalFailure(responseId, response); + } + + await Task.Delay(TimeSpan.FromSeconds(2)); + } + + throw new TimeoutException( + $"Response '{responseId}' did not complete within {timeout}."); + } + + private static async Task TryGetResponseAsync( + ResponsesClient responses, + string responseId) + { + try + { + return (await responses.GetResponseAsync(responseId)).Value; + } + catch (ClientResultException ex) when (ex.Status is 404 or 424) + { + return null; + } + } + + private static void ThrowIfTerminalFailure( + string responseId, + ResponseResult response) + { + if (response.Status is ResponseStatus.Cancelled + or ResponseStatus.Failed + or ResponseStatus.Incomplete) + { + throw new InvalidOperationException( + $"Response '{responseId}' terminated with status '{response.Status}': " + + response.Error?.Message); + } + } +} diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 index 33f4345af20..9aa0472bd57 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-bootstrap-agents.ps1 @@ -53,6 +53,8 @@ $Scenarios = @( 'session-files', 'agent-skills', 'user-identity', + 'resilient-workflow', + 'steerable-long-running', 'unsupported-protocol' ) diff --git a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1 b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1 index 2d938bb013b..944a043af4f 100644 --- a/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1 +++ b/dotnet/tests/Foundry.Hosting.IntegrationTests/scripts/it-build-image.ps1 @@ -83,13 +83,24 @@ $hashedDirs = @( $sourceFiles = @() foreach ($dir in $hashedDirs) { if (Test-Path $dir) { - $sourceFiles += @(git -c core.quotepath=false ls-files -- $dir) + $sourceFiles += @(git -c core.quotepath=false ls-files --cached --others --exclude-standard -- $dir) } } if ($sourceFiles.Count -eq 0) { - throw "No tracked files found under any of: $($hashedDirs -join ', ')" + throw "No source files found under any of: $($hashedDirs -join ', ')" } -$fileHashes = git hash-object -- $sourceFiles + +# Keep each git invocation below the Windows command-line length limit. +$fileHashes = @() +$maxHashBatchSize = 100 +for ($offset = 0; $offset -lt $sourceFiles.Count; $offset += $maxHashBatchSize) { + $end = [Math]::Min($offset + $maxHashBatchSize - 1, $sourceFiles.Count - 1) + $fileHashes += @(git hash-object -- $sourceFiles[$offset..$end]) + if ($LASTEXITCODE -ne 0) { + throw "git hash-object failed with exit code $LASTEXITCODE." + } +} + $shaInput = ($fileHashes -join "`n" | git hash-object --stdin).Trim() $tag = $shaInput.Substring(0, 12) $image = "$Registry/$Repository`:$tag" diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs new file mode 100644 index 00000000000..6b2567244ef --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerResilienceTests.cs @@ -0,0 +1,589 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Azure.AI.AgentServer.Responses; +using Azure.AI.AgentServer.Responses.Models; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Moq; +using MeaiTextContent = Microsoft.Extensions.AI.TextContent; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +/// +/// Deterministic tests for the resilient (crash-recovery) behavior of +/// . They drive the handler with a fake agent that +/// records the messages it receives and a fake session store, so recovery semantics can be asserted +/// without a real model, a real process crash, or timing. +/// +public class AgentFrameworkResponseHandlerResilienceTests +{ + private const string ResponseId = "resp_0000000000000000000000000000000000000000000000"; + + [Fact] + public async Task CreateAsync_Recovery_WithoutPersistedSession_ReinjectsInputAsync() + { + // Arrange: recovery ran before the first AgentSession snapshot was persisted. + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("original input"); + var context = CreateContext(isRecovery: true); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: no session exists to resume, so recovery must restart from the original input. + Assert.NotNull(recording.LastMessages); + Assert.Contains( + recording.LastMessages!, + message => message.Text.Contains("original input", StringComparison.Ordinal)); + } + + [Fact] + public async Task CreateAsync_Recovery_WithPersistedSession_DoesNotReinjectInputAsync() + { + // Arrange: a prior lifetime persisted an AgentSession for this response. + var recording = new RecordingAgent(); + var store = new AlwaysLoadedSessionStore(); + var handler = CreateHandler(recording, store, resilient: true); + var request = NewBackgroundStoreRequest("original input"); + var context = CreateContext(isRecovery: true); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: the restored session owns re-entry, so the original input is not duplicated. + Assert.NotNull(recording.LastMessages); + Assert.Empty(recording.LastMessages!); + } + + [Fact] + public async Task CreateAsync_FreshTurn_InjectsInputAsync() + { + // Arrange: the same request on a fresh (non-recovery) turn. + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("original input"); + var context = CreateContext(isRecovery: false); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: a fresh turn feeds the request input to the agent. + Assert.NotNull(recording.LastMessages); + Assert.Contains(recording.LastMessages!, m => m.Text.Contains("original input", StringComparison.Ordinal)); + } + + [Fact] + public async Task CreateAsync_ResilientTurn_MidStreamSaveFailure_StillCompletesAsync() + { + // Arrange: a store whose first save throws, mimicking the serialize race that can happen + // when the incremental (mid-stream) save runs while the workflow is still advancing. The + // later end-of-turn save succeeds. + var store = new ThrowOnceSessionStore(); + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, store, resilient: true); + var request = NewBackgroundStoreRequest("hello"); + var context = CreateContext(isRecovery: false); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: the failed incremental save was swallowed and the turn still reached a completed + // terminal event (it did not escape as a handler failure that leaves the response stuck). + Assert.True(store.SaveAttempts >= 1, "Expected at least one session save attempt."); + Assert.Contains(events, e => e is ResponseCompletedEvent); + Assert.DoesNotContain(events, e => e is ResponseFailedEvent); + } + + [Fact] + public async Task CreateAsync_Recovery_UsesAvailablePersistedResponseAsStreamSeedAsync() + { + // Arrange: AgentServer supplied a durable snapshot that happens to contain two output items. + // This regular test agent has no workflow checkpoint metadata; the test verifies how the + // handler consumes the available response snapshot on recovery. + var persisted = new ResponseObject("resp_" + new string('0', 46), "test"); + persisted.Output.Add(NewMessageItem("prior_1", "prior item one")); + persisted.Output.Add(NewMessageItem("prior_2", "prior item two")); + + var recording = new RecordingAgent(); + var handler = CreateHandler(recording, new InMemoryAgentSessionStore(), resilient: true); + var request = NewBackgroundStoreRequest("input"); + var context = CreateContext(isRecovery: true, persistedResponse: persisted); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: new items start after the output watermark carried by the available snapshot. The + // handler does not treat that watermark as the workflow checkpoint or re-emit seeded items. + var addedIndexes = events.OfType().Select(e => e.OutputIndex).ToList(); + Assert.NotEmpty(addedIndexes); + Assert.All(addedIndexes, i => Assert.True(i >= 2, $"New output item index {i} collided with a seeded item (0 or 1).")); + + // The final response retains the two items supplied by AgentServer and appends the newly + // emitted item. This does not assert that normal workflow recovery produces such a snapshot. + var completed = events.OfType().Single(); + Assert.Equal(3, completed.Response.Output.Count); + } + + [Fact] + public async Task CreateAsync_NewWorkflowCheckpoint_DefaultStore_PersistsOneResponseCheckpointPerIdAsync() + { + // Arrange: the agent reports one workflow checkpoint twice, followed by a new checkpoint. + var store = new CountingSessionStore(); + var handler = CreateHandler( + new CheckpointUpdateAgent( + await CreateWorkflowSessionAsync(), + "checkpoint-1", + "checkpoint-1", + "checkpoint-2"), + store, + resilient: true); + var request = NewBackgroundStoreRequest("start"); + request.Store = null; + var context = CreateContext(isRecovery: false); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: each distinct workflow checkpoint advances the durable response snapshot once. + Assert.Equal(2, events.Count(e => e.GetType().Name == "ResponseCheckpointEvent")); + Assert.Equal(3, store.SaveAttempts); + + var completed = events.OfType().Single(); + Assert.NotNull(completed.Response.Metadata); + string metadataJson = completed.Response.Metadata.AdditionalProperties["_internal_metadata"]; + using JsonDocument metadata = JsonDocument.Parse(metadataJson); + Assert.Equal( + "checkpoint-2", + metadata.RootElement.GetProperty("_last_checkpoint_id").GetString()); + } + + [Fact] + public async Task CreateAsync_WorkflowCheckpoint_WhenSessionSaveFails_KeepsPriorResponseCheckpointAsync() + { + // Arrange: the workflow creates a checkpoint, but its matching AgentSession cannot be saved. + var store = new ThrowOnceSessionStore(); + var handler = CreateHandler( + new CheckpointUpdateAgent( + await CreateWorkflowSessionAsync(), + "checkpoint-1"), + store, + resilient: true); + var request = NewBackgroundStoreRequest("start"); + var context = CreateContext(isRecovery: false); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: the final save succeeds, but the response snapshot never claims the unsaved boundary. + Assert.DoesNotContain(events, e => e.GetType().Name == "ResponseCheckpointEvent"); + Assert.Equal(2, store.SaveAttempts); + + var completed = events.OfType().Single(); + Assert.True( + completed.Response.Metadata?.AdditionalProperties.ContainsKey("_internal_metadata") is not true); + } + + [Fact] + public async Task CreateAsync_ShutdownAfterAgentAdvanced_DoesNotSaveUnemittedSessionStateAsync() + { + // Arrange: the agent advances its session and returns an update after shutdown is visible. + var store = new CountingSessionStore(); + var handler = CreateHandler( + new SessionAdvancingAgent(), + store, + resilient: true); + var request = NewBackgroundStoreRequest("input"); + var context = CreateContext(isRecovery: false, shutdownRequested: true); + + // Act + var events = await CollectEventsAsync(handler, request, context); + + // Assert: only the lifecycle prefix was emitted, so advanced session state must not be saved. + Assert.DoesNotContain(events, responseEvent => responseEvent is ResponseOutputItemDoneEvent); + Assert.Equal(0, store.SaveAttempts); + } + + [Fact] + public void Constructor_ExistingThreeParameterSignature_IsPreserved() + { + // Act + var constructor = typeof(AgentFrameworkResponseHandler).GetConstructor( + [ + typeof(IServiceProvider), + typeof(ILogger), + typeof(FoundryToolboxService), + ]); + + // Assert + Assert.NotNull(constructor); + } + + [Fact] + public void Constructor_OptionsSignature_IsPreferredForActivatorUtilities() + { + // Act + var constructor = typeof(AgentFrameworkResponseHandler).GetConstructor( + [ + typeof(IServiceProvider), + typeof(ILogger), + typeof(IOptions), + typeof(FoundryToolboxService), + ]); + + // Assert + Assert.NotNull(constructor); + Assert.NotNull( + constructor.GetCustomAttribute()); + } + + [Fact] + public async Task AddFoundryResponses_ResilientHandler_UsesConfiguredOptionsAsync() + { + // Arrange + var agent = new RecordingAgent(); + var store = new CountingSessionStore(); + var services = new ServiceCollection(); + services.AddFoundryResponses( + agent, + store, + options => options.ResilientBackground = true); + services.AddLogging(); + services.AddSingleton( + new FakeHostedSessionIsolationKeyProvider()); + using ServiceProvider provider = services.BuildServiceProvider(); + var handler = Assert.IsType( + provider.GetRequiredService()); + CreateResponse request = NewBackgroundStoreRequest("input"); + ResponseContext context = CreateContext(isRecovery: false); + + // Act + await CollectEventsAsync(handler, request, context); + + // Assert: one incremental save plus the final save proves the handler read the configured + // resilience option rather than the compatibility constructor's default options. + Assert.True(store.SaveAttempts >= 2); + } + + private static AgentFrameworkResponseHandler CreateHandler(AIAgent agent, AgentSessionStore store, bool resilient) + { + var services = new ServiceCollection(); + services.AddSingleton(store); + services.AddSingleton(agent); + services.AddSingleton(new FakeHostedSessionIsolationKeyProvider()); + var sp = services.BuildServiceProvider(); + + var options = Options.Create(new FoundryResponsesOptions { ResilientBackground = resilient }); + return new AgentFrameworkResponseHandler(sp, NullLogger.Instance, toolboxService: null, foundryResponsesOptions: options); + } + + private static CreateResponse NewBackgroundStoreRequest(string text) + { + var request = new CreateResponse { Model = "test", Background = true, Store = true }; + request.Input = BinaryData.FromObjectAsJson(new[] + { + new + { + type = "message", + id = "msg_in_1", + status = "completed", + role = "user", + content = new[] { new { type = "input_text", text } } + } + }); + return request; + } + + private static ResponseContext CreateContext( + bool isRecovery, + ResponseObject? persistedResponse = null, + bool shutdownRequested = false) + { + var mock = new Mock(ResponseId) { CallBase = true }; + mock.Setup(x => x.IsRecovery).Returns(isRecovery); + mock.Setup(x => x.PersistedResponse).Returns(persistedResponse); + mock.Setup(x => x.ExitForRecoveryAsync(It.IsAny())) + .Returns(Task.CompletedTask); + mock.Setup(x => x.GetHistoryAsync(It.IsAny())) + .ReturnsAsync(Array.Empty()); + mock.Setup(x => x.GetInputItemsAsync(It.IsAny(), It.IsAny())) + .ReturnsAsync(Array.Empty()); + if (shutdownRequested) + { + mock.Object.IsShutdownRequested = true; + } + + return mock.Object; + } + + private static OutputItemMessage NewMessageItem(string id, string text) => + new( + id: id, + role: MessageRole.Assistant, + content: [new MessageContentOutputTextContent(text, Array.Empty(), Array.Empty())], + status: MessageStatus.Completed); + + private static async Task> CollectEventsAsync( + AgentFrameworkResponseHandler handler, + CreateResponse request, + ResponseContext context) + { + var events = new List(); + await foreach (var evt in handler.CreateAsync(request, context, CancellationToken.None)) + { + events.Add(evt); + } + + return events; + } + + private static async Task CreateWorkflowSessionAsync() + { + AIAgent workflowAgent = AgentWorkflowBuilder + .BuildSequential( + "checkpoint-session-workflow", + new RecordingAgent()) + .AsAIAgent( + id: "checkpoint-session-agent", + name: "Checkpoint Session Agent"); + return await workflowAgent.CreateSessionAsync(); + } + + /// + /// A fake agent that records the messages passed to each run so a test can assert exactly what + /// the handler fed it (for example, that recovery injected nothing). + /// + private sealed class RecordingAgent : AIAgent + { + public IReadOnlyList? LastMessages { get; private set; } + + protected override string? IdCore => "recording-agent"; + + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + this.LastMessages = messages.ToList(); + yield return new AgentResponseUpdate + { + MessageId = "msg_rec_1", + Contents = [new MeaiTextContent("recorded")] + }; + await Task.CompletedTask; + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotImplementedException(); + + protected override ValueTask CreateSessionCoreAsync(CancellationToken cancellationToken = default) => + new(new RecordingSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(JsonSerializer.SerializeToElement(new { }, jsonSerializerOptions)); + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(new RecordingSession()); + + private sealed class RecordingSession : AgentSession + { + public RecordingSession() + { + } + } + } + + /// + /// A fake session store whose first throws (mimicking the + /// serialize race), then succeeds, while loads always create a fresh session. + /// + private sealed class ThrowOnceSessionStore : AgentSessionStore + { + private int _saveAttempts; + + public int SaveAttempts => this._saveAttempts; + + public override ValueTask SaveSessionAsync(AIAgent agent, string conversationId, AgentSession session, string? userId, CancellationToken cancellationToken = default) + { + var attempt = Interlocked.Increment(ref this._saveAttempts); + if (attempt == 1) + { + throw new InvalidOperationException("Collection was modified; enumeration operation may not execute."); + } + + return default; + } + + public override async ValueTask GetSessionAsync(AIAgent agent, string conversationId, string? userId, CancellationToken cancellationToken = default) => + await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + } + + private sealed class CountingSessionStore : AgentSessionStore + { + private int _saveAttempts; + + public int SaveAttempts => this._saveAttempts; + + public override ValueTask SaveSessionAsync( + AIAgent agent, + string conversationId, + AgentSession session, + string? userId, + CancellationToken cancellationToken = default) + { + Interlocked.Increment(ref this._saveAttempts); + return default; + } + + public override ValueTask GetSessionAsync( + AIAgent agent, + string conversationId, + string? userId, + CancellationToken cancellationToken = default) => + new((AgentSession?)null); + } + + private sealed class AlwaysLoadedSessionStore : AgentSessionStore + { + public override ValueTask SaveSessionAsync( + AIAgent agent, + string conversationId, + AgentSession session, + string? userId, + CancellationToken cancellationToken = default) => + default; + + public override async ValueTask GetSessionAsync( + AIAgent agent, + string conversationId, + string? userId, + CancellationToken cancellationToken = default) => + await agent.CreateSessionAsync(cancellationToken).ConfigureAwait(false); + } + + private sealed class SessionAdvancingAgent : AIAgent + { + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var advancingSession = Assert.IsType(session); + advancingSession.Phase = 1; + yield return new AgentResponseUpdate + { + MessageId = "msg_shutdown_1", + Contents = [new MeaiTextContent("not emitted")] + }; + await Task.CompletedTask; + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(new AdvancingSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + var advancingSession = Assert.IsType(session); + return new(JsonSerializer.SerializeToElement( + new { advancingSession.Phase }, + jsonSerializerOptions)); + } + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(new AdvancingSession + { + Phase = serializedState.GetProperty("Phase").GetInt32(), + }); + + private sealed class AdvancingSession : AgentSession + { + public int Phase { get; set; } + } + } + + private sealed class CheckpointUpdateAgent( + AgentSession workflowSession, + params string[] checkpointIds) : AIAgent + { + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var step = 0; + foreach (string checkpointId in checkpointIds) + { + var checkpoint = new CheckpointInfo("workflow-session", checkpointId); + var completion = new SuperStepCompletionInfo([]) { Checkpoint = checkpoint }; + yield return new AgentResponseUpdate + { + RawRepresentation = new SuperStepCompletedEvent(step++, completion), + }; + } + + await Task.CompletedTask; + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(workflowSession); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(JsonSerializer.SerializeToElement(new { }, jsonSerializerOptions)); + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) => + new(workflowSession); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerWorkflowTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerWorkflowTests.cs index 4c58daa39f4..f5194f5f6d1 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerWorkflowTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/AgentFrameworkResponseHandlerWorkflowTests.cs @@ -11,6 +11,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; using Moq; namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; @@ -161,7 +162,10 @@ public async Task WorkflowAgent_RegisteredWithKey_ResolvesCorrectlyAsync() } private static (AgentFrameworkResponseHandler handler, CreateResponse request, ResponseContext context) - CreateHandlerWithAgent(AIAgent agent, string userMessage) + CreateHandlerWithAgent( + AIAgent agent, + string userMessage, + bool resilient = false) { var services = new ServiceCollection(); services.AddSingleton(new InMemoryAgentSessionStore()); @@ -170,8 +174,20 @@ private static (AgentFrameworkResponseHandler handler, CreateResponse request, R services.AddSingleton(new FakeHostedSessionIsolationKeyProvider()); var sp = services.BuildServiceProvider(); - var handler = new AgentFrameworkResponseHandler(sp, NullLogger.Instance); - var request = new CreateResponse { Model = "test" }; + var handler = new AgentFrameworkResponseHandler( + sp, + NullLogger.Instance, + Options.Create( + new FoundryResponsesOptions + { + ResilientBackground = resilient, + })); + var request = new CreateResponse + { + Model = "test", + Background = resilient, + Store = resilient, + }; request.Input = CreateUserInput(userMessage); var mockContext = CreateMockContext(); diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryJsonCheckpointStoreTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryJsonCheckpointStoreTests.cs index c87e2b04ed0..8ada8a76dcb 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryJsonCheckpointStoreTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/FoundryJsonCheckpointStoreTests.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Azure.AI.AgentServer.Core.Storage; +using Azure.Core; using Microsoft.Agents.AI.Foundry.Hosting; using Microsoft.Agents.AI.Workflows; using Microsoft.Extensions.Logging; @@ -26,6 +27,59 @@ public void Constructor_WithoutCredential_IsAllowedForTheSdkLocalFallback() Assert.Equal(FoundryJsonCheckpointStore.DefaultStoreName, store.StoreName); } + [Fact] + public void DefaultMaxIndexUpdateAttempts_IsEight() + { + // Assert + Assert.Equal(8, FoundryJsonCheckpointStore.DefaultMaxIndexUpdateAttempts); + } + + [Fact] + public void Constructor_MaxIndexUpdateAttemptsLessThanOne_Throws() + { + // Act + var exception = Assert.Throws( + () => new FoundryJsonCheckpointStore(maxIndexUpdateAttempts: 0)); + + // Assert + Assert.Equal("maxIndexUpdateAttempts", exception.ParamName); + } + + [Fact] + public void Constructor_ExistingFiveParameterSignature_IsPreserved() + { + // Act + var constructor = typeof(FoundryJsonCheckpointStore).GetConstructor( + [ + typeof(Uri), + typeof(TokenCredential), + typeof(string), + typeof(int), + typeof(ILoggerFactory), + ]); + + // Assert + Assert.NotNull(constructor); + } + + [Fact] + public async Task CreateCheckpointAsync_CustomMaxIndexUpdateAttempts_LimitsRetriesAsync() + { + // Arrange + var backing = new FakeCheckpointStateStore(); + var store = NewStore(backing, maxIndexUpdateAttempts: 2); + await store.CreateCheckpointAsync("session-1", Json("{\"step\":1}")); + backing.FailNextIndexWrites = 2; + + // Act + var exception = await Assert.ThrowsAsync( + async () => await store.CreateCheckpointAsync("session-1", Json("{\"step\":2}"))); + + // Assert + Assert.Contains("after 2 attempts", exception.Message, StringComparison.Ordinal); + Assert.Equal(0, backing.FailNextIndexWrites); + } + [Fact] public async Task CreateCheckpointAsync_ThenRetrieveCheckpointAsync_RoundTripsAsync() { @@ -381,8 +435,14 @@ public void BuildCheckpointKey_DifferentSessionsNeverShareAKey() Assert.NotEqual(first, second); } - private static FoundryJsonCheckpointStore NewStore(FoundryStateStore backing, ILoggerFactory? loggerFactory = null) - => new(_ => Task.FromResult(backing), loggerFactory: loggerFactory); + private static FoundryJsonCheckpointStore NewStore( + FoundryStateStore backing, + ILoggerFactory? loggerFactory = null, + int maxIndexUpdateAttempts = FoundryJsonCheckpointStore.DefaultMaxIndexUpdateAttempts) + => new( + _ => Task.FromResult(backing), + loggerFactory: loggerFactory, + maxIndexUpdateAttempts: maxIndexUpdateAttempts); private static JsonElement Json(string json) { diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientTwoLifetimeIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientTwoLifetimeIntegrationTests.cs new file mode 100644 index 00000000000..8d72a2e4c34 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ResilientTwoLifetimeIntegrationTests.cs @@ -0,0 +1,649 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Agents.AI.Workflows; +using Microsoft.Agents.AI.Workflows.Checkpointing; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +[Collection(FoundryStateStoreLocalFallbackCollectionDefinition.CollectionName)] +public sealed class ResilientTwoLifetimeIntegrationTests +{ + [Fact] + public async Task StoppedHost_RecoversMafAgentFromPersistedSessionAsync() + { + // Arrange + string stateRoot = Path.Combine( + Path.GetTempPath(), + $"maf-recovery-{Guid.NewGuid():N}"); + string? previousStateRoot = + Environment.GetEnvironmentVariable("AGENTSERVER_STATE_ROOT"); + string? previousHostingEnvironment = + Environment.GetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT"); + var coordinator = new RecoveryCoordinator(); + + try + { + Environment.SetEnvironmentVariable("AGENTSERVER_STATE_ROOT", stateRoot); + Environment.SetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT", null); + + string conversationId = $"conv_{Guid.NewGuid():N}"; + string responseId; + + WebApplication firstHost = await StartServerAsync( + new ResumableAgent(coordinator), + new PhaseObservingSessionStore( + new FoundryAgentSessionStore(), + coordinator)); + try + { + using HttpClient firstClient = GetClient(firstHost); + responseId = await StartBackgroundResponseAsync( + firstClient, + conversationId); + try + { + await coordinator.PhasePersisted.Task.WaitAsync( + TimeSpan.FromSeconds(15)); + } + catch (TimeoutException ex) + { + throw new TimeoutException( + "Phase 1 was not observed in the persisted session. States: " + + string.Join(Environment.NewLine, coordinator.SerializedStates), + ex); + } + + using CancellationTokenSource stopTimeout = + new(TimeSpan.FromSeconds(15)); + await firstHost.StopAsync(stopTimeout.Token); + } + finally + { + await firstHost.DisposeAsync(); + } + + // Act + await using WebApplication secondHost = await StartServerAsync( + new ResumableAgent(coordinator), + new FoundryAgentSessionStore()); + using HttpClient secondClient = GetClient(secondHost); + JsonElement completed = await WaitForTerminalAsync( + secondClient, + responseId, + TimeSpan.FromSeconds(20)); + + // Assert + Assert.Equal("completed", completed.GetProperty("status").GetString()); + Assert.Contains( + "RECOVERED-COMPLETE", + GetOutputText(completed), + StringComparison.Ordinal); + Assert.Equal(1, coordinator.FreshRuns); + Assert.Equal(1, coordinator.RecoveryRuns); + Assert.Empty(coordinator.RecoveryMessages); + } + finally + { + Environment.SetEnvironmentVariable( + "AGENTSERVER_STATE_ROOT", + previousStateRoot); + Environment.SetEnvironmentVariable( + "FOUNDRY_HOSTING_ENVIRONMENT", + previousHostingEnvironment); + + if (Directory.Exists(stateRoot)) + { + try + { + Directory.Delete(stateRoot, recursive: true); + } + catch (IOException) + { + } + } + } + } + + [Fact] + public async Task StoppedHost_RecoversWorkflowWithCompleteOrderedOutputAsync() + { + // Arrange + string stateRoot = Path.Combine( + Path.GetTempPath(), + $"maf-workflow-recovery-{Guid.NewGuid():N}"); + string checkpointRoot = Path.Combine(stateRoot, "workflow-checkpoints"); + string? previousStateRoot = + Environment.GetEnvironmentVariable("AGENTSERVER_STATE_ROOT"); + string? previousHostingEnvironment = + Environment.GetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT"); + var coordinator = new CountdownRecoveryCoordinator(target: 6, blockAt: 3); + string sessionStoreName = $"agent-framework/sessions-{Guid.NewGuid():N}"; + + try + { + Environment.SetEnvironmentVariable("AGENTSERVER_STATE_ROOT", stateRoot); + Environment.SetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT", null); + + string conversationId = $"conv_{Guid.NewGuid():N}"; + string responseId; + + using (var checkpointStore = new FileSystemJsonCheckpointStore( + Directory.CreateDirectory(checkpointRoot))) + { + WebApplication firstHost = await StartServerAsync( + BuildCountdownWorkflowAgent(coordinator, checkpointStore), + new FoundryAgentSessionStore(storeName: sessionStoreName)); + try + { + using HttpClient firstClient = GetClient(firstHost); + responseId = await StartBackgroundResponseAsync( + firstClient, + conversationId, + agentName: "countdown-workflow", + input: "Count down from 6"); + await coordinator.Blocked.Task.WaitAsync(TimeSpan.FromSeconds(15)); + await WaitForResponseProgressAsync( + firstClient, + responseId, + ["6", "5", "4"], + minimumOutputItems: 12, + timeout: TimeSpan.FromSeconds(15)); + + using CancellationTokenSource stopTimeout = + new(TimeSpan.FromSeconds(15)); + await firstHost.StopAsync(stopTimeout.Token); + } + finally + { + await firstHost.DisposeAsync(); + } + } + + JsonElement persisted = ReadPersistedResponse(stateRoot, responseId); + Assert.Equal(["6", "5", "4"], GetOutputTexts(persisted)); + Assert.True( + persisted.TryGetProperty("metadata", out JsonElement metadata) + && metadata.TryGetProperty("_internal_metadata", out _), + persisted.GetRawText()); + + // Act + using var recoveryCheckpointStore = new FileSystemJsonCheckpointStore( + Directory.CreateDirectory(checkpointRoot)); + await using WebApplication secondHost = await StartServerAsync( + BuildCountdownWorkflowAgent(coordinator, recoveryCheckpointStore), + new FoundryAgentSessionStore(storeName: sessionStoreName)); + using HttpClient secondClient = GetClient(secondHost); + JsonElement completed = await WaitForTerminalAsync( + secondClient, + responseId, + TimeSpan.FromSeconds(20)); + + // Assert + Assert.Equal("completed", completed.GetProperty("status").GetString()); + Assert.Equal( + ["6", "5", "4", "3", "2", "1", "Countdown complete."], + GetOutputTexts(completed)); + } + finally + { + Environment.SetEnvironmentVariable( + "AGENTSERVER_STATE_ROOT", + previousStateRoot); + Environment.SetEnvironmentVariable( + "FOUNDRY_HOSTING_ENVIRONMENT", + previousHostingEnvironment); + + if (Directory.Exists(stateRoot)) + { + try + { + Directory.Delete(stateRoot, recursive: true); + } + catch (IOException) + { + } + } + } + } + + private static async Task StartServerAsync( + AIAgent agent, + AgentSessionStore sessionStore) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Services.AddFoundryResponses( + agent, + sessionStore, + options => options.ResilientBackground = true); + builder.Services.AddSingleton( + new FakeHostedSessionIsolationKeyProvider()); + builder.Services.AddLogging(); + + WebApplication app = builder.Build(); + app.MapFoundryResponses(); + await app.StartAsync(); + return app; + } + + private static HttpClient GetClient(WebApplication app) => + (app.Services.GetRequiredService() as TestServer + ?? throw new InvalidOperationException("TestServer not found.")) + .CreateClient(); + + private static async Task StartBackgroundResponseAsync( + HttpClient client, + string conversationId, + string agentName = "resumable-agent", + string input = "start durable work") + { + string body = JsonSerializer.Serialize(new + { + model = agentName, + input, + store = true, + background = true, + conversation = conversationId, + }); + using HttpResponseMessage response = await client.PostAsync( + new Uri("/responses", UriKind.Relative), + new StringContent(body, Encoding.UTF8, "application/json")); + response.EnsureSuccessStatusCode(); + + using JsonDocument document = JsonDocument.Parse( + await response.Content.ReadAsStringAsync()); + return document.RootElement.GetProperty("id").GetString() + ?? throw new InvalidOperationException( + "The background response did not contain an id."); + } + + private static async Task WaitForTerminalAsync( + HttpClient client, + string responseId, + TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + string last = "(none)"; + while (DateTimeOffset.UtcNow < deadline) + { + using HttpResponseMessage response = await client.GetAsync( + new Uri($"/responses/{responseId}", UriKind.Relative)); + string body = await response.Content.ReadAsStringAsync(); + last = $"{(int)response.StatusCode} {body}"; + if (response.StatusCode == HttpStatusCode.OK) + { + using JsonDocument document = JsonDocument.Parse(body); + JsonElement root = document.RootElement; + string? status = root.GetProperty("status").GetString(); + if (status == "completed") + { + return root.Clone(); + } + + if (status is "failed" or "cancelled" or "incomplete") + { + throw new InvalidOperationException( + $"Response '{responseId}' terminated with status '{status}': {body}"); + } + } + + await Task.Delay(TimeSpan.FromMilliseconds(50)); + } + + throw new TimeoutException( + $"Response '{responseId}' did not complete. Last response: {last}"); + } + + private static async Task WaitForResponseProgressAsync( + HttpClient client, + string responseId, + IReadOnlyList expected, + int minimumOutputItems, + TimeSpan timeout) + { + var deadline = DateTimeOffset.UtcNow + timeout; + List last = []; + while (DateTimeOffset.UtcNow < deadline) + { + using HttpResponseMessage response = await client.GetAsync( + new Uri($"/responses/{responseId}", UriKind.Relative)); + if (response.StatusCode == HttpStatusCode.OK) + { + using JsonDocument document = JsonDocument.Parse( + await response.Content.ReadAsStringAsync()); + JsonElement root = document.RootElement; + last = GetOutputTexts(root); + if (last.Count == expected.Count + && last.SequenceEqual(expected) + && root.GetProperty("output").GetArrayLength() >= minimumOutputItems) + { + return; + } + } + + await Task.Delay(TimeSpan.FromMilliseconds(25)); + } + + throw new TimeoutException( + $"Response '{responseId}' did not reach the expected checkpointed output. " + + $"Expected: {string.Join(", ", expected)}. Last: {string.Join(", ", last)}."); + } + + private static JsonElement ReadPersistedResponse( + string stateRoot, + string responseId) + { + string path = Path.Combine( + stateRoot, + "responses", + "envelopes", + $"{responseId}.json"); + using JsonDocument document = JsonDocument.Parse(File.ReadAllBytes(path)); + return document.RootElement.GetProperty("envelope").Clone(); + } + + private static string GetOutputText(JsonElement response) + { + StringBuilder text = new(); + foreach (JsonElement item in response.GetProperty("output").EnumerateArray()) + { + if (!item.TryGetProperty("content", out JsonElement content)) + { + continue; + } + + foreach (JsonElement part in content.EnumerateArray()) + { + if (part.TryGetProperty("text", out JsonElement value)) + { + text.Append(value.GetString()); + } + } + } + + return text.ToString(); + } + + private static List GetOutputTexts(JsonElement response) + { + List texts = []; + foreach (JsonElement item in response.GetProperty("output").EnumerateArray()) + { + if (!item.TryGetProperty("content", out JsonElement content)) + { + continue; + } + + foreach (JsonElement part in content.EnumerateArray()) + { + if (part.TryGetProperty("text", out JsonElement value) + && value.GetString() is { } text) + { + texts.Add(text); + } + } + } + + return texts; + } + + private static AIAgent BuildCountdownWorkflowAgent( + CountdownRecoveryCoordinator coordinator, + FileSystemJsonCheckpointStore checkpointStore) + { + var start = new CountdownStartExecutor(coordinator.Target); + var countdown = new CountdownExecutor(coordinator); + var complete = new CountdownCompleteExecutor(); + Workflow workflow = new WorkflowBuilder(start) + .AddEdge(start, countdown) + .AddEdge(countdown, countdown) + .AddEdge(countdown, complete) + .WithOutputFrom(countdown, complete) + .Build(); + + return workflow.AsAIAgent( + id: "countdown-workflow", + name: "countdown-workflow", + executionEnvironment: InProcessExecution.OffThread.WithCheckpointing( + CheckpointManager.CreateJson(checkpointStore)), + includeExceptionDetails: true, + includeWorkflowOutputsInResponse: true); + } + + [SendsMessage(typeof(int))] + private sealed class CountdownStartExecutor(int target) : ChatProtocolExecutor( + "start", + new ChatProtocolExecutorOptions { AutoSendTurnToken = false }) + { + protected override ProtocolBuilder ConfigureProtocol(ProtocolBuilder protocolBuilder) => + base.ConfigureProtocol(protocolBuilder).SendsMessage(); + + protected override ValueTask TakeTurnAsync( + List messages, + IWorkflowContext context, + bool? emitEvents, + CancellationToken cancellationToken = default) => + context.SendMessageAsync(target, cancellationToken: cancellationToken); + } + + [SendsMessage(typeof(int))] + [SendsMessage(typeof(string))] + [YieldsOutput(typeof(string))] + private sealed class CountdownExecutor(CountdownRecoveryCoordinator coordinator) : Executor("countdown") + { + public override async ValueTask HandleAsync( + int message, + IWorkflowContext context, + CancellationToken cancellationToken = default) + { + if (message <= 0) + { + await context.SendMessageAsync( + "Countdown complete.", + targetId: "complete", + cancellationToken: cancellationToken); + return; + } + + if (coordinator.ShouldBlock(message)) + { + coordinator.Blocked.TrySetResult(); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + } + + await context.YieldOutputAsync(message.ToString(), cancellationToken); + await context.SendMessageAsync( + message - 1, + targetId: "countdown", + cancellationToken: cancellationToken); + } + } + + [YieldsOutput(typeof(string))] + private sealed class CountdownCompleteExecutor() : Executor("complete") + { + public override ValueTask HandleAsync( + string message, + IWorkflowContext context, + CancellationToken cancellationToken = default) => + context.YieldOutputAsync(message, cancellationToken); + } + + private sealed class ResumableAgent(RecoveryCoordinator coordinator) : AIAgent + { + protected override string? IdCore => "resumable-agent"; + + public override string? Name => "resumable-agent"; + + protected override async IAsyncEnumerable + RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var resumableSession = session as ResumableSession + ?? throw new InvalidOperationException( + "The resumable agent requires a ResumableSession."); + string[] input = messages + .Select(message => message.Text) + .Where(text => text is not null) + .ToArray()!; + + if (resumableSession.Phase == 0) + { + Interlocked.Increment(ref coordinator.FreshRuns); + resumableSession.Phase = 1; + yield return NewUpdate("PHASE-1-COMPLETE"); + yield return NewUpdate("PHASE-2-STARTED"); + await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken); + yield break; + } + + Interlocked.Increment(ref coordinator.RecoveryRuns); + coordinator.RecoveryMessages = input; + resumableSession.Phase = 2; + yield return NewUpdate("RECOVERED-COMPLETE"); + await Task.CompletedTask; + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(new ResumableSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + var resumableSession = session as ResumableSession + ?? throw new InvalidOperationException( + "The resumable agent requires a ResumableSession."); + return new(JsonSerializer.SerializeToElement( + new SerializedSession(resumableSession.Phase), + jsonSerializerOptions)); + } + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + SerializedSession state = serializedState.Deserialize( + jsonSerializerOptions) + ?? throw new InvalidOperationException( + "Could not deserialize the resumable session."); + return new(new ResumableSession { Phase = state.Phase }); + } + + private static AgentResponseUpdate NewUpdate(string text) => + new() + { + MessageId = Guid.NewGuid().ToString("N"), + Contents = [new TextContent(text)], + }; + + private sealed class ResumableSession : AgentSession + { + public int Phase { get; set; } + } + + private sealed record SerializedSession(int Phase); + } + + private sealed class PhaseObservingSessionStore( + AgentSessionStore inner, + RecoveryCoordinator coordinator) : AgentSessionStore + { + public override async ValueTask SaveSessionAsync( + AIAgent agent, + string conversationId, + AgentSession session, + string? userId, + CancellationToken cancellationToken = default) + { + JsonElement state = await agent.SerializeSessionAsync( + session, + cancellationToken: cancellationToken); + coordinator.SerializedStates.Add(state.GetRawText()); + await inner.SaveSessionAsync( + agent, + conversationId, + session, + userId, + cancellationToken); + + JsonProperty? phaseProperty = state + .EnumerateObject() + .FirstOrDefault(property => string.Equals( + property.Name, + "phase", + StringComparison.OrdinalIgnoreCase)); + if (phaseProperty is { Value.ValueKind: JsonValueKind.Number } + && phaseProperty.Value.Value.GetInt32() == 1) + { + coordinator.PhasePersisted.TrySetResult(); + } + } + + public override ValueTask GetSessionAsync( + AIAgent agent, + string conversationId, + string? userId, + CancellationToken cancellationToken = default) => + inner.GetSessionAsync( + agent, + conversationId, + userId, + cancellationToken); + } + + private sealed class RecoveryCoordinator + { + public int FreshRuns; + public int RecoveryRuns; + + public string[] RecoveryMessages { get; set; } = []; + + public List SerializedStates { get; } = []; + + public TaskCompletionSource PhasePersisted { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + } + + private sealed class CountdownRecoveryCoordinator(int target, int blockAt) + { + private int _blocked; + + public int Target { get; } = target; + + public TaskCompletionSource Blocked { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public bool ShouldBlock(int value) => + value == blockAt && Interlocked.CompareExchange(ref this._blocked, 1, 0) == 0; + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs index 0efcaa5e657..855e7b861ff 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/ServiceCollectionExtensionsTests.cs @@ -15,6 +15,7 @@ using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Options; using Moq; using OpenAI.Responses; @@ -64,7 +65,11 @@ public void AddFoundryResponses_RegistersResponseHandler() var descriptor = services.FirstOrDefault( d => d.ServiceType == typeof(ResponseHandler)); Assert.NotNull(descriptor); - Assert.Equal(typeof(AgentFrameworkResponseHandler), descriptor.ImplementationType); + Assert.NotNull(descriptor.ImplementationFactory); + + using ServiceProvider provider = services.BuildServiceProvider(); + Assert.IsType( + provider.GetRequiredService()); } [Fact] @@ -95,6 +100,45 @@ public void AddFoundryResponses_CalledTwice_RegistersOnce() Assert.Equal(1, count); } + [Fact] + public void AddFoundryResponses_SecondCall_PreservesNonServerOptions() + { + // Arrange + var services = new ServiceCollection(); + services.AddLogging(); + + // Act + services.AddFoundryResponses(); + services.AddFoundryResponses(options => + options.AllowStoredOutputEnabled = true); + using ServiceProvider provider = services.BuildServiceProvider(); + + // Assert + Assert.True( + provider.GetRequiredService>() + .Value.AllowStoredOutputEnabled); + } + + [Fact] + public void AddFoundryResponses_SecondCallEnablesServerFeature_Throws() + { + // Arrange + var services = new ServiceCollection(); + services.AddLogging(); + services.AddFoundryResponses(); + + // Act + var exception = Assert.Throws( + () => services.AddFoundryResponses(options => + options.SteerableConversations = true)); + + // Assert + Assert.Contains( + "first AddFoundryResponses", + exception.Message, + StringComparison.Ordinal); + } + [Fact] public void AddFoundryResponses_NullServices_ThrowsArgumentNullException() { diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/SteerableLongRunningIntegrationTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/SteerableLongRunningIntegrationTests.cs new file mode 100644 index 00000000000..4025f183f18 --- /dev/null +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.Hosting.UnitTests/SteerableLongRunningIntegrationTests.cs @@ -0,0 +1,295 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting.Server; +using Microsoft.AspNetCore.TestHost; +using Microsoft.Extensions.AI; +using Microsoft.Extensions.DependencyInjection; + +namespace Microsoft.Agents.AI.Foundry.Hosting.UnitTests; + +[Collection(FoundryStateStoreLocalFallbackCollectionDefinition.CollectionName)] +public sealed class SteerableLongRunningIntegrationTests +{ + [Fact] + public async Task ActiveMafTurn_QueuesSteeringThenRunsItOnTheSameSessionAsync() + { + // Arrange + string stateRoot = Path.Combine(Path.GetTempPath(), $"maf-steering-{Guid.NewGuid():N}"); + string? previousStateRoot = Environment.GetEnvironmentVariable("AGENTSERVER_STATE_ROOT"); + string? previousHostingEnvironment = Environment.GetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT"); + var agent = new GatedSteeringAgent(); + + try + { + Environment.SetEnvironmentVariable("AGENTSERVER_STATE_ROOT", stateRoot); + Environment.SetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT", null); + + await using WebApplication app = await StartServerAsync(agent); + using HttpClient client = GetClient(app); + string conversationId = $"conv_{Guid.NewGuid():N}"; + + using HttpResponseMessage first = await PostTurnAsync(client, conversationId, "first instruction"); + using JsonDocument firstBody = await ParseAsync(first); + string firstResponseId = firstBody.RootElement.GetProperty("id").GetString()!; + await agent.FirstTurnEntered.Task.WaitAsync(TimeSpan.FromSeconds(10)); + + // Act + using HttpResponseMessage second = await PostTurnAsync(client, conversationId, "steering instruction"); + using JsonDocument secondBody = await ParseAsync(second); + string secondResponseId = secondBody.RootElement.GetProperty("id").GetString()!; + + // Assert: AgentServer queued the second input rather than invoking MAF concurrently. + Assert.Equal(HttpStatusCode.OK, second.StatusCode); + Assert.Equal("queued", secondBody.RootElement.GetProperty("status").GetString()); + Assert.Equal(1, agent.RunCount); + Assert.Equal(1, agent.MaxConcurrentRuns); + + agent.ReleaseFirstTurn.TrySetResult(); + await agent.SecondTurnEntered.Task.WaitAsync(TimeSpan.FromSeconds(10)); + await WaitForTerminalAsync(client, firstResponseId); + await WaitForTerminalAsync(client, secondResponseId); + + Assert.Equal(2, agent.RunCount); + Assert.Equal(1, agent.MaxConcurrentRuns); + Assert.Collection( + agent.ObservedTurns, + firstTurn => + { + Assert.Equal(1, firstTurn.SessionTurn); + Assert.Contains("first instruction", firstTurn.Input, StringComparison.Ordinal); + }, + secondTurn => + { + Assert.Equal(2, secondTurn.SessionTurn); + Assert.Contains("steering instruction", secondTurn.Input, StringComparison.Ordinal); + }); + } + finally + { + agent.ReleaseFirstTurn.TrySetResult(); + Environment.SetEnvironmentVariable("AGENTSERVER_STATE_ROOT", previousStateRoot); + Environment.SetEnvironmentVariable("FOUNDRY_HOSTING_ENVIRONMENT", previousHostingEnvironment); + + if (Directory.Exists(stateRoot)) + { + Directory.Delete(stateRoot, recursive: true); + } + } + } + + private static async Task StartServerAsync(AIAgent agent) + { + WebApplicationBuilder builder = WebApplication.CreateBuilder(); + builder.WebHost.UseTestServer(); + builder.Services.AddFoundryResponses( + agent, + new InMemoryAgentSessionStore(), + options => + { + options.ResilientBackground = true; + options.SteerableConversations = true; + }); + builder.Services.AddSingleton( + new FakeHostedSessionIsolationKeyProvider()); + builder.Services.AddLogging(); + + WebApplication app = builder.Build(); + app.MapFoundryResponses(); + await app.StartAsync(); + return app; + } + + private static HttpClient GetClient(WebApplication app) => + (app.Services.GetRequiredService() as TestServer + ?? throw new InvalidOperationException("TestServer not found.")) + .CreateClient(); + + private static Task PostTurnAsync( + HttpClient client, + string conversationId, + string input) + { + string body = JsonSerializer.Serialize(new + { + model = "steering-probe", + input, + store = true, + background = true, + conversation = conversationId, + }); + return client.PostAsync( + new Uri("/responses", UriKind.Relative), + new StringContent(body, Encoding.UTF8, "application/json")); + } + + private static async Task ParseAsync(HttpResponseMessage response) => + JsonDocument.Parse(await response.Content.ReadAsStringAsync()); + + private static async Task WaitForTerminalAsync(HttpClient client, string responseId) + { + var deadline = DateTimeOffset.UtcNow.AddSeconds(15); + string last = "(none)"; + while (DateTimeOffset.UtcNow < deadline) + { + using HttpResponseMessage response = await client.GetAsync( + new Uri($"/responses/{responseId}", UriKind.Relative)); + string body = await response.Content.ReadAsStringAsync(); + last = $"{(int)response.StatusCode} {body}"; + if (response.StatusCode == HttpStatusCode.OK) + { + using JsonDocument document = JsonDocument.Parse(body); + string? status = document.RootElement.GetProperty("status").GetString(); + if (status == "completed") + { + return; + } + + if (status is "failed" or "cancelled" or "incomplete") + { + throw new InvalidOperationException( + $"Response '{responseId}' terminated with status '{status}'."); + } + } + + await Task.Delay(TimeSpan.FromMilliseconds(25)); + } + + throw new TimeoutException( + $"Response '{responseId}' did not complete. Last response: {last}"); + } + + private sealed class GatedSteeringAgent : AIAgent + { + private readonly ConcurrentQueue _observedTurns = new(); + private int _activeRuns; + private int _maxConcurrentRuns; + private int _runCount; + + public TaskCompletionSource FirstTurnEntered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource ReleaseFirstTurn { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public TaskCompletionSource SecondTurnEntered { get; } = + new(TaskCreationOptions.RunContinuationsAsynchronously); + + public int RunCount => Volatile.Read(ref this._runCount); + + public int MaxConcurrentRuns => Volatile.Read(ref this._maxConcurrentRuns); + + public IReadOnlyList ObservedTurns => this._observedTurns.ToArray(); + + public override string? Name => "steering-probe"; + + protected override async IAsyncEnumerable RunCoreStreamingAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var probeSession = Assert.IsType(session); + int activeRuns = Interlocked.Increment(ref this._activeRuns); + UpdateMaximum(ref this._maxConcurrentRuns, activeRuns); + + try + { + int run = Interlocked.Increment(ref this._runCount); + int sessionTurn = ++probeSession.Turn; + string input = string.Join( + "\n", + messages.Select(message => message.Text).Where(text => text is not null)); + this._observedTurns.Enqueue(new(sessionTurn, input)); + + if (run == 1) + { + this.FirstTurnEntered.TrySetResult(); + await this.ReleaseFirstTurn.Task.WaitAsync(cancellationToken); + } + else + { + this.SecondTurnEntered.TrySetResult(); + } + + yield return new AgentResponseUpdate + { + MessageId = $"msg_{run}", + Contents = [new TextContent($"TURN-{sessionTurn}-COMPLETE")], + }; + } + finally + { + Interlocked.Decrement(ref this._activeRuns); + } + } + + protected override Task RunCoreAsync( + IEnumerable messages, + AgentSession? session, + AgentRunOptions? options, + CancellationToken cancellationToken = default) => + throw new NotSupportedException(); + + protected override ValueTask CreateSessionCoreAsync( + CancellationToken cancellationToken = default) => + new(new ProbeSession()); + + protected override ValueTask SerializeSessionCoreAsync( + AgentSession session, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + var probeSession = Assert.IsType(session); + return new(JsonSerializer.SerializeToElement( + new SerializedSession(probeSession.Turn), + jsonSerializerOptions)); + } + + protected override ValueTask DeserializeSessionCoreAsync( + JsonElement serializedState, + JsonSerializerOptions? jsonSerializerOptions, + CancellationToken cancellationToken = default) + { + SerializedSession state = serializedState.Deserialize( + jsonSerializerOptions) + ?? throw new InvalidOperationException("Could not deserialize the steering session."); + return new(new ProbeSession { Turn = state.Turn }); + } + + private static void UpdateMaximum(ref int maximum, int candidate) + { + int current; + do + { + current = Volatile.Read(ref maximum); + if (candidate <= current) + { + return; + } + } + while (Interlocked.CompareExchange(ref maximum, candidate, current) != current); + } + + private sealed class ProbeSession : AgentSession + { + public int Turn { get; set; } + } + + private sealed record SerializedSession(int Turn); + } + + private sealed record ObservedTurn(int SessionTurn, string Input); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostingExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostingExtensionsTests.cs index 2f3422c282a..b41becd5651 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostingExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Workflows.UnitTests/WorkflowHostingExtensionsTests.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Threading.Tasks; using Microsoft.Agents.AI.Workflows.InProc; namespace Microsoft.Agents.AI.Workflows.UnitTests; @@ -133,6 +134,41 @@ public void GetService_WorkflowAgentWithItsOwnStore_SaysSo() Assert.True(metadata.UsesOwnCheckpointStorage); } + [Fact] + public async Task GetService_WorkflowSession_ExposesCheckpointRecoveryAsync() + { + // Arrange + AIAgent agent = BuildWorkflowAgent( + InProcessExecution.Lockstep.WithCheckpointing(CheckpointManager.CreateInMemory())); + AgentSession session = await agent.CreateSessionAsync(); + WorkflowSessionCheckpointRecovery recovery = session.GetService() + ?? throw new InvalidOperationException("Workflow checkpoint recovery was not available."); + + // Act + bool prepared = recovery.TryPrepare("checkpoint-from-response"); + + // Assert + Assert.True(prepared); + CheckpointInfo? checkpoint = recovery.CurrentCheckpoint; + Assert.NotNull(checkpoint); + Assert.Equal("checkpoint-from-response", checkpoint.CheckpointId); + Assert.False(string.IsNullOrWhiteSpace(checkpoint.SessionId)); + } + + [Fact] + public void GetService_NonWorkflowSession_HasNoCheckpointRecovery() + { + // Arrange + var session = new NonWorkflowSession(); + + // Act + WorkflowSessionCheckpointRecovery? recovery = + session.GetService(); + + // Assert + Assert.Null(recovery); + } + [Fact] public void GetService_WorkflowAgentBehindAWrapper_IsStillFound() { @@ -180,4 +216,6 @@ private static AIAgent BuildWorkflowAgent(InProcessExecutionEnvironment? executi } private sealed class PassThroughAgent(AIAgent innerAgent) : DelegatingAIAgent(innerAgent); + + private sealed class NonWorkflowSession : AgentSession; }