From 7ece94264b576b1ef0e12c770c0f60c5c3d07bf0 Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 11:08:58 -0700 Subject: [PATCH 1/6] Add MCP tool-coverage drift check for Azure.Sdk.Tools.Mock (#15852) - New eng/scripts/Get-McpToolInventory.ps1 boots the live Azure.Sdk.Tools.Cli MCP server (via 'azsdk list -o json'), enumerates the IMockToolHandler implementations under Azure.Sdk.Tools.Mock, and reports the diff in three buckets: both / live-only / mock-only. - Cross-references mock-tier eval YAMLs under tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/ when present; gracefully no-ops when that folder hasn't landed yet (PR #15811). - '-CheckOnly' exits non-zero on (a) any stale handler that no longer maps to a live tool, or (b) any tool referenced by a mock-tier eval without a handler -- intended for the CI job tracked in #15829. - Documents the drift workflow in Azure.Sdk.Tools.Mock/README.md so a contributor flagged by the script knows how to add a handler. No stale handlers detected against the current live tool set. --- eng/scripts/Get-McpToolInventory.ps1 | 198 ++++++++++++++++++ .../azsdk-cli/Azure.Sdk.Tools.Mock/README.md | 35 ++++ 2 files changed, 233 insertions(+) create mode 100644 eng/scripts/Get-McpToolInventory.ps1 diff --git a/eng/scripts/Get-McpToolInventory.ps1 b/eng/scripts/Get-McpToolInventory.ps1 new file mode 100644 index 00000000000..b1b811b1b60 --- /dev/null +++ b/eng/scripts/Get-McpToolInventory.ps1 @@ -0,0 +1,198 @@ +<# +.SYNOPSIS + Inventory drift report between the live Azure.Sdk.Tools.Cli MCP server and + the Azure.Sdk.Tools.Mock handler set. + +.DESCRIPTION + Boots the live MCP server, captures its tool list, then enumerates the + IMockToolHandler implementations in Azure.Sdk.Tools.Mock and emits a diff: + + live-only - tool exists in the live server, no mock handler -> mock + returns the generic default response (potential gap). + mock-only - mock handler exists, tool no longer in live server + (stale handler). + both - tool exists on both sides. + + Also collects the set of MCP tools referenced by mock-tier evals + (tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/{unit,integration,e2e}/*.eval.yaml) + so -CheckOnly can fail the build only when drift affects something an eval + actually relies on. + +.PARAMETER CheckOnly + Exit non-zero when there is drift on any tool referenced by a mock-tier eval. + Intended for CI: see https://github.com/Azure/azure-sdk-tools/issues/15829. + +.PARAMETER OutputJson + Optional path to write the diff as JSON for downstream tooling. + +.PARAMETER SkipBuild + Skip `dotnet build` for the CLI and Mock projects (assumes they are up to date). + +.EXAMPLE + pwsh eng/scripts/Get-McpToolInventory.ps1 + +.EXAMPLE + pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly +#> +[CmdletBinding()] +param( + [switch]$CheckOnly, + [string]$OutputJson, + [switch]$SkipBuild +) + +Set-StrictMode -Version 4 +$ErrorActionPreference = 'Stop' + +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') +$cliProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Cli' +$mockProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Mock' +$evalsRoot = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals' + +if (-not (Test-Path $cliProject)) { throw "CLI project not found: $cliProject" } +if (-not (Test-Path $mockProject)) { throw "Mock project not found: $mockProject" } + +function Invoke-DotnetBuild { + param([string]$Project) + Write-Host "Building $Project ..." -ForegroundColor DarkGray + & dotnet build $Project --nologo --verbosity quiet + if ($LASTEXITCODE -ne 0) { + throw "dotnet build failed for $Project (exit $LASTEXITCODE)" + } +} + +function Get-LiveMcpTools { + param([string]$CliProject) + + Write-Host "Querying live MCP tool list via 'azsdk list -o json' ..." -ForegroundColor DarkGray + $json = & dotnet run --project $CliProject --no-build -- list -o json + if ($LASTEXITCODE -ne 0) { + throw "Failed to run 'list -o json' on $CliProject (exit $LASTEXITCODE)" + } + + $parsed = $json | ConvertFrom-Json + $names = @() + foreach ($t in $parsed.Tools) { + if ($t.McpToolName) { + $names += [string]$t.McpToolName + } + } + return $names | Sort-Object -Unique +} + +function Get-MockHandlerToolNames { + param([string]$MockProject) + + # The README documents the handler contract as + # public string ToolName => "azsdk_xxx"; + # Parsing source is robust and avoids loading the assembly + all its + # dependencies into the PowerShell process. + $pattern = [regex]'(?m)ToolName\s*=>\s*"([^"]+)"' + $names = @() + Get-ChildItem -LiteralPath (Join-Path $MockProject 'Handlers') -Recurse -Filter *.cs | + ForEach-Object { + $text = Get-Content -LiteralPath $_.FullName -Raw + foreach ($m in $pattern.Matches($text)) { + $names += $m.Groups[1].Value + } + } + return $names | Sort-Object -Unique +} + +function Get-EvalReferencedTools { + param([string]$EvalsRoot) + + if (-not (Test-Path $EvalsRoot)) { + return @() + } + $pattern = [regex]'azsdk_[A-Za-z0-9_]+' + $names = @() + Get-ChildItem -LiteralPath $EvalsRoot -Recurse -Filter *.yaml | + ForEach-Object { + $text = Get-Content -LiteralPath $_.FullName -Raw + foreach ($m in $pattern.Matches($text)) { + $names += $m.Value + } + } + return $names | Sort-Object -Unique +} + +function Write-Section { + param([string]$Title, [string[]]$Items, [ConsoleColor]$Color = [ConsoleColor]::Gray) + Write-Host "" + Write-Host "== $Title ($($Items.Count)) ==" -ForegroundColor $Color + if ($Items.Count -eq 0) { + Write-Host " (none)" -ForegroundColor DarkGray + } + else { + $Items | ForEach-Object { Write-Host " $_" -ForegroundColor $Color } + } +} + +if (-not $SkipBuild) { + Invoke-DotnetBuild -Project $cliProject + Invoke-DotnetBuild -Project $mockProject +} + +$liveTools = @(Get-LiveMcpTools -CliProject $cliProject) +$mockTools = @(Get-MockHandlerToolNames -MockProject $mockProject) +$evalTools = @(Get-EvalReferencedTools -EvalsRoot $evalsRoot) + +$liveSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$liveTools, [System.StringComparer]::OrdinalIgnoreCase) +$mockSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$mockTools, [System.StringComparer]::OrdinalIgnoreCase) +$evalSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$evalTools, [System.StringComparer]::OrdinalIgnoreCase) + +$liveOnly = @($liveTools | Where-Object { -not $mockSet.Contains($_) }) +$mockOnly = @($mockTools | Where-Object { -not $liveSet.Contains($_) }) +$both = @($liveTools | Where-Object { $mockSet.Contains($_) }) + +# Mock-tier eval gaps: tool referenced by an eval, exists live, but no mock handler. +$evalGaps = @($liveOnly | Where-Object { $evalSet.Contains($_) }) + +Write-Host "MCP tool inventory" -ForegroundColor White +Write-Host " live tools: $($liveTools.Count)" +Write-Host " mock handlers: $($mockTools.Count)" +Write-Host " eval refs: $($evalTools.Count)" + +Write-Section -Title 'both (live + mock handler)' -Items $both -Color Green +Write-Section -Title 'live-only (no mock handler)' -Items $liveOnly -Color Yellow +Write-Section -Title 'mock-only (stale handler)' -Items $mockOnly -Color Magenta +Write-Section -Title 'mock-tier eval gaps (require handler)' -Items $evalGaps -Color Red + +if ($OutputJson) { + $payload = [ordered]@{ + liveTools = $liveTools + mockTools = $mockTools + evalTools = $evalTools + both = $both + liveOnly = $liveOnly + mockOnly = $mockOnly + evalGaps = $evalGaps + } + $payload | ConvertTo-Json -Depth 4 | Out-File -LiteralPath $OutputJson -Encoding utf8 + Write-Host "Wrote $OutputJson" -ForegroundColor DarkGray +} + +if ($CheckOnly) { + $fail = $false + if ($mockOnly.Count -gt 0) { + Write-Host "" + Write-Host "Drift detected: $($mockOnly.Count) mock handler(s) target tools that no longer exist in the live MCP server." -ForegroundColor Red + Write-Host "Either delete the stale handler(s) or rename them to match the new live tool name." -ForegroundColor Red + $fail = $true + } + if ($evalGaps.Count -gt 0) { + Write-Host "" + Write-Host "Drift detected: $($evalGaps.Count) mock-tier eval tool(s) lack a mock handler." -ForegroundColor Red + Write-Host "Add a handler under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ for each tool above." -ForegroundColor Red + Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red + $fail = $true + } + if ($fail) { + exit 1 + } + Write-Host "" + Write-Host "OK - no stale handlers, no mock-tier eval gaps." -ForegroundColor Green +} + +return diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md index ea326f81f17..dae7e8963f4 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md @@ -115,3 +115,38 @@ This lets callers control the mock behavior through input: - `{"message": "Alice"}` → normal success response Use this pattern in any handler to test how your integration handles different scenarios without changing the mock server code. + +## Keeping the Mock in Sync with the Live MCP Server + +The mock reuses the live CLI's tool definitions, so the *set* of advertised tools is always identical. What can drift is which tools have a hand-written `IMockToolHandler`. Tools without a handler fall back to the generic default response — fine for noise, but it hides routing / arg regressions when a scenario actually depends on that tool returning a realistic shape. + +Use the inventory script to audit: + +```powershell +pwsh eng/scripts/Get-McpToolInventory.ps1 +``` + +It produces three buckets: + +- **both** — live tool with a hand-written handler. No action. +- **live-only** — live tool that falls back to the default response. Add a handler if any eval depends on it. +- **mock-only** — handler for a tool that no longer exists on the live server. Rename or delete the stale handler. + +CI runs the same script with `-CheckOnly`: + +```powershell +pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly +``` + +`-CheckOnly` exits non-zero when: + +1. There is a **mock-only** drift (stale handler that no longer maps to a live tool), or +2. A tool referenced by a mock-tier eval (under `tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/`) has no handler. + +### Workflow when the script flags a gap + +1. Look up the live tool's response type. Tool method signatures live under `tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/`. The return type is usually a typed `CommandResponse` in `Azure.Sdk.Tools.Cli.Models.Responses.*`. +2. Add a new file under `Handlers//` (e.g., `Handlers/Pipeline/MyToolHandler.cs`). +3. Implement `IMockToolHandler`. Set `ToolName` to the exact `[McpServerTool(Name = "…")]` value from the real tool. +4. Return an instance of the same response type the real tool returns, populated with realistic sample data. For scenarios that need to exercise multiple branches, switch on `arguments` (see `HelloWorldHandler` above). +5. Re-run the inventory script to confirm the tool moved from **live-only** to **both**. From 7038cb6a841ffb201efc0a0a93c4642e23938cd6 Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 11:24:07 -0700 Subject: [PATCH 2/6] Add mock handlers for remaining live MCP tools; drop eval scanning from inventory script (#15852) - 13 new handler files covering 63 live tools that previously fell back to the default response (APIView, Codeowners, EngSys, GitHub, Package, Pipeline, ReleasePlan, TypeSpec, Verify, Core, Example). - Get-McpToolInventory.ps1: pure live-vs-mock parity (removes Vally eval cross-reference); -CheckOnly fails if either bucket is non-empty. - README: updated sync workflow to reflect parity-only check. --- eng/scripts/Get-McpToolInventory.ps1 | 46 ++---- .../Handlers/APIView/ApiViewHandlers.cs | 96 +++++++++++ .../Handlers/Config/ConfigHandlers.cs | 149 ++++++++++++++++++ .../Handlers/Core/UpgradeHandler.cs | 23 +++ .../Handlers/EngSys/EngSysHandlers.cs | 94 +++++++++++ .../Example/ExampleAndDebugHandlers.cs | 105 ++++++++++++ .../Handlers/GitHub/GitHubHandlers.cs | 85 ++++++++++ .../Package/PackageOperationHandlers.cs | 127 +++++++++++++++ .../Pipeline/AnalyzeAndArtifactsHandlers.cs | 66 ++++++++ .../ReleasePlanRemainingHandlers.cs | 126 +++++++++++++++ .../Handlers/TypeSpec/TypeSpecHandlers.cs | 130 +++++++++++++++ .../Handlers/Verify/VerifySetupHandler.cs | 34 ++++ .../azsdk-cli/Azure.Sdk.Tools.Mock/README.md | 9 +- 13 files changed, 1047 insertions(+), 43 deletions(-) create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/APIView/ApiViewHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Config/ConfigHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Core/UpgradeHandler.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/EngSys/EngSysHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Example/ExampleAndDebugHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/GitHub/GitHubHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Package/PackageOperationHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Pipeline/AnalyzeAndArtifactsHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/TypeSpec/TypeSpecHandlers.cs create mode 100644 tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Verify/VerifySetupHandler.cs diff --git a/eng/scripts/Get-McpToolInventory.ps1 b/eng/scripts/Get-McpToolInventory.ps1 index b1b811b1b60..fb5e45c0d40 100644 --- a/eng/scripts/Get-McpToolInventory.ps1 +++ b/eng/scripts/Get-McpToolInventory.ps1 @@ -47,7 +47,6 @@ $ErrorActionPreference = 'Stop' $repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') $cliProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Cli' $mockProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Mock' -$evalsRoot = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals' if (-not (Test-Path $cliProject)) { throw "CLI project not found: $cliProject" } if (-not (Test-Path $mockProject)) { throw "Mock project not found: $mockProject" } @@ -99,24 +98,6 @@ function Get-MockHandlerToolNames { return $names | Sort-Object -Unique } -function Get-EvalReferencedTools { - param([string]$EvalsRoot) - - if (-not (Test-Path $EvalsRoot)) { - return @() - } - $pattern = [regex]'azsdk_[A-Za-z0-9_]+' - $names = @() - Get-ChildItem -LiteralPath $EvalsRoot -Recurse -Filter *.yaml | - ForEach-Object { - $text = Get-Content -LiteralPath $_.FullName -Raw - foreach ($m in $pattern.Matches($text)) { - $names += $m.Value - } - } - return $names | Sort-Object -Unique -} - function Write-Section { param([string]$Title, [string[]]$Items, [ConsoleColor]$Color = [ConsoleColor]::Gray) Write-Host "" @@ -136,38 +117,29 @@ if (-not $SkipBuild) { $liveTools = @(Get-LiveMcpTools -CliProject $cliProject) $mockTools = @(Get-MockHandlerToolNames -MockProject $mockProject) -$evalTools = @(Get-EvalReferencedTools -EvalsRoot $evalsRoot) $liveSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$liveTools, [System.StringComparer]::OrdinalIgnoreCase) $mockSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$mockTools, [System.StringComparer]::OrdinalIgnoreCase) -$evalSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$evalTools, [System.StringComparer]::OrdinalIgnoreCase) $liveOnly = @($liveTools | Where-Object { -not $mockSet.Contains($_) }) $mockOnly = @($mockTools | Where-Object { -not $liveSet.Contains($_) }) $both = @($liveTools | Where-Object { $mockSet.Contains($_) }) -# Mock-tier eval gaps: tool referenced by an eval, exists live, but no mock handler. -$evalGaps = @($liveOnly | Where-Object { $evalSet.Contains($_) }) - Write-Host "MCP tool inventory" -ForegroundColor White -Write-Host " live tools: $($liveTools.Count)" +Write-Host " live tools: $($liveTools.Count)" Write-Host " mock handlers: $($mockTools.Count)" -Write-Host " eval refs: $($evalTools.Count)" Write-Section -Title 'both (live + mock handler)' -Items $both -Color Green Write-Section -Title 'live-only (no mock handler)' -Items $liveOnly -Color Yellow Write-Section -Title 'mock-only (stale handler)' -Items $mockOnly -Color Magenta -Write-Section -Title 'mock-tier eval gaps (require handler)' -Items $evalGaps -Color Red if ($OutputJson) { $payload = [ordered]@{ liveTools = $liveTools mockTools = $mockTools - evalTools = $evalTools both = $both liveOnly = $liveOnly mockOnly = $mockOnly - evalGaps = $evalGaps } $payload | ConvertTo-Json -Depth 4 | Out-File -LiteralPath $OutputJson -Encoding utf8 Write-Host "Wrote $OutputJson" -ForegroundColor DarkGray @@ -175,24 +147,24 @@ if ($OutputJson) { if ($CheckOnly) { $fail = $false - if ($mockOnly.Count -gt 0) { + if ($liveOnly.Count -gt 0) { Write-Host "" - Write-Host "Drift detected: $($mockOnly.Count) mock handler(s) target tools that no longer exist in the live MCP server." -ForegroundColor Red - Write-Host "Either delete the stale handler(s) or rename them to match the new live tool name." -ForegroundColor Red + Write-Host "Drift detected: $($liveOnly.Count) live tool(s) have no mock handler." -ForegroundColor Red + Write-Host "Add a handler under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ for each tool above." -ForegroundColor Red + Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red $fail = $true } - if ($evalGaps.Count -gt 0) { + if ($mockOnly.Count -gt 0) { Write-Host "" - Write-Host "Drift detected: $($evalGaps.Count) mock-tier eval tool(s) lack a mock handler." -ForegroundColor Red - Write-Host "Add a handler under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ for each tool above." -ForegroundColor Red - Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red + Write-Host "Drift detected: $($mockOnly.Count) mock handler(s) target tools that no longer exist in the live MCP server." -ForegroundColor Red + Write-Host "Either delete the stale handler(s) or rename them to match the new live tool name." -ForegroundColor Red $fail = $true } if ($fail) { exit 1 } Write-Host "" - Write-Host "OK - no stale handlers, no mock-tier eval gaps." -ForegroundColor Green + Write-Host "OK - mock handler set matches the live MCP tool list." -ForegroundColor Green } return diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/APIView/ApiViewHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/APIView/ApiViewHandlers.cs new file mode 100644 index 00000000000..6e0e8099901 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/APIView/ApiViewHandlers.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses; + +namespace Azure.Sdk.Tools.Mock.Handlers.APIView; + +/// +/// Mock handler for azsdk_apiview_get_comments. Returns a small fixed comment payload so +/// callers can exercise the "consume APIView feedback" path deterministically. +/// +public class ApiViewGetCommentsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_apiview_get_comments"; + + public CommandResponse Handle(Dictionary? arguments) => new APIViewResponse + { + Message = "Retrieved APIView comments", + Language = arguments?.GetValueOrDefault("language")?.ToString() ?? ".NET", + PackageName = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso", + Result = new[] + { + new + { + id = "comment-1", + line = 42, + text = "Consider renaming this property for clarity.", + author = "reviewer@microsoft.com", + resolved = false + } + } + }; +} + +/// +/// Mock handler for azsdk_apiview_get_review_url. Returns a deterministic review URL for the +/// requested package + language. +/// +public class ApiViewGetReviewUrlHandler : IMockToolHandler +{ + public string ToolName => "azsdk_apiview_get_review_url"; + + public CommandResponse Handle(Dictionary? arguments) + { + var language = arguments?.GetValueOrDefault("language")?.ToString() ?? "dotnet"; + var package = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso"; + return new APIViewResponse + { + Message = "APIView URL resolved", + Language = language, + PackageName = package, + Result = $"https://apiview.dev/Assemblies/Review/mock-{language}-{package}".ToLowerInvariant() + }; + } +} + +/// +/// Mock handler for azsdk_apiview_request_copilot_review. Returns a deterministic job ID +/// callers can poll with azsdk_apiview_get_copilot_review. +/// +public class ApiViewRequestCopilotReviewHandler : IMockToolHandler +{ + public string ToolName => "azsdk_apiview_request_copilot_review"; + + public CommandResponse Handle(Dictionary? arguments) => new APIViewResponse + { + Message = "Copilot review submitted", + Language = arguments?.GetValueOrDefault("language")?.ToString() ?? ".NET", + PackageName = arguments?.GetValueOrDefault("packageName")?.ToString() ?? "Azure.Template.Contoso", + Result = "mock-copilot-job-00000001" + }; +} + +/// +/// Mock handler for azsdk_apiview_get_copilot_review. Returns a "completed" review with a +/// single sample comment. +/// +public class ApiViewGetCopilotReviewHandler : IMockToolHandler +{ + public string ToolName => "azsdk_apiview_get_copilot_review"; + + public CommandResponse Handle(Dictionary? arguments) => new APIViewResponse + { + Message = "Copilot review complete", + Result = new + { + jobId = arguments?.GetValueOrDefault("jobId")?.ToString() ?? "mock-copilot-job-00000001", + status = "Completed", + comments = new[] + { + new { line = 24, severity = "info", text = "Mock Copilot suggestion: tighten the parameter type." } + } + } + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Config/ConfigHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Config/ConfigHandlers.cs new file mode 100644 index 00000000000..eb995af3e80 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Config/ConfigHandlers.cs @@ -0,0 +1,149 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses; +using Azure.Sdk.Tools.Cli.Models.Responses.Codeowners; + +namespace Azure.Sdk.Tools.Mock.Handlers.Config; + +/// Mock handler for azsdk_check_service_label. +public class CheckServiceLabelHandler : IMockToolHandler +{ + public string ToolName => "azsdk_check_service_label"; + public CommandResponse Handle(Dictionary? arguments) => new ServiceLabelResponse + { + Label = arguments?.GetValueOrDefault("label")?.ToString() ?? "Contoso.WidgetManager", + Status = "Exists" + }; +} + +/// Mock handler for azsdk_create_service_label. +public class CreateServiceLabelHandler : IMockToolHandler +{ + public string ToolName => "azsdk_create_service_label"; + public CommandResponse Handle(Dictionary? arguments) + { + var label = arguments?.GetValueOrDefault("label")?.ToString() ?? "Contoso.WidgetManager"; + return new ServiceLabelResponse + { + Label = label, + Status = "Created", + PullRequestUrl = $"https://github.com/Azure/azure-sdk-tools/pull/99001" + }; + } +} + +/// Mock handler for azsdk_engsys_codeowner_view. +public class CodeownerViewHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_view"; + public CommandResponse Handle(Dictionary? arguments) => new CodeownersViewResponse + { + Packages = + [ + new PackageResponse + { + WorkItemId = 70001, + PackageName = "Azure.Template.Contoso", + Language = ".NET", + PackageType = "client", + Owners = + [ + new OwnerResponse { GitHubAlias = "contoso-owner-1" }, + new OwnerResponse { GitHubAlias = "contoso-owner-2" } + ], + Labels = ["Contoso.WidgetManager"] + } + ] + }; +} + +/// Mock handler for azsdk_engsys_codeowner_check_package. +public class CodeownerCheckPackageHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_check_package"; + public CommandResponse Handle(Dictionary? arguments) => new CheckPackageResponse + { + DirectoryPath = arguments?.GetValueOrDefault("directoryPath")?.ToString() ?? "sdk/contoso/Azure.Template.Contoso", + Owners = ["contoso-owner-1", "contoso-owner-2"], + PRLabels = ["Contoso.WidgetManager"], + ServiceOwners = ["service-team-lead"], + ServiceLabels = ["Service Attention", "Contoso.WidgetManager"] + }; +} + +internal static class CodeownersModifyMockResponses +{ + public static CodeownersModifyResponse OkWithMessage() => new() + { + View = new CodeownersViewResponse + { + Packages = + [ + new PackageResponse + { + WorkItemId = 70001, + PackageName = "Azure.Template.Contoso", + Language = ".NET", + PackageType = "client", + Owners = [new OwnerResponse { GitHubAlias = "contoso-owner-1" }], + Labels = ["Contoso.WidgetManager"] + } + ] + } + }; +} + +/// Mock handler for azsdk_engsys_codeowner_add_package_owner. +public class CodeownerAddPackageOwnerHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_add_package_owner"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_add_package_label. +public class CodeownerAddPackageLabelHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_add_package_label"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_add_label_owner. +public class CodeownerAddLabelOwnerHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_add_label_owner"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_remove_package_owner. +public class CodeownerRemovePackageOwnerHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_remove_package_owner"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_remove_package_label. +public class CodeownerRemovePackageLabelHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_remove_package_label"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_remove_label_owner. +public class CodeownerRemoveLabelOwnerHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_remove_label_owner"; + public CommandResponse Handle(Dictionary? arguments) => CodeownersModifyMockResponses.OkWithMessage(); +} + +/// Mock handler for azsdk_engsys_codeowner_update_cache. +public class CodeownerUpdateCacheHandler : IMockToolHandler +{ + public string ToolName => "azsdk_engsys_codeowner_update_cache"; + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "CODEOWNERS cache refreshed (mock)", + Result = new { packagesRefreshed = 1, labelOwnersRefreshed = 1 } + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Core/UpgradeHandler.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Core/UpgradeHandler.cs new file mode 100644 index 00000000000..71f767a7cb9 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Core/UpgradeHandler.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; + +namespace Azure.Sdk.Tools.Mock.Handlers.Core; + +/// +/// Mock handler for azsdk_upgrade. Always reports the current "mock" version is up to date so +/// callers exercising the upgrade flow don't trigger a real download. +/// +public class UpgradeHandler : IMockToolHandler +{ + public string ToolName => "azsdk_upgrade"; + + public CommandResponse Handle(Dictionary? arguments) => new UpgradeResponse + { + OldVersion = "0.0.0-mock", + NewVersion = "0.0.0-mock", + Message = "azsdk is already up to date (mock).", + RestartRequired = false + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/EngSys/EngSysHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/EngSys/EngSysHandlers.cs new file mode 100644 index 00000000000..f0dce4fe65a --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/EngSys/EngSysHandlers.cs @@ -0,0 +1,94 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses; + +namespace Azure.Sdk.Tools.Mock.Handlers.EngSys; + +/// Mock handler for azsdk_analyze_log_file. Returns a single fake build error. +public class AnalyzeLogFileHandler : IMockToolHandler +{ + public string ToolName => "azsdk_analyze_log_file"; + public CommandResponse Handle(Dictionary? arguments) => new LogAnalysisResponse + { + Summary = "Detected one CS0246 compile error in the build log.", + SuggestedFix = "Add a `using Azure.Core;` directive at the top of the file.", + Errors = + [ + new LogEntry + { + File = "src/Contoso.Widgets/Generated/WidgetsClient.cs", + Line = 42, + Message = "error CS0246: The type or namespace name 'WidgetOptions' could not be found" + } + ] + }; +} + +/// Mock handler for azsdk_get_failed_test_cases. +public class GetFailedTestCasesHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_failed_test_cases"; + public CommandResponse Handle(Dictionary? arguments) => new FailedTestRunListResponse + { + Items = + [ + new FailedTestRunResponse + { + RunId = 100001, + TestCaseTitle = "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget", + Outcome = "Failed", + ErrorMessage = "Expected status 200, got 404.", + StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()" + } + ] + }; +} + +/// Mock handler for azsdk_get_failed_test_case_data. +public class GetFailedTestCaseDataHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_failed_test_case_data"; + public CommandResponse Handle(Dictionary? arguments) => new FailedTestRunResponse + { + RunId = 100001, + TestCaseTitle = arguments?.GetValueOrDefault("testCaseTitle")?.ToString() + ?? "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget", + Outcome = "Failed", + ErrorMessage = "Expected status 200, got 404.", + StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()", + Uri = "https://dev.azure.com/azure-sdk/internal/_test/cases?id=100001" + }; +} + +/// Mock handler for azsdk_get_failed_test_run_data. +public class GetFailedTestRunDataHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_failed_test_run_data"; + public CommandResponse Handle(Dictionary? arguments) => new FailedTestRunListResponse + { + Items = + [ + new FailedTestRunResponse + { + RunId = int.TryParse(arguments?.GetValueOrDefault("runId")?.ToString(), out var id) ? id : 100001, + TestCaseTitle = "Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget", + Outcome = "Failed", + ErrorMessage = "Expected status 200, got 404.", + StackTrace = "at Contoso.Widgets.Tests.WidgetClientLiveTests.GetWidget()" + } + ] + }; +} + +/// Mock handler for azsdk_cleanup_ai_agents. +public class CleanupAiAgentsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_cleanup_ai_agents"; + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "AI agents cleaned up (mock)", + Result = new { agentsDeleted = 3, threadsDeleted = 5 } + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Example/ExampleAndDebugHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Example/ExampleAndDebugHandlers.cs new file mode 100644 index 00000000000..8ee1da1ba65 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Example/ExampleAndDebugHandlers.cs @@ -0,0 +1,105 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; + +namespace Azure.Sdk.Tools.Mock.Handlers.Example; + +/// +/// Mock handler for azsdk_hello_world_fail. Always returns an error response so callers +/// can exercise failure-path handling deterministically. +/// +public class HelloWorldFailHandler : IMockToolHandler +{ + public string ToolName => "azsdk_hello_world_fail"; + + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "Simulated failure from azsdk_hello_world_fail", + ResponseError = "MOCK_HELLO_WORLD_FAIL" + }; +} + +/// +/// Shared mock for all azsdk_example_* tools. Each tool maps to a fixed service name and operation +/// label so callers can tell which path was exercised without changing the response shape. +/// +internal static class ExampleResponses +{ + public static CommandResponse Build(string serviceName, string operation, string result) => + new ExampleServiceResponse + { + ServiceName = serviceName, + Operation = operation, + Result = result, + Details = new Dictionary + { + ["mock"] = "true", + ["correlationId"] = "00000000-0000-0000-0000-000000000001" + } + }; +} + +public class ExampleAzureServiceHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_azure_service"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("AzureStorage", "ListBlobs", "OK"); +} + +public class ExampleDevOpsServiceHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_devops_service"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("AzureDevOps", "GetBuild", "OK"); +} + +public class ExampleGitHubServiceHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_github_service"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("GitHub", "GetIssue", "OK"); +} + +public class ExampleAiServiceHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_ai_service"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("AzureOpenAI", "Completion", "OK"); +} + +public class ExampleErrorHandlingHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_error_handling"; + public CommandResponse Handle(Dictionary? arguments) + { + var response = ExampleResponses.Build("ErrorHandling", "Simulated", "Failed"); + response.ResponseError = "SIMULATED_ERROR"; + return response; + } +} + +public class ExampleProcessExecutionHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_process_execution"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("Process", "sleep 1", "exit 0"); +} + +public class ExamplePowershellExecutionHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_powershell_execution"; + public CommandResponse Handle(Dictionary? arguments) => + ExampleResponses.Build("PowerShell", "Get-Date", "exit 0"); +} + +public class ExampleAgentFibonacciHandler : IMockToolHandler +{ + public string ToolName => "azsdk_example_agent_fibonacci"; + public CommandResponse Handle(Dictionary? arguments) + { + var n = arguments?.GetValueOrDefault("n")?.ToString() ?? "10"; + // 10th Fibonacci number; intentionally fixed so the mock is deterministic regardless of input. + return ExampleResponses.Build("AgentFibonacci", $"fib({n})", "55"); + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/GitHub/GitHubHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/GitHub/GitHubHandlers.cs new file mode 100644 index 00000000000..9a71f695a56 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/GitHub/GitHubHandlers.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses.GitHub; + +namespace Azure.Sdk.Tools.Mock.Handlers.GitHub; + +/// Mock handler for azsdk_get_github_user_details. +public class GetGitHubUserDetailsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_github_user_details"; + public CommandResponse Handle(Dictionary? arguments) + { + var login = arguments?.GetValueOrDefault("userName")?.ToString() ?? "contoso-user"; + return new DefaultCommandResponse + { + Message = $"Retrieved user details for {login}", + Result = new + { + login, + name = "Contoso User", + email = $"{login}@microsoft.com", + company = "Microsoft" + } + }; + } +} + +internal static class GitHubMockResponses +{ + public static PullRequestDetails ContosoPr(string? url = null) => new() + { + pullRequestNumber = 45001, + Author = "contoso-user", + Status = "open", + Url = url ?? "https://github.com/Azure/azure-sdk-for-net/pull/45001", + IsMerged = false, + IsMergeable = true, + Checks = ["ci: passed", "live-tests: pending"], + Comments = [] + }; +} + +/// Mock handler for azsdk_get_pull_request_link_for_current_branch. +public class GetPrLinkForCurrentBranchHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_pull_request_link_for_current_branch"; + public CommandResponse Handle(Dictionary? arguments) + { + var url = "https://github.com/Azure/azure-sdk-for-net/pull/45001"; + return new GetPullRequestResponse + { + PullRequestUrl = url, + PullRequest = GitHubMockResponses.ContosoPr(url) + }; + } +} + +/// Mock handler for azsdk_create_pull_request. +public class CreatePullRequestHandler : IMockToolHandler +{ + public string ToolName => "azsdk_create_pull_request"; + public CommandResponse Handle(Dictionary? arguments) => new CreatePullRequestResponse + { + PullRequestUrl = "https://github.com/Azure/azure-sdk-for-net/pull/45002", + Messages = ["Pull request created (mock)"] + }; +} + +/// Mock handler for azsdk_get_pull_request. +public class GetPullRequestHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_pull_request"; + public CommandResponse Handle(Dictionary? arguments) + { + var url = arguments?.GetValueOrDefault("pullRequestUrl")?.ToString() + ?? "https://github.com/Azure/azure-sdk-for-net/pull/45001"; + return new GetPullRequestResponse + { + PullRequestUrl = url, + PullRequest = GitHubMockResponses.ContosoPr(url) + }; + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Package/PackageOperationHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Package/PackageOperationHandlers.cs new file mode 100644 index 00000000000..157e7ca4a13 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Package/PackageOperationHandlers.cs @@ -0,0 +1,127 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.Responses.Package; + +namespace Azure.Sdk.Tools.Mock.Handlers.Package; + +internal static class PackageMockResponses +{ + public static PackageInfo ContosoPackage(string? language = null) => new() + { + PackageName = "Azure.Template.Contoso", + PackageVersion = "1.0.0-beta.1", + Language = language is null ? SdkLanguage.DotNet : SdkLanguageHelpers.GetSdkLanguage(language), + SdkType = SdkType.Dataplane + }; +} + +/// Mock handler for azsdk_package_generate_code. +public class PackageGenerateCodeHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_generate_code"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + "SDK code generated successfully (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString()), + typespecProjectPath: arguments?.GetValueOrDefault("typeSpecProjectPath")?.ToString()); +} + +/// Mock handler for azsdk_package_build_code. +public class PackageBuildCodeHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_build_code"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + "Package built successfully (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); +} + +/// Mock handler for azsdk_package_run_tests. +public class PackageRunTestsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_run_tests"; + public CommandResponse Handle(Dictionary? arguments) => + new TestRunResponse(exitCode: 0, testRunOutput: "Test run successful. 42 passed, 0 failed (mock)."); +} + +/// Mock handler for azsdk_package_run_check. +public class PackageRunCheckHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_run_check"; + public CommandResponse Handle(Dictionary? arguments) => + new PackageCheckResponse( + packageName: "Azure.Template.Contoso", + language: SdkLanguage.DotNet, + exitCode: 0, + checkStatusDetails: "All checks passed (mock)."); +} + +/// Mock handler for azsdk_package_pack. +public class PackagePackHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_pack"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + "Package archive created (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); +} + +/// Mock handler for azsdk_package_update_version. +public class PackageUpdateVersionHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_update_version"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + $"Version updated to {arguments?.GetValueOrDefault("newVersion")?.ToString() ?? "1.0.0-beta.2"} (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); +} + +/// Mock handler for azsdk_package_update_metadata. +public class PackageUpdateMetadataHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_update_metadata"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + "Metadata updated (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); +} + +/// Mock handler for azsdk_package_update_changelog_content. +public class PackageUpdateChangelogContentHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_update_changelog_content"; + public CommandResponse Handle(Dictionary? arguments) => + PackageOperationResponse.CreateSuccess( + "Changelog updated (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); +} + +/// Mock handler for azsdk_package_generate_samples. +public class PackageGenerateSamplesHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_generate_samples"; + public CommandResponse Handle(Dictionary? arguments) + { + var resp = PackageOperationResponse.CreateSuccess( + "Samples generated (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); + resp.Result = new { samples_count = 3 }; + return resp; + } +} + +/// Mock handler for azsdk_package_translate_samples. +public class PackageTranslateSamplesHandler : IMockToolHandler +{ + public string ToolName => "azsdk_package_translate_samples"; + public CommandResponse Handle(Dictionary? arguments) + { + var resp = PackageOperationResponse.CreateSuccess( + "Samples translated (mock)", + PackageMockResponses.ContosoPackage(arguments?.GetValueOrDefault("language")?.ToString())); + resp.Result = new { samples_count = 3 }; + return resp; + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Pipeline/AnalyzeAndArtifactsHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Pipeline/AnalyzeAndArtifactsHandlers.cs new file mode 100644 index 00000000000..5d72a7d130c --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Pipeline/AnalyzeAndArtifactsHandlers.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; + +namespace Azure.Sdk.Tools.Mock.Handlers.Pipeline; + +/// Mock handler for azsdk_analyze_pipeline. Returns a fake-build summary with a single failed test + task. +public class AnalyzePipelineHandler : IMockToolHandler +{ + public string ToolName => "azsdk_analyze_pipeline"; + + public CommandResponse Handle(Dictionary? arguments) + { + var buildId = arguments?.GetValueOrDefault("buildId")?.ToString() ?? "90001"; + return new AnalyzePipelineResponse + { + PipelineUrl = $"https://dev.azure.com/azure-sdk/internal/_build/results?buildId={buildId}", + FailedTests = new Dictionary> + { + ["Contoso.Widgets.Tests"] = ["WidgetClientLiveTests.GetWidget"] + }, + FailedTasks = + [ + new LogAnalysisResponse + { + Summary = "1 test failure detected in WidgetClientLiveTests", + SuggestedFix = "Check the live test recording for stale data and re-record.", + Errors = + [ + new LogEntry + { + File = "logs/test.log", + Line = 128, + Message = "Test WidgetClientLiveTests.GetWidget failed: expected 200 got 404" + } + ] + } + ] + }; + } +} + +/// Mock handler for azsdk_get_pipeline_llm_artifacts. +public class GetPipelineLlmArtifactsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_pipeline_llm_artifacts"; + + public CommandResponse Handle(Dictionary? arguments) + { + var buildId = arguments?.GetValueOrDefault("buildId")?.ToString() ?? "90001"; + return new ObjectCommandResponse + { + Message = $"Retrieved LLM artifacts for build {buildId} (mock)", + Result = new + { + buildId, + artifacts = new[] + { + new { name = "log-analysis.json", sizeBytes = 4096 }, + new { name = "failed-tests.json", sizeBytes = 2048 } + } + } + }; + } +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs new file mode 100644 index 00000000000..fafdf2de999 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.AzureDevOps; +using Azure.Sdk.Tools.Cli.Models.Responses.ReleasePlan; + +namespace Azure.Sdk.Tools.Mock.Handlers.ReleasePlan; + +internal static class ReleasePlanMockResponses +{ + public static ReleasePlanWorkItem ContosoWorkItem(string? typespecPath = null, string? releaseMonth = null) => new() + { + WorkItemId = 35000, + Title = "Release Plan - Contoso.WidgetManager", + Status = "Active", + Owner = "testuser@microsoft.com", + SDKReleaseMonth = releaseMonth ?? "06/2026", + ReleasePlanId = 50001, + IsDataPlane = true, + SpecType = "TypeSpec", + ActiveSpecPullRequest = "https://github.com/Azure/azure-rest-api-specs/pull/12345", + APISpecProjectPath = typespecPath ?? "specification/contosowidgetmanager/Contoso.WidgetManager", + SDKReleaseType = "beta", + SDKInfo = + [ + new SDKInfo { Language = ".NET", PackageName = "Azure.Template.Contoso" }, + new SDKInfo { Language = "Python", PackageName = "azure-contoso-widgetmanager" } + ] + }; + + public static ReleaseWorkflowResponse Workflow(string status, params string[] details) => new() + { + Language = SdkLanguage.DotNet, + Status = status, + TypeSpecProject = "specification/contosowidgetmanager/Contoso.WidgetManager", + Details = details.ToList() + }; +} + +/// Mock handler for azsdk_abandon_release_plan. +public class AbandonReleasePlanHandler : IMockToolHandler +{ + public string ToolName => "azsdk_abandon_release_plan"; + public CommandResponse Handle(Dictionary? arguments) => + ReleasePlanMockResponses.Workflow("Abandoned", "Release plan abandoned (mock)"); +} + +/// Mock handler for azsdk_update_release_plan. +public class UpdateReleasePlanHandler : IMockToolHandler +{ + public string ToolName => "azsdk_update_release_plan"; + public CommandResponse Handle(Dictionary? arguments) => new ReleasePlanResponse + { + TypeSpecProject = "specification/contosowidgetmanager/Contoso.WidgetManager", + PackageType = SdkType.Dataplane, + Message = "Release plan updated successfully (mock)", + ReleasePlanDetails = ReleasePlanMockResponses.ContosoWorkItem() + }; +} + +/// Mock handler for azsdk_get_release_plan_for_spec_pr. +public class GetReleasePlanForSpecPrHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_release_plan_for_spec_pr"; + public CommandResponse Handle(Dictionary? arguments) => new ReleasePlanResponse + { + TypeSpecProject = "specification/contosowidgetmanager/Contoso.WidgetManager", + PackageType = SdkType.Dataplane, + Message = "Release plan found for spec PR (mock)", + ReleasePlanDetails = ReleasePlanMockResponses.ContosoWorkItem() + }; +} + +/// Mock handler for azsdk_check_api_spec_ready_for_sdk. +public class CheckApiSpecReadyForSdkHandler : IMockToolHandler +{ + public string ToolName => "azsdk_check_api_spec_ready_for_sdk"; + public CommandResponse Handle(Dictionary? arguments) => + ReleasePlanMockResponses.Workflow("Ready", "API spec is signed off and ready for SDK generation (mock)"); +} + +/// Mock handler for azsdk_get_kpi_attestation_status. +public class GetKpiAttestationStatusHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_kpi_attestation_status"; + public CommandResponse Handle(Dictionary? arguments) => + ReleasePlanMockResponses.Workflow("Attested", "All required KPIs attested for this release (mock)"); +} + +/// Mock handler for azsdk_get_service_details_by_typespec_path. +public class GetServiceDetailsByTypeSpecPathHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_service_details_by_typespec_path"; + public CommandResponse Handle(Dictionary? arguments) + { + var path = arguments?.GetValueOrDefault("typeSpecProjectPath")?.ToString() + ?? "specification/contosowidgetmanager/Contoso.WidgetManager"; + return ReleasePlanMockResponses.Workflow( + "Found", + $"TypeSpec path: {path}", + "Service: Contoso.WidgetManager", + "ServiceTreeId: 00000000-0000-0000-0000-000000000099"); + } +} + +/// Mock handler for azsdk_update_api_spec_pull_request_in_release_plan. +public class UpdateApiSpecPullRequestInReleasePlanHandler : IMockToolHandler +{ + public string ToolName => "azsdk_update_api_spec_pull_request_in_release_plan"; + public CommandResponse Handle(Dictionary? arguments) => + ReleasePlanMockResponses.Workflow( + "Updated", + "API spec pull request URL updated on release plan (mock)"); +} + +/// Mock handler for azsdk_update_language_exclusion_justification. +public class UpdateLanguageExclusionJustificationHandler : IMockToolHandler +{ + public string ToolName => "azsdk_update_language_exclusion_justification"; + public CommandResponse Handle(Dictionary? arguments) => + ReleasePlanMockResponses.Workflow( + "Updated", + "Language exclusion justification recorded (mock)"); +} + diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/TypeSpec/TypeSpecHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/TypeSpec/TypeSpecHandlers.cs new file mode 100644 index 00000000000..29685abac32 --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/TypeSpec/TypeSpecHandlers.cs @@ -0,0 +1,130 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; +using Azure.Sdk.Tools.Cli.Models.AzureSdkKnowledgeAICompletion; +using Azure.Sdk.Tools.Cli.Models.Responses.Package; +using Azure.Sdk.Tools.Cli.Models.Responses.TypeSpec; + +namespace Azure.Sdk.Tools.Mock.Handlers.TypeSpec; + +internal static class TypeSpecMockResponses +{ + public const string ContosoTypeSpecProject = "specification/contosowidgetmanager/Contoso.WidgetManager"; +} + +/// Mock handler for azsdk_typespec_generate_authoring_plan. +public class TypeSpecGenerateAuthoringPlanHandler : IMockToolHandler +{ + public string ToolName => "azsdk_typespec_generate_authoring_plan"; + public CommandResponse Handle(Dictionary? arguments) => new TypeSpecAuthoringResponse + { + TypeSpecProject = TypeSpecMockResponses.ContosoTypeSpecProject, + Solution = "Add a new operation `GetWidgetById` to the WidgetService and bump the api-version.", + References = + [ + new DocumentReference + { + Title = "TypeSpec authoring guide", + Source = "azure-rest-api-specs-wiki", + Link = "https://aka.ms/typespec-azure-guide" + } + ] + }; +} + +/// Mock handler for azsdk_typespec_init_project. +public class TypeSpecInitProjectHandler : IMockToolHandler +{ + public string ToolName => "azsdk_typespec_init_project"; + public CommandResponse Handle(Dictionary? arguments) => new TspToolResponse + { + TypeSpecProject = TypeSpecMockResponses.ContosoTypeSpecProject, + IsSuccessful = true, + NextSteps = ["Run `tsp compile .` to validate the new project."] + }; +} + +/// Mock handler for azsdk_convert_swagger_to_typespec. +public class ConvertSwaggerToTypeSpecHandler : IMockToolHandler +{ + public string ToolName => "azsdk_convert_swagger_to_typespec"; + public CommandResponse Handle(Dictionary? arguments) => new TspToolResponse + { + TypeSpecProject = TypeSpecMockResponses.ContosoTypeSpecProject, + IsSuccessful = true, + NextSteps = + [ + "Review the converted TypeSpec for any TODOs.", + "Run `tsp compile .` to validate." + ] + }; +} + +/// Mock handler for azsdk_run_typespec_validation. +public class RunTypeSpecValidationHandler : IMockToolHandler +{ + public string ToolName => "azsdk_run_typespec_validation"; + public CommandResponse Handle(Dictionary? arguments) => new TypeSpecValidationResponse + { + TypeSpecProject = TypeSpecMockResponses.ContosoTypeSpecProject, + PackageType = SdkType.Dataplane, + Message = "TypeSpec validation passed (mock).", + validationResults = ["No issues found."] + }; +} + +/// Mock handler for azsdk_get_modified_typespec_projects. +public class GetModifiedTypeSpecProjectsHandler : IMockToolHandler +{ + public string ToolName => "azsdk_get_modified_typespec_projects"; + public CommandResponse Handle(Dictionary? arguments) => new ObjectCommandResponse + { + Message = "Found 1 modified TypeSpec project (mock).", + Result = new[] { TypeSpecMockResponses.ContosoTypeSpecProject } + }; +} + +/// Mock handler for azsdk_typespec_check_project_in_public_repo. +public class TypeSpecCheckProjectInPublicRepoHandler : IMockToolHandler +{ + public string ToolName => "azsdk_typespec_check_project_in_public_repo"; + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "TypeSpec project is present in the public azure-rest-api-specs repository (mock).", + Result = true + }; +} + +/// Mock handler for azsdk_typespec_delegate_apiview_feedback. +public class TypeSpecDelegateApiViewFeedbackHandler : IMockToolHandler +{ + public string ToolName => "azsdk_typespec_delegate_apiview_feedback"; + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "APIView feedback delegated to the TypeSpec author (mock).", + Result = new + { + commentsDelegated = 2, + assignee = "contoso-typespec-author" + } + }; +} + +/// Mock handler for azsdk_customized_code_update. +public class CustomizedCodeUpdateHandler : IMockToolHandler +{ + public string ToolName => "azsdk_customized_code_update"; + public CommandResponse Handle(Dictionary? arguments) => new CustomizedCodeUpdateResponse + { + Success = true, + Message = "Customized code updated and rebuilt successfully (mock).", + AppliedPatches = + [ + new AppliedPatch( + FilePath: "src/Generated/Customization/WidgetClientCustomization.cs", + Description: "Renamed Get to GetWidget", + ReplacementCount: 2) + ] + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Verify/VerifySetupHandler.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Verify/VerifySetupHandler.cs new file mode 100644 index 00000000000..98edcd4d9cd --- /dev/null +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/Verify/VerifySetupHandler.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using Azure.Sdk.Tools.Cli.Models; + +namespace Azure.Sdk.Tools.Mock.Handlers.Verify; + +/// +/// Mock handler for azsdk_verify_setup. Returns an all-good response so downstream tools +/// can proceed without environment prerequisites in mock mode. +/// +public class VerifySetupHandler : IMockToolHandler +{ + public string ToolName => "azsdk_verify_setup"; + + public CommandResponse Handle(Dictionary? arguments) => new VerifySetupResponse + { + Results = + [ + new RequirementCheckResult + { + Requirement = "dotnet", + Instructions = ["dotnet SDK is installed."], + RequirementStatusDetails = "dotnet 8.0.100 detected" + }, + new RequirementCheckResult + { + Requirement = "git", + Instructions = ["git is installed."], + RequirementStatusDetails = "git 2.45.0 detected" + } + ] + }; +} diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md index dae7e8963f4..06d2e438c02 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md @@ -120,7 +120,7 @@ Use this pattern in any handler to test how your integration handles different s The mock reuses the live CLI's tool definitions, so the *set* of advertised tools is always identical. What can drift is which tools have a hand-written `IMockToolHandler`. Tools without a handler fall back to the generic default response — fine for noise, but it hides routing / arg regressions when a scenario actually depends on that tool returning a realistic shape. -Use the inventory script to audit: +Use the inventory script to audit live-vs-mock parity: ```powershell pwsh eng/scripts/Get-McpToolInventory.ps1 @@ -129,7 +129,7 @@ pwsh eng/scripts/Get-McpToolInventory.ps1 It produces three buckets: - **both** — live tool with a hand-written handler. No action. -- **live-only** — live tool that falls back to the default response. Add a handler if any eval depends on it. +- **live-only** — live tool that falls back to the default response. Add a handler. - **mock-only** — handler for a tool that no longer exists on the live server. Rename or delete the stale handler. CI runs the same script with `-CheckOnly`: @@ -138,10 +138,7 @@ CI runs the same script with `-CheckOnly`: pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly ``` -`-CheckOnly` exits non-zero when: - -1. There is a **mock-only** drift (stale handler that no longer maps to a live tool), or -2. A tool referenced by a mock-tier eval (under `tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/`) has no handler. +`-CheckOnly` exits non-zero when either bucket is non-empty. ### Workflow when the script flags a gap From af0a8a32323f0bdd4fe31b86de1fcc7c46919264 Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 11:34:56 -0700 Subject: [PATCH 3/6] Simplify Get-McpToolInventory.ps1: no parameters, always exits non-zero on drift (#15852) --- eng/scripts/Get-McpToolInventory.ps1 | 98 +++++-------------- .../azsdk-cli/Azure.Sdk.Tools.Mock/README.md | 18 ++-- 2 files changed, 31 insertions(+), 85 deletions(-) diff --git a/eng/scripts/Get-McpToolInventory.ps1 b/eng/scripts/Get-McpToolInventory.ps1 index fb5e45c0d40..9a78fe1ecca 100644 --- a/eng/scripts/Get-McpToolInventory.ps1 +++ b/eng/scripts/Get-McpToolInventory.ps1 @@ -1,51 +1,29 @@ <# .SYNOPSIS - Inventory drift report between the live Azure.Sdk.Tools.Cli MCP server and - the Azure.Sdk.Tools.Mock handler set. + Verify the Azure.Sdk.Tools.Mock handler set matches the live + Azure.Sdk.Tools.Cli MCP tool list. .DESCRIPTION - Boots the live MCP server, captures its tool list, then enumerates the - IMockToolHandler implementations in Azure.Sdk.Tools.Mock and emits a diff: + Builds the CLI + Mock projects (incremental), queries the live tool list, + enumerates IMockToolHandler implementations, and prints three buckets: - live-only - tool exists in the live server, no mock handler -> mock - returns the generic default response (potential gap). - mock-only - mock handler exists, tool no longer in live server - (stale handler). - both - tool exists on both sides. + both - tool exists on both sides (no action). + live-only - live tool with no mock handler (add one). + mock-only - mock handler with no live tool (delete or rename). - Also collects the set of MCP tools referenced by mock-tier evals - (tools/azsdk-cli/Azure.Sdk.Tools.Vally/evals/{unit,integration,e2e}/*.eval.yaml) - so -CheckOnly can fail the build only when drift affects something an eval - actually relies on. - -.PARAMETER CheckOnly - Exit non-zero when there is drift on any tool referenced by a mock-tier eval. - Intended for CI: see https://github.com/Azure/azure-sdk-tools/issues/15829. - -.PARAMETER OutputJson - Optional path to write the diff as JSON for downstream tooling. - -.PARAMETER SkipBuild - Skip `dotnet build` for the CLI and Mock projects (assumes they are up to date). + Exits non-zero on any drift. Same command for local dev and CI. .EXAMPLE pwsh eng/scripts/Get-McpToolInventory.ps1 - -.EXAMPLE - pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly #> [CmdletBinding()] -param( - [switch]$CheckOnly, - [string]$OutputJson, - [switch]$SkipBuild -) +param() Set-StrictMode -Version 4 $ErrorActionPreference = 'Stop' -$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') -$cliProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Cli' +$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') +$cliProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Cli' $mockProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Mock' if (-not (Test-Path $cliProject)) { throw "CLI project not found: $cliProject" } @@ -82,10 +60,7 @@ function Get-LiveMcpTools { function Get-MockHandlerToolNames { param([string]$MockProject) - # The README documents the handler contract as - # public string ToolName => "azsdk_xxx"; - # Parsing source is robust and avoids loading the assembly + all its - # dependencies into the PowerShell process. + # Parse `public string ToolName => "azsdk_xxx";` from handler source files. $pattern = [regex]'(?m)ToolName\s*=>\s*"([^"]+)"' $names = @() Get-ChildItem -LiteralPath (Join-Path $MockProject 'Handlers') -Recurse -Filter *.cs | @@ -110,13 +85,11 @@ function Write-Section { } } -if (-not $SkipBuild) { - Invoke-DotnetBuild -Project $cliProject - Invoke-DotnetBuild -Project $mockProject -} +Invoke-DotnetBuild -Project $cliProject +Invoke-DotnetBuild -Project $mockProject -$liveTools = @(Get-LiveMcpTools -CliProject $cliProject) -$mockTools = @(Get-MockHandlerToolNames -MockProject $mockProject) +$liveTools = @(Get-LiveMcpTools -CliProject $cliProject) +$mockTools = @(Get-MockHandlerToolNames -MockProject $mockProject) $liveSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$liveTools, [System.StringComparer]::OrdinalIgnoreCase) $mockSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$mockTools, [System.StringComparer]::OrdinalIgnoreCase) @@ -129,42 +102,21 @@ Write-Host "MCP tool inventory" -ForegroundColor White Write-Host " live tools: $($liveTools.Count)" Write-Host " mock handlers: $($mockTools.Count)" -Write-Section -Title 'both (live + mock handler)' -Items $both -Color Green +Write-Section -Title 'both (live + mock handler)' -Items $both -Color Green Write-Section -Title 'live-only (no mock handler)' -Items $liveOnly -Color Yellow Write-Section -Title 'mock-only (stale handler)' -Items $mockOnly -Color Magenta -if ($OutputJson) { - $payload = [ordered]@{ - liveTools = $liveTools - mockTools = $mockTools - both = $both - liveOnly = $liveOnly - mockOnly = $mockOnly - } - $payload | ConvertTo-Json -Depth 4 | Out-File -LiteralPath $OutputJson -Encoding utf8 - Write-Host "Wrote $OutputJson" -ForegroundColor DarkGray -} - -if ($CheckOnly) { - $fail = $false +if ($liveOnly.Count -gt 0 -or $mockOnly.Count -gt 0) { + Write-Host "" if ($liveOnly.Count -gt 0) { - Write-Host "" - Write-Host "Drift detected: $($liveOnly.Count) live tool(s) have no mock handler." -ForegroundColor Red - Write-Host "Add a handler under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ for each tool above." -ForegroundColor Red - Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red - $fail = $true + Write-Host "Drift: $($liveOnly.Count) live tool(s) have no mock handler. Add one under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/." -ForegroundColor Red } if ($mockOnly.Count -gt 0) { - Write-Host "" - Write-Host "Drift detected: $($mockOnly.Count) mock handler(s) target tools that no longer exist in the live MCP server." -ForegroundColor Red - Write-Host "Either delete the stale handler(s) or rename them to match the new live tool name." -ForegroundColor Red - $fail = $true - } - if ($fail) { - exit 1 + Write-Host "Drift: $($mockOnly.Count) mock handler(s) target tools that no longer exist. Delete or rename them." -ForegroundColor Red } - Write-Host "" - Write-Host "OK - mock handler set matches the live MCP tool list." -ForegroundColor Green + Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red + exit 1 } -return +Write-Host "" +Write-Host "OK - mock handler set matches the live MCP tool list." -ForegroundColor Green diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md index 06d2e438c02..ab7a3499525 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md @@ -120,25 +120,19 @@ Use this pattern in any handler to test how your integration handles different s The mock reuses the live CLI's tool definitions, so the *set* of advertised tools is always identical. What can drift is which tools have a hand-written `IMockToolHandler`. Tools without a handler fall back to the generic default response — fine for noise, but it hides routing / arg regressions when a scenario actually depends on that tool returning a realistic shape. -Use the inventory script to audit live-vs-mock parity: +Run the inventory script to check parity: ```powershell pwsh eng/scripts/Get-McpToolInventory.ps1 ``` -It produces three buckets: +It prints three buckets and exits non-zero on any drift: - **both** — live tool with a hand-written handler. No action. -- **live-only** — live tool that falls back to the default response. Add a handler. -- **mock-only** — handler for a tool that no longer exists on the live server. Rename or delete the stale handler. +- **live-only** — live tool with no handler. Add one. +- **mock-only** — handler for a tool that no longer exists. Delete or rename it. -CI runs the same script with `-CheckOnly`: - -```powershell -pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly -``` - -`-CheckOnly` exits non-zero when either bucket is non-empty. +The same command is used locally and in CI. ### Workflow when the script flags a gap @@ -146,4 +140,4 @@ pwsh eng/scripts/Get-McpToolInventory.ps1 -CheckOnly 2. Add a new file under `Handlers//` (e.g., `Handlers/Pipeline/MyToolHandler.cs`). 3. Implement `IMockToolHandler`. Set `ToolName` to the exact `[McpServerTool(Name = "…")]` value from the real tool. 4. Return an instance of the same response type the real tool returns, populated with realistic sample data. For scenarios that need to exercise multiple branches, switch on `arguments` (see `HelloWorldHandler` above). -5. Re-run the inventory script to confirm the tool moved from **live-only** to **both**. +5. Re-run the script to confirm the tool moved from **live-only** to **both**. From c2e9b1e531fdb0ef76ff1c0b017db842e62f9bb4 Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 12:02:59 -0700 Subject: [PATCH 4/6] Fix 3 release-plan handler response types to match live tools (#15852) Addresses Copilot review on PR #15854: - azsdk_get_kpi_attestation_status: ReleaseWorkflowResponse -> ReleasePlanListResponse - azsdk_get_service_details_by_typespec_path: ReleaseWorkflowResponse -> ProductInfoResponse - azsdk_update_language_exclusion_justification: ReleaseWorkflowResponse -> DefaultCommandResponse --- .../ReleasePlanRemainingHandlers.cs | 37 +++++++++++-------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs index fafdf2de999..b092008b4db 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/ReleasePlan/ReleasePlanRemainingHandlers.cs @@ -4,6 +4,7 @@ using Azure.Sdk.Tools.Cli.Models; using Azure.Sdk.Tools.Cli.Models.AzureDevOps; using Azure.Sdk.Tools.Cli.Models.Responses.ReleasePlan; +using Azure.Sdk.Tools.Cli.Models.Responses.ReleasePlanList; namespace Azure.Sdk.Tools.Mock.Handlers.ReleasePlan; @@ -84,24 +85,30 @@ public CommandResponse Handle(Dictionary? arguments) => public class GetKpiAttestationStatusHandler : IMockToolHandler { public string ToolName => "azsdk_get_kpi_attestation_status"; - public CommandResponse Handle(Dictionary? arguments) => - ReleasePlanMockResponses.Workflow("Attested", "All required KPIs attested for this release (mock)"); + public CommandResponse Handle(Dictionary? arguments) => new ReleasePlanListResponse + { + ReleasePlanDetailsList = [ReleasePlanMockResponses.ContosoWorkItem()], + Message = "All required KPIs attested for this release (mock)." + }; } /// Mock handler for azsdk_get_service_details_by_typespec_path. public class GetServiceDetailsByTypeSpecPathHandler : IMockToolHandler { public string ToolName => "azsdk_get_service_details_by_typespec_path"; - public CommandResponse Handle(Dictionary? arguments) + public CommandResponse Handle(Dictionary? arguments) => new ProductInfoResponse { - var path = arguments?.GetValueOrDefault("typeSpecProjectPath")?.ToString() - ?? "specification/contosowidgetmanager/Contoso.WidgetManager"; - return ReleasePlanMockResponses.Workflow( - "Found", - $"TypeSpec path: {path}", - "Service: Contoso.WidgetManager", - "ServiceTreeId: 00000000-0000-0000-0000-000000000099"); - } + ProductInfo = new ProductInfo + { + ProductServiceTreeId = "00000000-0000-0000-0000-000000000099", + ServiceId = "00000000-0000-0000-0000-000000000042", + PackageDisplayName = "Azure SDK for Contoso WidgetManager", + ProductServiceTreeLink = "https://servicetree.example.com/products/00000000-0000-0000-0000-000000000099", + WorkItemId = 36000, + Title = "Contoso.WidgetManager" + }, + Message = "Product details resolved from TypeSpec path (mock)." + }; } /// Mock handler for azsdk_update_api_spec_pull_request_in_release_plan. @@ -118,9 +125,9 @@ public CommandResponse Handle(Dictionary? arguments) => public class UpdateLanguageExclusionJustificationHandler : IMockToolHandler { public string ToolName => "azsdk_update_language_exclusion_justification"; - public CommandResponse Handle(Dictionary? arguments) => - ReleasePlanMockResponses.Workflow( - "Updated", - "Language exclusion justification recorded (mock)"); + public CommandResponse Handle(Dictionary? arguments) => new DefaultCommandResponse + { + Message = "Updated language exclusion justification in release plan (mock)." + }; } From 07f732d01ae3f202bb3cefc65ff629f3669ada4b Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 12:18:52 -0700 Subject: [PATCH 5/6] Drop Get-McpToolInventory.ps1 (#15852) Per review discussion: the script only checked that an IMockToolHandler exists with the right ToolName; it could not detect handlers that exist but just return the placeholder DefaultCommandResponse. That blind spot makes the script of limited value. A unit test in Cli.Tests is a better fit for actual drift enforcement and is tracked as a follow-up. README updated to drop the script reference. --- eng/scripts/Get-McpToolInventory.ps1 | 122 --------------------------- 1 file changed, 122 deletions(-) delete mode 100644 eng/scripts/Get-McpToolInventory.ps1 diff --git a/eng/scripts/Get-McpToolInventory.ps1 b/eng/scripts/Get-McpToolInventory.ps1 deleted file mode 100644 index 9a78fe1ecca..00000000000 --- a/eng/scripts/Get-McpToolInventory.ps1 +++ /dev/null @@ -1,122 +0,0 @@ -<# -.SYNOPSIS - Verify the Azure.Sdk.Tools.Mock handler set matches the live - Azure.Sdk.Tools.Cli MCP tool list. - -.DESCRIPTION - Builds the CLI + Mock projects (incremental), queries the live tool list, - enumerates IMockToolHandler implementations, and prints three buckets: - - both - tool exists on both sides (no action). - live-only - live tool with no mock handler (add one). - mock-only - mock handler with no live tool (delete or rename). - - Exits non-zero on any drift. Same command for local dev and CI. - -.EXAMPLE - pwsh eng/scripts/Get-McpToolInventory.ps1 -#> -[CmdletBinding()] -param() - -Set-StrictMode -Version 4 -$ErrorActionPreference = 'Stop' - -$repoRoot = Resolve-Path (Join-Path $PSScriptRoot '../..') -$cliProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Cli' -$mockProject = Join-Path $repoRoot 'tools/azsdk-cli/Azure.Sdk.Tools.Mock' - -if (-not (Test-Path $cliProject)) { throw "CLI project not found: $cliProject" } -if (-not (Test-Path $mockProject)) { throw "Mock project not found: $mockProject" } - -function Invoke-DotnetBuild { - param([string]$Project) - Write-Host "Building $Project ..." -ForegroundColor DarkGray - & dotnet build $Project --nologo --verbosity quiet - if ($LASTEXITCODE -ne 0) { - throw "dotnet build failed for $Project (exit $LASTEXITCODE)" - } -} - -function Get-LiveMcpTools { - param([string]$CliProject) - - Write-Host "Querying live MCP tool list via 'azsdk list -o json' ..." -ForegroundColor DarkGray - $json = & dotnet run --project $CliProject --no-build -- list -o json - if ($LASTEXITCODE -ne 0) { - throw "Failed to run 'list -o json' on $CliProject (exit $LASTEXITCODE)" - } - - $parsed = $json | ConvertFrom-Json - $names = @() - foreach ($t in $parsed.Tools) { - if ($t.McpToolName) { - $names += [string]$t.McpToolName - } - } - return $names | Sort-Object -Unique -} - -function Get-MockHandlerToolNames { - param([string]$MockProject) - - # Parse `public string ToolName => "azsdk_xxx";` from handler source files. - $pattern = [regex]'(?m)ToolName\s*=>\s*"([^"]+)"' - $names = @() - Get-ChildItem -LiteralPath (Join-Path $MockProject 'Handlers') -Recurse -Filter *.cs | - ForEach-Object { - $text = Get-Content -LiteralPath $_.FullName -Raw - foreach ($m in $pattern.Matches($text)) { - $names += $m.Groups[1].Value - } - } - return $names | Sort-Object -Unique -} - -function Write-Section { - param([string]$Title, [string[]]$Items, [ConsoleColor]$Color = [ConsoleColor]::Gray) - Write-Host "" - Write-Host "== $Title ($($Items.Count)) ==" -ForegroundColor $Color - if ($Items.Count -eq 0) { - Write-Host " (none)" -ForegroundColor DarkGray - } - else { - $Items | ForEach-Object { Write-Host " $_" -ForegroundColor $Color } - } -} - -Invoke-DotnetBuild -Project $cliProject -Invoke-DotnetBuild -Project $mockProject - -$liveTools = @(Get-LiveMcpTools -CliProject $cliProject) -$mockTools = @(Get-MockHandlerToolNames -MockProject $mockProject) - -$liveSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$liveTools, [System.StringComparer]::OrdinalIgnoreCase) -$mockSet = [System.Collections.Generic.HashSet[string]]::new([string[]]$mockTools, [System.StringComparer]::OrdinalIgnoreCase) - -$liveOnly = @($liveTools | Where-Object { -not $mockSet.Contains($_) }) -$mockOnly = @($mockTools | Where-Object { -not $liveSet.Contains($_) }) -$both = @($liveTools | Where-Object { $mockSet.Contains($_) }) - -Write-Host "MCP tool inventory" -ForegroundColor White -Write-Host " live tools: $($liveTools.Count)" -Write-Host " mock handlers: $($mockTools.Count)" - -Write-Section -Title 'both (live + mock handler)' -Items $both -Color Green -Write-Section -Title 'live-only (no mock handler)' -Items $liveOnly -Color Yellow -Write-Section -Title 'mock-only (stale handler)' -Items $mockOnly -Color Magenta - -if ($liveOnly.Count -gt 0 -or $mockOnly.Count -gt 0) { - Write-Host "" - if ($liveOnly.Count -gt 0) { - Write-Host "Drift: $($liveOnly.Count) live tool(s) have no mock handler. Add one under tools/azsdk-cli/Azure.Sdk.Tools.Mock/Handlers/." -ForegroundColor Red - } - if ($mockOnly.Count -gt 0) { - Write-Host "Drift: $($mockOnly.Count) mock handler(s) target tools that no longer exist. Delete or rename them." -ForegroundColor Red - } - Write-Host "See tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md for the contract." -ForegroundColor Red - exit 1 -} - -Write-Host "" -Write-Host "OK - mock handler set matches the live MCP tool list." -ForegroundColor Green From b5e804338101d7b6b32e60c372141717f9a07caf Mon Sep 17 00:00:00 2001 From: helen229 Date: Wed, 3 Jun 2026 12:19:04 -0700 Subject: [PATCH 6/6] Update Mock README: drop reference to removed inventory script (#15852) --- .../azsdk-cli/Azure.Sdk.Tools.Mock/README.md | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md index ab7a3499525..45e12ba2bf5 100644 --- a/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md +++ b/tools/azsdk-cli/Azure.Sdk.Tools.Mock/README.md @@ -118,26 +118,11 @@ Use this pattern in any handler to test how your integration handles different s ## Keeping the Mock in Sync with the Live MCP Server -The mock reuses the live CLI's tool definitions, so the *set* of advertised tools is always identical. What can drift is which tools have a hand-written `IMockToolHandler`. Tools without a handler fall back to the generic default response — fine for noise, but it hides routing / arg regressions when a scenario actually depends on that tool returning a realistic shape. +The mock reuses the live CLI's tool definitions (`SharedOptions.ToolsList`), so the *set* of advertised tools is always identical. What can drift is which tools have a hand-written `IMockToolHandler`. Tools without a handler fall back to the generic `{"message":"Success"}` default — fine for routing tests but useless for scenarios that chain calls together (e.g. consume an ID returned by a previous tool). -Run the inventory script to check parity: +When you add or rename an MCP tool in `Azure.Sdk.Tools.Cli`, add a matching handler under `Handlers//`: -```powershell -pwsh eng/scripts/Get-McpToolInventory.ps1 -``` - -It prints three buckets and exits non-zero on any drift: - -- **both** — live tool with a hand-written handler. No action. -- **live-only** — live tool with no handler. Add one. -- **mock-only** — handler for a tool that no longer exists. Delete or rename it. - -The same command is used locally and in CI. - -### Workflow when the script flags a gap - -1. Look up the live tool's response type. Tool method signatures live under `tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/`. The return type is usually a typed `CommandResponse` in `Azure.Sdk.Tools.Cli.Models.Responses.*`. -2. Add a new file under `Handlers//` (e.g., `Handlers/Pipeline/MyToolHandler.cs`). +1. Look up the live tool's response type under `tools/azsdk-cli/Azure.Sdk.Tools.Cli/Tools/`. The return type is usually a typed `CommandResponse` in `Azure.Sdk.Tools.Cli.Models.Responses.*`. +2. Create a new file under `Handlers//` (e.g., `Handlers/Pipeline/MyToolHandler.cs`). 3. Implement `IMockToolHandler`. Set `ToolName` to the exact `[McpServerTool(Name = "…")]` value from the real tool. 4. Return an instance of the same response type the real tool returns, populated with realistic sample data. For scenarios that need to exercise multiple branches, switch on `arguments` (see `HelloWorldHandler` above). -5. Re-run the script to confirm the tool moved from **live-only** to **both**.