diff --git a/cli/azd/cmd/middleware/error.go b/cli/azd/cmd/middleware/error.go index 3afe57ffff7..f2c723dd9a7 100644 --- a/cli/azd/cmd/middleware/error.go +++ b/cli/azd/cmd/middleware/error.go @@ -29,6 +29,7 @@ import ( "github.com/azure/azure-dev/cli/azd/pkg/environment/azdcontext" "github.com/azure/azure-dev/cli/azd/pkg/errorhandler" "github.com/azure/azure-dev/cli/azd/pkg/extensions" + "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning/bicep" "github.com/azure/azure-dev/cli/azd/pkg/input" "github.com/azure/azure-dev/cli/azd/pkg/output" @@ -85,6 +86,11 @@ func shouldSkipAgentHandling(err error) bool { errors.Is(err, consent.ErrElicitationDenied) || errors.Is(err, consent.ErrSamplingDenied) || errors.Is(err, internal.ErrAbortedByUser) || + errors.Is(err, provisioning.ErrDeploymentInterruptedLeaveRunning) || + errors.Is(err, provisioning.ErrDeploymentCanceledByUser) || + errors.Is(err, provisioning.ErrDeploymentCancelTimeout) || + errors.Is(err, provisioning.ErrDeploymentCancelTooLate) || + errors.Is(err, provisioning.ErrDeploymentCancelFailed) || errors.Is(err, environment.ErrNotFound) || errors.Is(err, environment.ErrNameNotSpecified) || diff --git a/cli/azd/docs/provision-cancellation.md b/cli/azd/docs/provision-cancellation.md new file mode 100644 index 00000000000..9875cf9d1fb --- /dev/null +++ b/cli/azd/docs/provision-cancellation.md @@ -0,0 +1,82 @@ +# Provision cancellation (Ctrl+C) + +When `azd provision` (or `azd up`) submits a Bicep deployment to Azure, the +deployment runs asynchronously on the Azure side. If the user presses +Ctrl+C while azd is waiting for that deployment to +finish, azd will pause and ask what to do instead of exiting immediately. + +## Behavior + +1. azd stops the live progress reporter and presents an interactive prompt + that includes the Azure portal URL of the running deployment. +2. The user picks one of: + - **Leave the Azure deployment running and stop azd** (default). azd + exits with a non-zero status; the Azure deployment continues to + completion. The user can monitor or cancel it from the portal link. + - **Cancel the Azure deployment**. azd submits an ARM cancel request + against the deployment and waits up to **5 minutes** (a single global + budget) for Azure to confirm a terminal state (`Canceled`, `Failed`, + or `Succeeded`). Once the top-level deployment reaches `Canceled`, + azd best-effort cancels and waits for any descendant (nested) + deployments within the same 5-minute budget so leftover children do + not keep running on Azure. +3. Additional Ctrl+C presses while the prompt is + showing (or while a cancel request is in flight) are ignored so the user + can finish reading and choose deliberately. + +## Outcomes when "Cancel" is selected + +| Outcome | When | +|---------|------| +| Cancellation confirmed | Azure transitions the deployment to `Canceled` within the wait budget. azd exits non-zero with a clear message. | +| Cancel raced succeeded | Azure reached `Succeeded` before cancel took effect. azd surfaces this as a success-toned message — resources *are* deployed. | +| Cancel raced failed | Azure reached `Failed` before cancel took effect. azd surfaces the failure plus the portal URL. | +| Cancel raced deleted | The deployment record was deleted before cancel took effect (unusual; suggests an external actor). | +| Cancel still pending | Azure does not reach a terminal state within the wait budget. azd warns that cancellation may still complete and prints the portal URL. | +| Cancel request failed | The ARM `Cancel` API itself returned an error. azd surfaces that the deployment is still running and prints the portal URL. | + +When the deployment URL is available, azd prints it so the user can follow +up manually from the browser. The URL is omitted if azd was unable to +resolve it (for example, when the ARM service is unreachable). + +## Provider scope + +| Provider | Behavior on Ctrl+C during provision | +|---------|--------------------------------------| +| Bicep (subscription scope) | Interactive prompt (described above). | +| Bicep (resource group scope) | Interactive prompt (described above). | +| Deployment Stacks | Currently treated as "leave running" — the stacks ARM API does not expose a per-deployment cancel surface today. | +| Terraform | Unchanged: the Terraform CLI does not expose a safe per-apply cancel; pressing Ctrl+C exits azd and Terraform handles its own teardown. | + +## Telemetry + +A `provision.cancellation` attribute is recorded on the provisioning span +with one of: + +- `none` — provisioning completed normally without an interrupt. +- `leave_running` — user chose to let the Azure deployment continue. +- `canceled` — cancel request succeeded and Azure reached `Canceled`. +- `cancel_raced_succeeded` — Azure reached `Succeeded` before cancel took + effect (resources are deployed; we surface this as a success-toned message + rather than a failure). +- `cancel_raced_failed` — Azure reached `Failed` before cancel took effect. +- `cancel_raced_deleted` — the deployment record was deleted before cancel + could take effect (unusual; suggests an external actor). +- `cancel_too_late` — fallback for unexpected terminal states (kept for + backwards-compat; the three `cancel_raced_*` values cover the documented + terminal states). +- `cancel_timed_out` — top-level deployment did not reach a terminal state + within the wait budget (5 minutes). +- `cancel_timed_out_nested` — top-level reached `Canceled`, but one or + more descendant (nested) deployments did not reach a terminal state + within the same 5-minute global budget. The user-facing output lists + the stuck deployment(s) with portal links so they can be investigated + manually. +- `cancel_failed` — the ARM `Cancel` API call itself returned an error. + +## Non-interactive mode + +If azd is running without a TTY (e.g. CI), the prompt cannot be displayed. +In that case azd defaults to **leave running** behavior so that an +unattended deployment is never silently cancelled by an environment +signal. diff --git a/cli/azd/internal/cmd/errors.go b/cli/azd/internal/cmd/errors.go index 32fc26169c2..35c3391f405 100644 --- a/cli/azd/internal/cmd/errors.go +++ b/cli/azd/internal/cmd/errors.go @@ -284,8 +284,20 @@ func classifySentinel(err error) string { return "internal.not_git_repo" case errors.Is(err, azapi.ErrPreviewNotSupported): return "internal.preview_not_supported" + case errors.Is(err, azapi.ErrCancelNotSupported): + return "internal.cancel_not_supported" case errors.Is(err, provisioning.ErrBindMountOperationDisabled): return "internal.bind_mount_disabled" + case errors.Is(err, provisioning.ErrDeploymentInterruptedLeaveRunning): + return "user.canceled.leave_running" + case errors.Is(err, provisioning.ErrDeploymentCanceledByUser): + return "user.canceled.deployment_canceled" + case errors.Is(err, provisioning.ErrDeploymentCancelTimeout): + return "user.canceled.cancel_timed_out" + case errors.Is(err, provisioning.ErrDeploymentCancelTooLate): + return "user.canceled.cancel_too_late" + case errors.Is(err, provisioning.ErrDeploymentCancelFailed): + return "user.canceled.cancel_failed" case errors.Is(err, update.ErrNeedsElevation): return "update.elevationRequired" case errors.Is(err, pipeline.ErrRemoteHostIsNotAzDo): diff --git a/cli/azd/internal/tracing/fields/fields.go b/cli/azd/internal/tracing/fields/fields.go index f3817711ee1..a8d503d5e4c 100644 --- a/cli/azd/internal/tracing/fields/fields.go +++ b/cli/azd/internal/tracing/fields/fields.go @@ -430,6 +430,30 @@ var ( } ) +// Provision-related fields +var ( + // ProvisionCancellationKey records how a Ctrl+C interrupt during + // `azd provision` / `azd up` was handled. + // + // Example: "none" (no interrupt observed), "leave_running" (user chose to + // keep the Azure deployment running), "canceled" (Azure confirmed the + // deployment reached the Canceled state), "cancel_timed_out" (cancel was + // submitted but azd stopped waiting for the top-level terminal state), + // "cancel_timed_out_nested" (top-level was canceled, but one or more + // descendant deployments did not reach terminal state within the global + // budget), "cancel_raced_succeeded" / "cancel_raced_failed" / + // "cancel_raced_deleted" (Azure reached the corresponding terminal state + // before the cancel took effect — split from the legacy "cancel_too_late" + // so dashboards can answer "how often does cancel race a *successful* + // deployment?"), "cancel_too_late" (fallback for unexpected terminal + // states), "cancel_failed" (the cancel request itself returned an error). + ProvisionCancellationKey = AttributeKey{ + Key: attribute.Key("provision.cancellation"), + Classification: SystemMetadata, + Purpose: FeatureInsight, + } +) + // The value used for ServiceNameKey const ServiceNameAzd = "azd" diff --git a/cli/azd/pkg/azapi/deployments.go b/cli/azd/pkg/azapi/deployments.go index 1e079370a4c..b7650c4c8bc 100644 --- a/cli/azd/pkg/azapi/deployments.go +++ b/cli/azd/pkg/azapi/deployments.go @@ -32,6 +32,11 @@ const ( var ErrPreviewNotSupported = errors.New("preview not supported") +// ErrCancelNotSupported indicates that the deployment provider does not support +// cancelling an in-flight deployment (e.g. deployment stacks). Callers can use +// errors.Is to detect this case and fall back to "leave running" behavior. +var ErrCancelNotSupported = errors.New("cancel not supported for this deployment kind") + const emptySubscriptionArmTemplate = `{ "$schema": "https://schema.management.azure.com/schemas/2018-05-01/subscriptionDeploymentTemplate.json#", "contentVersion": "1.0.0.0", @@ -226,6 +231,25 @@ type DeploymentService interface { options map[string]any, progress *async.Progress[DeleteDeploymentProgress], ) error + // CancelSubscriptionDeployment requests Azure to cancel a running + // subscription-scoped deployment. The call returns immediately after the + // cancel request is accepted; callers should poll the deployment to observe + // the terminal state (Canceled, Failed, or Succeeded). + CancelSubscriptionDeployment( + ctx context.Context, + subscriptionId string, + deploymentName string, + ) error + // CancelResourceGroupDeployment requests Azure to cancel a running + // resource-group-scoped deployment. The call returns immediately after the + // cancel request is accepted; callers should poll the deployment to observe + // the terminal state (Canceled, Failed, or Succeeded). + CancelResourceGroupDeployment( + ctx context.Context, + subscriptionId string, + resourceGroupName string, + deploymentName string, + ) error } type DeleteResourceState string diff --git a/cli/azd/pkg/azapi/stack_deployments.go b/cli/azd/pkg/azapi/stack_deployments.go index 7a7b0ba71ec..563e505dc75 100644 --- a/cli/azd/pkg/azapi/stack_deployments.go +++ b/cli/azd/pkg/azapi/stack_deployments.go @@ -660,6 +660,29 @@ func (d *StackDeployments) CalculateTemplateHash( return d.standardDeployments.CalculateTemplateHash(ctx, subscriptionId, template) } +// CancelSubscriptionDeployment is not supported for deployment stacks. The +// deployment stacks ARM API does not expose a per-stack cancel operation; +// stopping a stack mid-deployment requires deleting the stack itself. Returns +// ErrCancelNotSupported so callers can distinguish this from a real failure. +func (d *StackDeployments) CancelSubscriptionDeployment( + ctx context.Context, + subscriptionId string, + deploymentName string, +) error { + return ErrCancelNotSupported +} + +// CancelResourceGroupDeployment is not supported for deployment stacks. See +// CancelSubscriptionDeployment for details. +func (d *StackDeployments) CancelResourceGroupDeployment( + ctx context.Context, + subscriptionId string, + resourceGroupName string, + deploymentName string, +) error { + return ErrCancelNotSupported +} + func (d *StackDeployments) createClient(ctx context.Context, subscriptionId string) (*armdeploymentstacks.Client, error) { credential, err := d.credentialProvider.CredentialForSubscription(ctx, subscriptionId) if err != nil { diff --git a/cli/azd/pkg/azapi/standard_deployments.go b/cli/azd/pkg/azapi/standard_deployments.go index efc55ed44cb..8e08df4b181 100644 --- a/cli/azd/pkg/azapi/standard_deployments.go +++ b/cli/azd/pkg/azapi/standard_deployments.go @@ -551,6 +551,47 @@ func (ds *StandardDeployments) DeleteResourceGroupDeployment( return nil } +// CancelSubscriptionDeployment requests Azure to cancel a running +// subscription-scoped deployment. The ARM Cancel call returns immediately once +// the request is accepted; callers should poll the deployment to observe the +// terminal state (Canceled, Failed, or Succeeded). +func (ds *StandardDeployments) CancelSubscriptionDeployment( + ctx context.Context, + subscriptionId string, + deploymentName string, +) error { + deploymentClient, err := ds.createDeploymentsClient(ctx, subscriptionId) + if err != nil { + return fmt.Errorf("creating deployments client: %w", err) + } + + if _, err := deploymentClient.CancelAtSubscriptionScope(ctx, deploymentName, nil); err != nil { + return fmt.Errorf("cancelling subscription deployment: %w", err) + } + return nil +} + +// CancelResourceGroupDeployment requests Azure to cancel a running +// resource-group-scoped deployment. The ARM Cancel call returns immediately +// once the request is accepted; callers should poll the deployment to observe +// the terminal state (Canceled, Failed, or Succeeded). +func (ds *StandardDeployments) CancelResourceGroupDeployment( + ctx context.Context, + subscriptionId string, + resourceGroupName string, + deploymentName string, +) error { + deploymentClient, err := ds.createDeploymentsClient(ctx, subscriptionId) + if err != nil { + return fmt.Errorf("creating deployments client: %w", err) + } + + if _, err := deploymentClient.Cancel(ctx, resourceGroupName, deploymentName, nil); err != nil { + return fmt.Errorf("cancelling resource group deployment: %w", err) + } + return nil +} + func (ds *StandardDeployments) WhatIfDeployToSubscription( ctx context.Context, subscriptionId string, diff --git a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go index d49d347b050..d40751bb2c4 100644 --- a/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go +++ b/cli/azd/pkg/infra/provisioning/bicep/bicep_provider.go @@ -767,18 +767,46 @@ func (p *BicepProvider) Deploy(ctx context.Context) (*provisioning.DeployResult, // Start the deployment p.console.ShowSpinner(ctx, "Creating/Updating resources", input.Step) + deployCtx, interruptStarted, interruptCh, markDeployCompleted, interruptCleanup := + p.installDeploymentInterruptHandler(ctx, deployment, cancelProgress) + cleanupOnce := sync.OnceFunc(interruptCleanup) + defer cleanupOnce() + deployResult, err := p.deployModule( - ctx, + deployCtx, deployment, planned.RawArmTemplate, planned.Parameters, deploymentTags, optionsMap, ) + + // Try to atomically claim the "completed" state. If the interrupt + // handler already claimed "interrupting", the CAS fails and we must + // wait for the handler's outcome so the user's Ctrl+C is never + // silently dropped. + if !markDeployCompleted() { + // Handler has claimed the interrupt — wait for its outcome. + <-interruptStarted + outcome := <-interruptCh + cleanupOnce() + tracing.SetUsageAttributes( + fields.ProvisionCancellationKey.String(outcome.telemetryValue)) + return nil, applyInterruptOutcome(outcome, err) + } + + // Deploy completed naturally — tear the handler down before + // post-processing to avoid resurfacing the cancel/leave prompt over + // subsequent output. + cleanupOnce() + if err != nil { + tracing.SetUsageAttributes(fields.ProvisionCancellationKey.String("none")) return nil, err } + tracing.SetUsageAttributes(fields.ProvisionCancellationKey.String("none")) + result.Outputs = provisioning.OutputParametersFromArmOutputs( planned.Template.Outputs, azapi.CreateDeploymentOutput(deployResult.Outputs), diff --git a/cli/azd/pkg/infra/provisioning/bicep/interrupt.go b/cli/azd/pkg/infra/provisioning/bicep/interrupt.go new file mode 100644 index 00000000000..712eec3d85d --- /dev/null +++ b/cli/azd/pkg/infra/provisioning/bicep/interrupt.go @@ -0,0 +1,845 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package bicep + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "sync/atomic" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/azure/azure-dev/cli/azd/pkg/azapi" + "github.com/azure/azure-dev/cli/azd/pkg/infra" + "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" + "github.com/azure/azure-dev/cli/azd/pkg/input" + "github.com/azure/azure-dev/cli/azd/pkg/output" +) + +// Default timeouts for interrupt-driven cancellation. They are package-level +// vars (not consts) so tests can override them to keep poll/timeout-driven +// flows fast. +var ( + // cancelRequestTimeout bounds the time spent waiting for the ARM Cancel + // API call itself to return. + cancelRequestTimeout = 30 * time.Second + // cancelOverallTimeout is the global budget covering EVERYTHING that + // happens after the cancel request is accepted: waiting for the + // top-level deployment to reach a terminal state, discovering and + // canceling any descendant deployments, and waiting for those to also + // reach a terminal state. The user will see at most ~this much time + // between selecting "Cancel" and azd reporting an outcome. + cancelOverallTimeout = 5 * time.Minute + // cancelPollInterval controls how often we poll the deployment for state + // changes after submitting cancel. + cancelPollInterval = 5 * time.Second +) + +// cancelNestedConcurrency caps the number of concurrent ARM API calls when +// canceling/polling descendant deployments to avoid throttling on large trees. +const cancelNestedConcurrency = 5 + +// User-facing labels for the interrupt prompt. Kept as constants so tests can +// reason about the prompt selection without depending on copy edits. +const ( + interruptOptionCancel = "Cancel the Azure deployment" + interruptOptionLeaveRunning = "Leave the Azure deployment running and stop azd" +) + +// interruptOutcome is produced by the interrupt handler and consumed by the +// main deploy goroutine after the ARM operation unblocks. +type interruptOutcome struct { + // err is the typed sentinel error from pkg/infra/provisioning that + // describes how the interrupt was handled. + err error + // telemetryValue is the value to record on the cancellation telemetry + // attribute (see fields.ProvisionCancellationKey). + telemetryValue string +} + +// deployState tracks the lifecycle of the deployment so the interrupt handler +// and the Deploy goroutine can coordinate without races. +type deployState int32 + +const ( + deployStateRunning deployState = iota // ARM deploy is in flight + deployStateInterrupting // handler claimed the Ctrl+C + deployStateCompleted // Deploy returned naturally +) + +// installDeploymentInterruptHandler registers a Ctrl+C handler covering the +// in-flight ARM deployment. It returns: +// +// - deployCtx: a context derived from ctx that the caller MUST pass to the +// ARM deploy call; it will be cancelled as soon as the user presses +// Ctrl+C, which unblocks PollUntilDone and returns control to Deploy. +// - startedCh: closed as soon as the user presses Ctrl+C (before the prompt +// is shown). Callers should check it after the deploy call returns to +// decide whether to block-wait for an interrupt outcome instead of taking +// the normal success path. This is what guarantees that a Ctrl+C arriving +// while the deployment happens to finish naturally cannot be silently +// dropped. +// - outcomeCh: receives the interrupt outcome once the user has chosen. +// The channel is buffered (size 1). +// - markCompleted: must be called by Deploy right after deployModule returns +// (before the select on startedCh) to atomically claim the "completed" +// state. If the interrupt handler already claimed "interrupting", this +// returns false and the caller must wait for the outcome. +// - cleanup: must be called (via defer) to unregister the interrupt handler +// and release the deploy context. +// +// onInterruptStart, if non-nil, is invoked synchronously at the start of the +// interrupt handler before any prompt is shown. Callers use this hook to stop +// background activity (e.g. the deployment progress reporter) so it doesn't +// stomp on the prompt rendering. +func (p *BicepProvider) installDeploymentInterruptHandler( + ctx context.Context, + deployment infra.Deployment, + onInterruptStart func(), +) ( + deployCtx context.Context, + startedCh <-chan struct{}, + outcomeCh <-chan interruptOutcome, + markCompleted func() bool, + cleanup func(), +) { + deployCtx, cancelDeploy := context.WithCancel(ctx) + ch := make(chan interruptOutcome, 1) + started := make(chan struct{}) + + var state atomic.Int32 // deployState values + + pop := input.PushInterruptHandler(sync.OnceValue(func() bool { + // Try to claim the "interrupting" state. If Deploy already set + // "completed", the prompt is unnecessary — the deployment finished + // naturally and the success path should run instead. + if !state.CompareAndSwap( + int32(deployStateRunning), + int32(deployStateInterrupting), + ) { + return false + } + + // Signal interrupt-in-progress and unblock the ARM deploy call + // immediately so Deploy can transition to "wait for outcome" mode + // rather than racing against a natural completion. + close(started) + cancelDeploy() + + if onInterruptStart != nil { + onInterruptStart() + } + // Stop the in-progress spinner so we can render the prompt cleanly. + p.console.StopSpinner(ctx, "", input.Step) + + outcome := p.runInterruptPrompt(ctx, deployment) + ch <- outcome + // Returning true tells the runtime that we own the shutdown sequence. + // We don't actually os.Exit here — Deploy will return the typed + // sentinel error and the action / error middleware translates that + // into the user-facing exit message. + return true + })) + + markCompleted = func() bool { + return state.CompareAndSwap( + int32(deployStateRunning), + int32(deployStateCompleted), + ) + } + + cleanup = func() { + pop() + cancelDeploy() + } + return deployCtx, started, ch, markCompleted, cleanup +} + +// printLeaveRunningMessage emits the standard "Azure deployment will continue +// running" message with a clickable portal link and a hint for cancelling +// later from the CLI. Always emits at least one line so non-interactive runs +// (CI logs) record a breadcrumb to the abandoned deployment. +func (p *BicepProvider) printLeaveRunningMessage( + ctx context.Context, + deployment infra.Deployment, + portalUrl string, +) { + if portalUrl != "" { + p.console.Message(ctx, + output.WithHighLightFormat("The Azure deployment will continue running. Track it here:\n %s", + output.WithLinkFormat(portalUrl))) + } else { + // Fallback breadcrumb when the portal URL fetch failed — the user + // still needs *some* pointer to find the running deployment. + p.console.Message(ctx, + output.WithHighLightFormat( + "The Azure deployment will continue running. "+ + "Find it in the Azure Portal under "+ + "Subscription → Deployments (look for %q).", + deployment.Name())) + } + // Hint for cancelling from another shell — works for both subscription + // and resource-group scopes (the CLI sub-command differs but the hint + // is short enough to mention both). + p.console.Message(ctx, + output.WithGrayFormat( + "To cancel later: az deployment sub cancel --name %s "+ + "(or `az deployment group cancel --name %s --resource-group `)", + deployment.Name(), deployment.Name())) +} + +// runInterruptPrompt presents the user with the choice of cancelling the +// running Azure deployment or leaving it to run. It returns the outcome that +// should be propagated back to Deploy. +func (p *BicepProvider) runInterruptPrompt( + ctx context.Context, + deployment infra.Deployment, +) interruptOutcome { + // Best-effort URL fetch — bounded so a slow/unreachable ARM endpoint + // doesn't block the prompt indefinitely. + urlCtx, urlDone := context.WithTimeout( + context.WithoutCancel(ctx), cancelRequestTimeout) + portalUrl, urlErr := deployment.DeploymentUrl(urlCtx) + urlDone() + if urlErr != nil { + // Not fatal — we just won't include the URL in the prompt. + log.Printf("interrupt handler: failed to fetch deployment URL: %v", urlErr) + } + + help := "An Azure deployment is currently in progress." + if portalUrl != "" { + help = fmt.Sprintf("%s\nPortal: %s", help, portalUrl) + } else { + help = fmt.Sprintf("%s\nFind it in the Azure Portal under "+ + "Subscription → Deployments (look for %q).", help, deployment.Name()) + } + + choice, err := p.console.Select(ctx, input.ConsoleOptions{ + Message: "You pressed Ctrl+C. An Azure deployment is still running — what do you want to do?", + Help: help, + Options: []string{ + interruptOptionLeaveRunning, + interruptOptionCancel, + }, + DefaultValue: interruptOptionLeaveRunning, + }) + if err != nil { + // If we can't even show the prompt (e.g. non-interactive), fall back + // to the safer "leave running" behavior so the user can decide + // manually via the portal. We still log a breadcrumb to stderr so + // CI runs (the most common non-interactive case) record an explicit + // trace of the abandoned deployment instead of silently leaking it. + log.Printf("interrupt handler: failed to show prompt, defaulting to leave-running: %v", err) + p.printLeaveRunningMessage(ctx, deployment, portalUrl) + return interruptOutcome{ + err: provisioning.ErrDeploymentInterruptedLeaveRunning, + telemetryValue: "leave_running", + } + } + + switch choice { + case 0: // leave running + p.printLeaveRunningMessage(ctx, deployment, portalUrl) + return interruptOutcome{ + err: provisioning.ErrDeploymentInterruptedLeaveRunning, + telemetryValue: "leave_running", + } + case 1: // cancel + return p.cancelAndAwaitTerminal(ctx, deployment, portalUrl) + default: + // Should never happen, but fall back to leave-running. + return interruptOutcome{ + err: provisioning.ErrDeploymentInterruptedLeaveRunning, + telemetryValue: "leave_running", + } + } +} + +// cancelAndAwaitTerminal submits the Azure cancel request and polls the +// deployment until it reaches a terminal provisioning state (Canceled, Failed, +// or Succeeded) or the wait budget is exhausted. +func (p *BicepProvider) cancelAndAwaitTerminal( + ctx context.Context, + deployment infra.Deployment, + portalUrl string, +) interruptOutcome { + p.console.ShowSpinner(ctx, "Canceling Azure deployment", input.Step) + + // Use a fresh context for the cancel API call so it isn't affected by + // the deploy-side cancellation we issue right after. + cancelReqCtx, cancelReqDone := context.WithTimeout( + context.WithoutCancel(ctx), cancelRequestTimeout) + defer cancelReqDone() + + if err := deployment.Cancel(cancelReqCtx); err != nil { + // Some providers (e.g. Deployment Stacks) do not support per-deployment + // cancel. Surface that as the safer "leave running" outcome rather + // than a cancel failure so the user gets consistent UX/telemetry with + // the documented provider behavior. + if errors.Is(err, azapi.ErrCancelNotSupported) { + p.console.StopSpinner(ctx, "Cancel is not supported for this deployment kind", input.StepWarning) + p.printLeaveRunningMessage(ctx, deployment, portalUrl) + return interruptOutcome{ + err: provisioning.ErrDeploymentInterruptedLeaveRunning, + telemetryValue: "leave_running", + } + } + // If the deployment is already in a terminal state, route through + // the same terminal-outcome reporter so the user sees consistent + // messaging (including the portal URL). + getCtx, getDone := context.WithTimeout( + context.WithoutCancel(ctx), cancelRequestTimeout) + defer getDone() + + if state, getErr := deployment.Get(getCtx); getErr == nil && + isTerminalProvisioningState(state.ProvisioningState) { + return p.terminalToOutcome(ctx, state.ProvisioningState, portalUrl) + } else if getErr != nil { + // Don't drop this — it's useful for diagnosing the cancel-failed + // path in production logs (the user-facing error is still the + // original cancel failure). + log.Printf("interrupt handler: post-cancel Get failed: %v", getErr) + } + p.console.StopSpinner(ctx, + "Couldn't cancel — Azure deployment is still running", input.StepFailed) + log.Printf("interrupt handler: cancel request failed: %v", err) + if portalUrl != "" { + p.console.Message(ctx, + output.WithWarningFormat( + "The cancel request was rejected by Azure. The deployment will continue. Track it here:\n %s", + output.WithLinkFormat(portalUrl))) + } else { + p.console.Message(ctx, + output.WithWarningFormat( + "The cancel request was rejected by Azure. The deployment will continue. "+ + "Find it in the Azure Portal under Subscription → Deployments (look for %q).", + deployment.Name())) + } + return interruptOutcome{ + err: fmt.Errorf("%w: %w", + provisioning.ErrDeploymentCancelFailed, err), + telemetryValue: "cancel_failed", + } + } + + p.console.StopSpinner(ctx, "", input.Step) + p.console.ShowSpinner(ctx, "Waiting for Azure to confirm cancellation", input.Step) + + // Single global deadline covering BOTH the top-level wait and any + // descendant-deployment cleanup. The user shouldn't wait more than + // cancelOverallTimeout total between pressing Ctrl+C and seeing an + // outcome reported by azd. + pollCtx, pollDone := context.WithTimeout( + context.WithoutCancel(ctx), cancelOverallTimeout) + defer pollDone() + + state, timedOut := p.awaitTopLevelTerminal(pollCtx, deployment) + if timedOut { + p.console.StopSpinner(ctx, + "Azure is still canceling — azd will exit", input.StepWarning) + if portalUrl != "" { + p.console.Message(ctx, + output.WithWarningFormat( + "Azure didn't confirm cancellation within 5 minutes. "+ + "Cancellation may still complete. Track:\n %s", + output.WithLinkFormat(portalUrl))) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTimeout, + telemetryValue: "cancel_timed_out", + } + } + + // When the cancel actually took effect on the top-level deployment, also + // wait for any descendant deployments to reach a terminal state. ARM's + // cancel cascade is asynchronous, and a child deployment can still hold + // an active deployment lease for several minutes after the parent reports + // "Canceled" — which would cause the next `azd provision` to fail with + // a `DeploymentActive` validation error. We skip this for Succeeded / + // Failed / Deleted because in those cases the children should already be + // terminal (a parent only reports Succeeded once its children are done, + // and Failed/Deleted reflect a settled state). + if state == azapi.DeploymentProvisioningStateCanceled { + stuck := p.cancelAndAwaitNested(pollCtx, deployment) + if len(stuck) > 0 { + return p.nestedTimeoutOutcome(ctx, stuck, portalUrl) + } + } + + return p.terminalToOutcome(ctx, state, portalUrl) +} + +// awaitTopLevelTerminal polls the top-level deployment until it reaches a +// terminal provisioning state or the supplied context is canceled (e.g. by +// the global cancelOverallTimeout). The first Get is issued immediately so +// the fast path doesn't pay a poll-interval penalty; subsequent polls are +// ticker-spaced. Returns (state, timedOut). When timedOut is true, state is +// the zero value and the caller should report the timeout outcome. +func (p *BicepProvider) awaitTopLevelTerminal( + pollCtx context.Context, + deployment infra.Deployment, +) (azapi.DeploymentProvisioningState, bool) { + // Issue the first Get immediately after the cancel request was accepted + // — Azure can transition to a terminal state very quickly for deployments + // that were just starting, and we don't want to make the user wait a full + // poll interval for that fast path. + if state, err := deployment.Get(pollCtx); err == nil { + if isTerminalProvisioningState(state.ProvisioningState) { + return state.ProvisioningState, false + } + } else { + log.Printf("interrupt handler: initial poll Get failed (will retry): %v", err) + } + + // Subsequent polls are ticker-driven so a slow Get cannot push the loop + // into back-to-back ARM calls (and trigger throttling). + ticker := time.NewTicker(cancelPollInterval) + defer ticker.Stop() + + for { + select { + case <-pollCtx.Done(): + return "", true + case <-ticker.C: + } + + state, err := deployment.Get(pollCtx) + if err == nil { + if isTerminalProvisioningState(state.ProvisioningState) { + return state.ProvisioningState, false + } + } else { + // Don't fail the whole flow on a transient Get error — keep + // polling until either we observe a terminal state or the + // timeout fires. + log.Printf("interrupt handler: poll Get failed (will retry): %v", err) + } + } +} + +// terminalToOutcome maps a terminal provisioning state to the interrupt outcome +// that should be propagated back to Deploy. Copy and telemetry value are both +// branched per state so users can tell apart "we canceled it" vs "Azure +// finished it before we could cancel" vs "the deployment record was deleted", +// and so dashboards can answer "how often does cancel race a *successful* +// deployment?". +func (p *BicepProvider) terminalToOutcome( + ctx context.Context, + state azapi.DeploymentProvisioningState, + portalUrl string, +) interruptOutcome { + switch state { + case azapi.DeploymentProvisioningStateCanceled: + p.console.StopSpinner(ctx, "Deployment canceled", input.StepDone) + if portalUrl != "" { + p.console.Message(ctx, + output.WithHighLightFormat( + "Canceled deployment is recorded in the portal:\n %s", + output.WithLinkFormat(portalUrl))) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCanceledByUser, + telemetryValue: "canceled", + } + case azapi.DeploymentProvisioningStateSucceeded: + // Cancel arrived after the deployment had already succeeded. Frame + // this as a *success* (resources are deployed) rather than a + // failure — the user pressed Ctrl+C suspecting a hang and needs to + // know their resources are live. + p.console.StopSpinner(ctx, + "Azure deployment completed successfully before cancellation took effect", + input.StepDone) + if portalUrl != "" { + p.console.Message(ctx, + output.WithHighLightFormat( + "Your resources are deployed. Details:\n %s", + output.WithLinkFormat(portalUrl))) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTooLate, + telemetryValue: "cancel_raced_succeeded", + } + case azapi.DeploymentProvisioningStateFailed: + p.console.StopSpinner(ctx, + "Azure deployment failed before cancellation took effect", + input.StepFailed) + if portalUrl != "" { + p.console.Message(ctx, + output.WithErrorFormat( + "Review the failure:\n %s", + output.WithLinkFormat(portalUrl))) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTooLate, + telemetryValue: "cancel_raced_failed", + } + case azapi.DeploymentProvisioningStateDeleted: + p.console.StopSpinner(ctx, "Deployment was deleted", input.StepWarning) + if portalUrl != "" { + p.console.Message(ctx, + output.WithWarningFormat( + "Someone or something deleted the Azure deployment record "+ + "before cancellation could take effect (unusual — check "+ + "audit logs if unexpected). Review:\n %s", + output.WithLinkFormat(portalUrl))) + } else { + p.console.Message(ctx, + output.WithWarningFormat( + "Someone or something deleted the Azure deployment record "+ + "before cancellation could take effect (unusual — check "+ + "audit logs if unexpected).")) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTooLate, + telemetryValue: "cancel_raced_deleted", + } + default: + // isTerminalProvisioningState should prevent reaching here, but be + // defensive: stop the spinner and warn the user so the UI is left in + // a clean state, then surface as too-late so the caller exits. + p.console.StopSpinner(ctx, "Deployment reached an unexpected terminal state", input.StepWarning) + if portalUrl != "" { + p.console.Message(ctx, + output.WithWarningFormat( + "The Azure deployment reached unexpected terminal state %q after the cancel request. Review:\n %s", + string(state), output.WithLinkFormat(portalUrl))) + } else { + p.console.Message(ctx, + output.WithWarningFormat( + "The Azure deployment reached unexpected terminal state %q after the cancel request.", + string(state))) + } + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTooLate, + telemetryValue: "cancel_too_late", + } + } +} + +// isTerminalProvisioningState reports whether an Azure deployment provisioning +// state represents a terminal outcome (no further transitions expected). +func isTerminalProvisioningState(state azapi.DeploymentProvisioningState) bool { + switch state { + case azapi.DeploymentProvisioningStateCanceled, + azapi.DeploymentProvisioningStateFailed, + azapi.DeploymentProvisioningStateSucceeded, + azapi.DeploymentProvisioningStateDeleted: + return true + } + return false +} + +// isNestedDeployment reports whether a deployment-operation refers to a +// child Microsoft.Resources/deployments resource. Mirrors the predicate +// used by pkg/infra.WalkDeploymentOperations so we can reuse the same +// "child deployment" semantics without exporting that package's internal. +func isNestedDeployment(op *armresources.DeploymentOperation) bool { + if op == nil || op.Properties == nil || + op.Properties.TargetResource == nil || + op.Properties.TargetResource.ResourceType == nil || + op.Properties.ProvisioningOperation == nil { + return false + } + return *op.Properties.TargetResource.ResourceType == + string(azapi.AzureResourceTypeDeployment) && + *op.Properties.ProvisioningOperation == armresources.ProvisioningOperationCreate +} + +// cancelAndAwaitNested discovers the descendant deployments of a freshly +// canceled top-level deployment, best-effort cancels any that are still +// non-terminal, and polls them until either all have reached a terminal +// state or pollCtx is canceled (typically by the global cancelOverallTimeout). +// +// Returns the descendant deployments that were still non-terminal when the +// wait deadline fired; on success the returned slice is empty. +// +// Any failure to discover descendants (e.g. transient ARM error listing +// operations) is logged and treated as "no descendants found" — the +// interrupt UX should never explode just because we couldn't enumerate +// child deployments. +func (p *BicepProvider) cancelAndAwaitNested( + pollCtx context.Context, + parent infra.Deployment, +) []infra.Deployment { + descendants, err := p.discoverDescendantDeployments(pollCtx, parent, p.deploymentForResourceID) + if err != nil { + log.Printf("interrupt handler: failed to discover descendant deployments: %v", err) + return nil + } + if len(descendants) == 0 { + return nil + } + + // Best-effort cancel of any descendant still in a non-terminal state. + // We deliberately swallow per-descendant errors here (logged) — the + // authoritative signal is the polling loop below, and a failed Cancel + // for a descendant that is already terminal is expected. + p.cancelDescendants(pollCtx, descendants) + + // Poll concurrently with a small worker pool, returning whichever + // deployments remain non-terminal at the deadline. + return p.pollDescendantsTerminal(pollCtx, descendants) +} + +// discoverDescendantDeployments walks the parent's deployment-operations tree +// and returns one infra.Deployment per *unique* nested deployment found at +// any depth. The deployment objects are constructed by `factory` from each +// operation's TargetResource.ID, so a sub-scope parent with RG-scope +// children is handled correctly. The factory is injected for test +// purposes — production callers should pass `p.deploymentForResourceID`. +func (p *BicepProvider) discoverDescendantDeployments( + ctx context.Context, + parent infra.Deployment, + factory func(*arm.ResourceID) infra.Deployment, +) ([]infra.Deployment, error) { + type frame struct { + ops []*armresources.DeploymentOperation + } + + rootOps, err := parent.Operations(ctx) + if err != nil { + return nil, fmt.Errorf("listing operations: %w", err) + } + + seen := map[string]struct{}{} + var out []infra.Deployment + queue := []frame{{ops: rootOps}} + + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + + for _, op := range cur.ops { + if op == nil || op.Properties == nil { + continue + } + if !isNestedDeployment(op) { + continue + } + if op.Properties.TargetResource == nil || + op.Properties.TargetResource.ID == nil { + continue + } + id := *op.Properties.TargetResource.ID + if _, dup := seen[id]; dup { + continue + } + seen[id] = struct{}{} + + parsed, err := arm.ParseResourceID(id) + if err != nil { + log.Printf("interrupt handler: skipping unparsable nested deployment id %q: %v", id, err) + continue + } + + child := factory(parsed) + out = append(out, child) + + // Recurse into this child's operations to pick up grandchildren. + childOps, err := child.Operations(ctx) + if err != nil { + log.Printf("interrupt handler: failed to list operations for nested deployment %q: %v", + parsed.Name, err) + continue + } + queue = append(queue, frame{ops: childOps}) + } + } + return out, nil +} + +// deploymentForResourceID constructs an infra.Deployment from the parsed +// resource ID of a nested Microsoft.Resources/deployments resource. We pick +// the right scope (subscription vs resource group) from the ID itself rather +// than inheriting from the parent, so a sub-scope parent with RG-scope +// children works correctly. +func (p *BicepProvider) deploymentForResourceID(id *arm.ResourceID) infra.Deployment { + if id.ResourceGroupName != "" { + // Cancel/Get on RG-scope deployments don't depend on the + // deployment manager's location, so we pass through subscription / + // RG / name directly. + scope := p.deploymentManager.ResourceGroupScope(id.SubscriptionID, id.ResourceGroupName) + return p.deploymentManager.ResourceGroupDeployment(scope, id.Name) + } + // Sub-scope: location is irrelevant for Cancel/Get/DeploymentUrl. + scope := p.deploymentManager.SubscriptionScope(id.SubscriptionID, "") + return p.deploymentManager.SubscriptionDeployment(scope, id.Name) +} + +// cancelDescendants issues a best-effort Cancel on each descendant that is +// not already in a terminal state. Per-descendant errors (including +// already-terminal "Conflict" responses and ErrCancelNotSupported) are +// logged at DEBUG and otherwise ignored — the polling loop is what decides +// success. +func (p *BicepProvider) cancelDescendants( + pollCtx context.Context, + descendants []infra.Deployment, +) { + sem := make(chan struct{}, cancelNestedConcurrency) + var wg sync.WaitGroup + + for _, d := range descendants { + select { + case <-pollCtx.Done(): + return + default: + } + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + + // Skip if already terminal — saves an unnecessary Cancel call + // (which would just return Conflict). + if state, err := d.Get(pollCtx); err == nil && + isTerminalProvisioningState(state.ProvisioningState) { + return + } + + // Bound the cancel call itself so a single hung ARM endpoint + // can't consume the global budget. + cancelCtx, cancelDone := context.WithTimeout(pollCtx, cancelRequestTimeout) + defer cancelDone() + + if err := d.Cancel(cancelCtx); err != nil { + if !errors.Is(err, azapi.ErrCancelNotSupported) { + log.Printf("interrupt handler: cancel failed for nested deployment %q: %v", + d.Name(), err) + } + } + }) + } + wg.Wait() +} + +// pollDescendantsTerminal polls each non-terminal descendant deployment until +// it reaches a terminal state or pollCtx fires. Returns the slice of +// descendants that were still non-terminal when pollCtx fired (empty slice +// on full success). +func (p *BicepProvider) pollDescendantsTerminal( + pollCtx context.Context, + descendants []infra.Deployment, +) []infra.Deployment { + type result struct { + d infra.Deployment + terminal bool + } + + results := make(chan result, len(descendants)) + sem := make(chan struct{}, cancelNestedConcurrency) + var wg sync.WaitGroup + + for _, d := range descendants { + sem <- struct{}{} + wg.Go(func() { + defer func() { <-sem }() + results <- result{d: d, terminal: pollSingleTerminal(pollCtx, d)} + }) + } + + go func() { + wg.Wait() + close(results) + }() + + var stuck []infra.Deployment + for r := range results { + if !r.terminal { + stuck = append(stuck, r.d) + } + } + return stuck +} + +// pollSingleTerminal polls a single deployment until terminal or pollCtx fires. +// Returns true if a terminal state was observed. +func pollSingleTerminal(pollCtx context.Context, d infra.Deployment) bool { + if state, err := d.Get(pollCtx); err == nil && + isTerminalProvisioningState(state.ProvisioningState) { + return true + } + ticker := time.NewTicker(cancelPollInterval) + defer ticker.Stop() + for { + select { + case <-pollCtx.Done(): + return false + case <-ticker.C: + } + state, err := d.Get(pollCtx) + if err != nil { + log.Printf("interrupt handler: poll Get failed for nested deployment %q (will retry): %v", + d.Name(), err) + continue + } + if isTerminalProvisioningState(state.ProvisioningState) { + return true + } + } +} + +// nestedTimeoutOutcome reports the timeout outcome when one or more +// descendant deployments did not reach a terminal state within the global +// cancelOverallTimeout. The user-facing message lists portal URLs for the +// stuck deployments so they can investigate. +func (p *BicepProvider) nestedTimeoutOutcome( + ctx context.Context, + stuck []infra.Deployment, + parentPortalUrl string, +) interruptOutcome { + p.console.StopSpinner(ctx, + fmt.Sprintf("%d nested Azure deployment(s) did not finish within %s", + len(stuck), cancelOverallTimeout), + input.StepWarning) + + var lines []string + if parentPortalUrl != "" { + lines = append(lines, + fmt.Sprintf("Top-level deployment was canceled, but the following nested "+ + "deployment(s) are still running. Track them in the portal:")) + } else { + lines = append(lines, + "Top-level deployment was canceled, but the following nested "+ + "deployment(s) are still running. They may block the next deployment "+ + "until they reach a terminal state:") + } + + for _, d := range stuck { + url, err := d.DeploymentUrl(ctx) + if err != nil || url == "" { + lines = append(lines, fmt.Sprintf(" - %s", d.Name())) + } else { + lines = append(lines, + fmt.Sprintf(" - %s\n %s", d.Name(), output.WithLinkFormat(url))) + } + } + + for _, l := range lines { + p.console.Message(ctx, output.WithWarningFormat("%s", l)) + } + + return interruptOutcome{ + err: provisioning.ErrDeploymentCancelTimeout, + telemetryValue: "cancel_timed_out_nested", + } +} + +// applyInterruptOutcome decides what to return from BicepProvider.Deploy when +// an interrupt outcome was produced. It composes any pre-existing deploy error +// with the interrupt sentinel so error wrapping (`errors.Is`) keeps working. +func applyInterruptOutcome(outcome interruptOutcome, deployErr error) error { + if deployErr == nil { + return outcome.err + } + // Most likely deployErr is "context canceled" wrapped by the SDK (because + // we cancelled deployCtx to unblock PollUntilDone). Prefer the typed + // interrupt sentinel for the user-visible error chain. + if errors.Is(deployErr, context.Canceled) { + return outcome.err + } + return fmt.Errorf("%w: %w", outcome.err, deployErr) +} diff --git a/cli/azd/pkg/infra/provisioning/bicep/interrupt_test.go b/cli/azd/pkg/infra/provisioning/bicep/interrupt_test.go new file mode 100644 index 00000000000..5c5d8c4b016 --- /dev/null +++ b/cli/azd/pkg/infra/provisioning/bicep/interrupt_test.go @@ -0,0 +1,712 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package bicep + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/arm" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/azure/azure-dev/cli/azd/pkg/async" + "github.com/azure/azure-dev/cli/azd/pkg/azapi" + "github.com/azure/azure-dev/cli/azd/pkg/azure" + "github.com/azure/azure-dev/cli/azd/pkg/infra" + "github.com/azure/azure-dev/cli/azd/pkg/infra/provisioning" + "github.com/azure/azure-dev/cli/azd/pkg/input" + "github.com/azure/azure-dev/cli/azd/test/mocks" + "github.com/stretchr/testify/require" +) + +func TestIsTerminalProvisioningState(t *testing.T) { + terminal := []azapi.DeploymentProvisioningState{ + azapi.DeploymentProvisioningStateCanceled, + azapi.DeploymentProvisioningStateFailed, + azapi.DeploymentProvisioningStateSucceeded, + azapi.DeploymentProvisioningStateDeleted, + } + nonTerminal := []azapi.DeploymentProvisioningState{ + azapi.DeploymentProvisioningStateAccepted, + azapi.DeploymentProvisioningStateCanceling, + azapi.DeploymentProvisioningStateRunning, + azapi.DeploymentProvisioningStateDeploying, + azapi.DeploymentProvisioningStateValidating, + azapi.DeploymentProvisioningStateWaiting, + azapi.DeploymentProvisioningStateNotSpecified, + "", + } + for _, s := range terminal { + require.Truef(t, isTerminalProvisioningState(s), "expected %q to be terminal", s) + } + for _, s := range nonTerminal { + require.Falsef(t, isTerminalProvisioningState(s), "expected %q to NOT be terminal", s) + } +} + +func TestApplyInterruptOutcome(t *testing.T) { + leave := interruptOutcome{ + err: provisioning.ErrDeploymentInterruptedLeaveRunning, + telemetryValue: "leave_running", + } + canceled := interruptOutcome{ + err: provisioning.ErrDeploymentCanceledByUser, + telemetryValue: "canceled", + } + + t.Run("nil deploy error returns outcome err", func(t *testing.T) { + require.ErrorIs(t, applyInterruptOutcome(leave, nil), + provisioning.ErrDeploymentInterruptedLeaveRunning) + require.ErrorIs(t, applyInterruptOutcome(canceled, nil), + provisioning.ErrDeploymentCanceledByUser) + }) + + t.Run("context canceled is replaced by outcome err", func(t *testing.T) { + err := applyInterruptOutcome(canceled, context.Canceled) + require.ErrorIs(t, err, provisioning.ErrDeploymentCanceledByUser) + require.NotErrorIs(t, err, context.Canceled) + }) + + t.Run("wrapped context canceled is replaced by outcome err", func(t *testing.T) { + wrapped := fmt.Errorf("PollUntilDone: %w", context.Canceled) + err := applyInterruptOutcome(leave, wrapped) + require.ErrorIs(t, err, provisioning.ErrDeploymentInterruptedLeaveRunning) + require.NotErrorIs(t, err, context.Canceled) + }) + + t.Run("non-cancel deploy error is preserved alongside outcome", func(t *testing.T) { + other := errors.New("template validation failed") + err := applyInterruptOutcome(canceled, other) + require.ErrorIs(t, err, provisioning.ErrDeploymentCanceledByUser) + require.ErrorIs(t, err, other) + }) +} + +// fakeDeployment is a programmable infra.Deployment used by interrupt tests. +// Only the methods that the interrupt flow exercises (Cancel, Get, +// DeploymentUrl, Operations) have meaningful behavior; the rest panic if +// invoked. +type fakeDeployment struct { + deploymentUrl string + deploymentUrlErr error + + cancelCalls atomic.Int32 + cancelFn func(ctx context.Context) error + + getCalls atomic.Int32 + // getFn is invoked on each Get; the int passed in is the 1-based call + // index so tests can sequence different responses. + getFn func(ctx context.Context, callIndex int32) (*azapi.ResourceDeployment, error) + + operationsCalls atomic.Int32 + // operationsFn lets tests inject descendant deployments; nil means + // "no descendants". + operationsFn func(ctx context.Context) ([]*armresources.DeploymentOperation, error) +} + +func (f *fakeDeployment) Cancel(ctx context.Context) error { + n := f.cancelCalls.Add(1) + if f.cancelFn == nil { + return nil + } + _ = n + return f.cancelFn(ctx) +} + +func (f *fakeDeployment) Get(ctx context.Context) (*azapi.ResourceDeployment, error) { + n := f.getCalls.Add(1) + if f.getFn == nil { + return &azapi.ResourceDeployment{}, nil + } + return f.getFn(ctx, n) +} + +func (f *fakeDeployment) DeploymentUrl(ctx context.Context) (string, error) { + return f.deploymentUrl, f.deploymentUrlErr +} + +// The remaining infra.Deployment surface is unused by the interrupt flow. +func (f *fakeDeployment) SubscriptionId() string { panic("unused") } +func (f *fakeDeployment) ListDeployments(context.Context) ([]*azapi.ResourceDeployment, error) { + panic("unused") +} +func (f *fakeDeployment) Deployment(string) infra.Deployment { panic("unused") } +func (f *fakeDeployment) Name() string { return "fake-deployment" } +func (f *fakeDeployment) PortalUrl(context.Context) (string, error) { panic("unused") } +func (f *fakeDeployment) OutputsUrl(context.Context) (string, error) { panic("unused") } +func (f *fakeDeployment) ValidatePreflight( + context.Context, azure.RawArmTemplate, azure.ArmParameters, map[string]*string, map[string]any, +) error { + panic("unused") +} +func (f *fakeDeployment) Deploy( + context.Context, azure.RawArmTemplate, azure.ArmParameters, map[string]*string, map[string]any, +) (*azapi.ResourceDeployment, error) { + panic("unused") +} +func (f *fakeDeployment) Delete( + context.Context, map[string]any, *async.Progress[azapi.DeleteDeploymentProgress], +) error { + panic("unused") +} +func (f *fakeDeployment) DeployPreview( + context.Context, azure.RawArmTemplate, azure.ArmParameters, +) (*armresources.WhatIfOperationResult, error) { + panic("unused") +} +func (f *fakeDeployment) Operations(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + f.operationsCalls.Add(1) + if f.operationsFn == nil { + return nil, nil + } + return f.operationsFn(ctx) +} +func (f *fakeDeployment) Resources(context.Context) ([]*armresources.ResourceReference, error) { + panic("unused") +} + +// withFastInterruptPolling shrinks the cancel poll/timeout knobs for tests +// that exercise cancelAndAwaitTerminal so the suite stays sub-second. +func withFastInterruptPolling(t *testing.T) { + t.Helper() + prevReq, prevTerm, prevPoll := cancelRequestTimeout, cancelOverallTimeout, cancelPollInterval + cancelRequestTimeout = 100 * time.Millisecond + cancelOverallTimeout = 200 * time.Millisecond + cancelPollInterval = 5 * time.Millisecond + t.Cleanup(func() { + cancelRequestTimeout = prevReq + cancelOverallTimeout = prevTerm + cancelPollInterval = prevPoll + }) +} + +// newTestProvider builds a minimal BicepProvider with only the fields the +// interrupt flow touches populated (the console). +func newTestProvider(mockContext *mocks.MockContext) *BicepProvider { + return &BicepProvider{console: mockContext.Console} +} + +func TestRunInterruptPrompt_LeaveRunning(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + mockContext.Console.WhenSelect(func(o input.ConsoleOptions) bool { + return true + }).RespondFn(func(o input.ConsoleOptions) (any, error) { + require.Equal(t, interruptOptionLeaveRunning, o.Options[0]) + require.Equal(t, interruptOptionCancel, o.Options[1]) + return 0, nil + }) + + provider := newTestProvider(mockContext) + deployment := &fakeDeployment{deploymentUrl: "https://portal/deployment"} + + outcome := provider.runInterruptPrompt(t.Context(), deployment) + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentInterruptedLeaveRunning) + require.Equal(t, "leave_running", outcome.telemetryValue) + require.Equal(t, int32(0), deployment.cancelCalls.Load(), + "leave-running must not submit a cancel request") +} + +func TestRunInterruptPrompt_PromptError_FallsBackToLeaveRunning(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + mockContext.Console.WhenSelect(func(o input.ConsoleOptions) bool { return true }). + RespondFn(func(o input.ConsoleOptions) (any, error) { + return 0, errors.New("non-interactive: stdin closed") + }) + + provider := newTestProvider(mockContext) + deployment := &fakeDeployment{deploymentUrl: "https://portal/deployment"} + + outcome := provider.runInterruptPrompt(t.Context(), deployment) + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentInterruptedLeaveRunning) + require.Equal(t, "leave_running", outcome.telemetryValue) + require.Equal(t, int32(0), deployment.cancelCalls.Load()) +} + +func TestRunInterruptPrompt_DeploymentUrlError_StillPrompts(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + mockContext.Console.WhenSelect(func(o input.ConsoleOptions) bool { return true }). + RespondFn(func(o input.ConsoleOptions) (any, error) { + require.NotContains(t, o.Help, "Portal:", + "prompt help must omit Portal line when URL is unavailable") + return 0, nil + }) + + provider := newTestProvider(mockContext) + deployment := &fakeDeployment{deploymentUrlErr: errors.New("ARM unreachable")} + + outcome := provider.runInterruptPrompt(t.Context(), deployment) + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentInterruptedLeaveRunning) +} + +func TestCancelAndAwaitTerminal_CancelNotSupported_ReturnsLeaveRunning(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + deployment := &fakeDeployment{ + cancelFn: func(ctx context.Context) error { return azapi.ErrCancelNotSupported }, + } + + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentInterruptedLeaveRunning) + require.Equal(t, "leave_running", outcome.telemetryValue) + require.Equal(t, int32(1), deployment.cancelCalls.Load()) + require.Equal(t, int32(0), deployment.getCalls.Load(), + "cancel-not-supported must short-circuit before any Get poll") +} + +func TestCancelAndAwaitTerminal_CancelFailed_NoTerminalState(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + cancelErr := errors.New("ARM 503") + deployment := &fakeDeployment{ + cancelFn: func(ctx context.Context) error { return cancelErr }, + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return nil, errors.New("ARM Get also failing") + }, + } + + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCancelFailed) + require.ErrorIs(t, outcome.err, cancelErr) + require.Equal(t, "cancel_failed", outcome.telemetryValue) +} + +func TestCancelAndAwaitTerminal_CancelFailed_ButDeploymentAlreadyTerminal(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + deployment := &fakeDeployment{ + cancelFn: func(ctx context.Context) error { return errors.New("ARM 409 conflict") }, + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateSucceeded, + }, nil + }, + } + + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCancelTooLate) + require.Equal(t, "cancel_raced_succeeded", outcome.telemetryValue) +} + +func TestCancelAndAwaitTerminal_FirstGetIsImmediate(t *testing.T) { + // Use a very long poll interval so that if the impl regresses to + // "tick-then-Get", this test would block far longer than the deadline. + t.Helper() + prevReq, prevTerm, prevPoll := cancelRequestTimeout, cancelOverallTimeout, cancelPollInterval + cancelRequestTimeout = 100 * time.Millisecond + cancelOverallTimeout = 5 * time.Second + cancelPollInterval = 5 * time.Second + t.Cleanup(func() { + cancelRequestTimeout = prevReq + cancelOverallTimeout = prevTerm + cancelPollInterval = prevPoll + }) + + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + deployment := &fakeDeployment{ + // First Get already returns Canceled — no poll-interval wait needed. + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + require.Equal(t, int32(1), n, + "first Get must be issued before any poll-interval wait") + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateCanceled, + }, nil + }, + } + + start := time.Now() + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + elapsed := time.Since(start) + + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCanceledByUser) + require.Less(t, elapsed, time.Second, + "fast-path cancellation should not wait a full poll interval; took %s", elapsed) + require.Equal(t, int32(1), deployment.getCalls.Load()) +} + +func TestCancelAndAwaitTerminal_PollsUntilCanceled(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + deployment := &fakeDeployment{ + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + if n < 3 { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateRunning, + }, nil + } + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateCanceled, + }, nil + }, + } + + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCanceledByUser) + require.Equal(t, "canceled", outcome.telemetryValue) + require.GreaterOrEqual(t, deployment.getCalls.Load(), int32(3)) +} + +func TestCancelAndAwaitTerminal_TimeoutWhilePolling(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + // Always Running → never reaches a terminal state, so the poll budget + // must elapse and we must report cancel_timed_out. + deployment := &fakeDeployment{ + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateRunning, + }, nil + }, + } + + outcome := provider.cancelAndAwaitTerminal(t.Context(), deployment, "https://portal/x") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCancelTimeout) + require.Equal(t, "cancel_timed_out", outcome.telemetryValue) +} + +func TestInstallDeploymentInterruptHandler_MarkCompletedWinsRace(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + deployment := &fakeDeployment{} + + deployCtx, started, _, markCompleted, cleanup := provider.installDeploymentInterruptHandler( + t.Context(), deployment, nil) + defer cleanup() + + // Deploy completes naturally first → markCompleted wins the CAS. + require.True(t, markCompleted(), + "markCompleted must succeed when no interrupt has fired") + + // A handler invocation that arrives after completion must be a no-op: + // it must not close started and must not cancel deployCtx. + stack := input.SnapshotInterruptStack() + require.NotEmpty(t, stack, "handler should still be on the stack pre-cleanup") + handled := stack[len(stack)-1]() + require.False(t, handled, + "handler must return false (decline ownership) when deploy already completed") + + select { + case <-started: + t.Fatal("started must NOT be closed when markCompleted wins the race") + case <-deployCtx.Done(): + t.Fatal("deployCtx must NOT be cancelled when markCompleted wins the race") + default: + } +} + +func TestInstallDeploymentInterruptHandler_InterruptWinsRace(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + mockContext.Console.WhenSelect(func(o input.ConsoleOptions) bool { return true }). + RespondFn(func(o input.ConsoleOptions) (any, error) { + // "Leave running" — keeps the test deterministic and avoids + // exercising the cancel poll loop here. + return 0, nil + }) + + provider := newTestProvider(mockContext) + deployment := &fakeDeployment{deploymentUrl: "https://portal/x"} + + onStartCalled := false + deployCtx, started, outcomeCh, markCompleted, cleanup := provider.installDeploymentInterruptHandler( + t.Context(), deployment, func() { onStartCalled = true }) + defer cleanup() + + stack := input.SnapshotInterruptStack() + require.NotEmpty(t, stack) + handled := stack[len(stack)-1]() + require.True(t, handled, + "handler must claim shutdown ownership when the interrupt wins the race") + require.True(t, onStartCalled, "onInterruptStart must be invoked") + + // started must be closed and deployCtx must be cancelled synchronously + // before the prompt is shown so Deploy can flip to wait-for-outcome mode. + select { + case <-started: + default: + t.Fatal("started must be closed when the handler runs") + } + require.ErrorIs(t, deployCtx.Err(), context.Canceled, + "deployCtx must be cancelled when the handler runs") + + // Outcome must be available on the channel. + select { + case outcome := <-outcomeCh: + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentInterruptedLeaveRunning) + case <-time.After(time.Second): + t.Fatal("outcome was not delivered on outcomeCh") + } + + // Once the interrupt has won, markCompleted must fail. + require.False(t, markCompleted(), + "markCompleted must return false after the interrupt path has claimed the state") +} + +// makeNestedDeploymentOp builds a fake DeploymentOperation referring to a +// child Microsoft.Resources/deployments resource. Used by the descendant- +// discovery tests below. +func makeNestedDeploymentOp(targetID string) *armresources.DeploymentOperation { + resourceType := string(azapi.AzureResourceTypeDeployment) + provisioningOp := armresources.ProvisioningOperationCreate + id := targetID + return &armresources.DeploymentOperation{ + Properties: &armresources.DeploymentOperationProperties{ + ProvisioningOperation: &provisioningOp, + TargetResource: &armresources.TargetResource{ + ID: &id, + ResourceType: &resourceType, + }, + }, + } +} + +// makeNonNestedOp builds a fake DeploymentOperation for a regular +// (non-deployment) resource, e.g. Microsoft.Storage/storageAccounts. +func makeNonNestedOp(targetID, resourceType string) *armresources.DeploymentOperation { + provisioningOp := armresources.ProvisioningOperationCreate + id := targetID + rt := resourceType + return &armresources.DeploymentOperation{ + Properties: &armresources.DeploymentOperationProperties{ + ProvisioningOperation: &provisioningOp, + TargetResource: &armresources.TargetResource{ + ID: &id, + ResourceType: &rt, + }, + }, + } +} + +func TestDiscoverDescendantDeployments_FlatTree(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + parent := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + return []*armresources.DeploymentOperation{ + makeNonNestedOp( + "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Storage/storageAccounts/sa", + "Microsoft.Storage/storageAccounts"), + makeNestedDeploymentOp( + "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Resources/deployments/child1"), + makeNestedDeploymentOp( + "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Resources/deployments/child2"), + }, nil + }, + } + + var built []string + factory := func(id *arm.ResourceID) infra.Deployment { + built = append(built, id.Name) + // Children with no further operations. + return &fakeDeployment{} + } + + descendants, err := provider.discoverDescendantDeployments(t.Context(), parent, factory) + require.NoError(t, err) + require.Len(t, descendants, 2) + require.ElementsMatch(t, []string{"child1", "child2"}, built) +} + +func TestDiscoverDescendantDeployments_RecursesAndDedupes(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + const childID = "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Resources/deployments/child1" + const grandID = "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Resources/deployments/grand1" + + parent := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + // Reference child1 twice — should be deduped. + return []*armresources.DeploymentOperation{ + makeNestedDeploymentOp(childID), + makeNestedDeploymentOp(childID), + }, nil + }, + } + + child := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + return []*armresources.DeploymentOperation{ + makeNestedDeploymentOp(grandID), + }, nil + }, + } + + grand := &fakeDeployment{} + + factory := func(id *arm.ResourceID) infra.Deployment { + switch id.Name { + case "child1": + return child + case "grand1": + return grand + } + t.Fatalf("unexpected child %q", id.Name) + return nil + } + + descendants, err := provider.discoverDescendantDeployments(t.Context(), parent, factory) + require.NoError(t, err) + require.Len(t, descendants, 2, "child1 must be deduped, grandchild must be discovered") +} + +func TestDiscoverDescendantDeployments_OperationsErrorPropagates(t *testing.T) { + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + parent := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + return nil, errors.New("ARM unavailable") + }, + } + + _, err := provider.discoverDescendantDeployments(t.Context(), parent, + func(*arm.ResourceID) infra.Deployment { + t.Fatal("factory should not be called on operations error") + return nil + }) + require.Error(t, err) +} + +func TestCancelAndAwaitNested_NoDescendants_ReturnsEmpty(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + parent := &fakeDeployment{} // Operations returns nil + + stuck := provider.cancelAndAwaitNested(t.Context(), parent) + require.Empty(t, stuck) +} + +func TestCancelAndAwaitNested_OperationsError_ReturnsEmpty(t *testing.T) { + // Discovery failure must not crash the interrupt flow — it should be + // swallowed (logged) and treated as "no descendants". + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + parent := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + return nil, errors.New("transient ARM error") + }, + } + + stuck := provider.cancelAndAwaitNested(t.Context(), parent) + require.Empty(t, stuck) +} + +func TestCancelAndAwaitTerminal_NestedStuck_ReturnsTimeoutWithNestedTelemetry(t *testing.T) { + // End-to-end: top-level transitions to Canceled quickly, but a child + // deployment never reaches a terminal state. The interrupt outcome must + // be ErrDeploymentCancelTimeout with telemetry "cancel_timed_out_nested", + // and the user-facing message must list the stuck child. + withFastInterruptPolling(t) + // Make sure we have time for: top-level Get + nested Cancel + nested Get + // poll loop hitting deadline. + prevOverall := cancelOverallTimeout + cancelOverallTimeout = 50 * time.Millisecond + t.Cleanup(func() { cancelOverallTimeout = prevOverall }) + + mockContext := mocks.NewMockContext(context.Background()) + + const childID = "/subscriptions/SUB/resourceGroups/rg/providers/Microsoft.Resources/deployments/stuck-child" + + // Stub the deployment manager — only the child factory is exercised here + // because we override discoverDescendantDeployments via the factory + // argument injected by cancelAndAwaitNested → we need to override the + // method itself. Instead, we rely on the test reaching cancelAndAwaitNested + // and supplying a fake child via parent.Operations + the production + // deploymentForResourceID being unreachable in this path because we + // re-route through a custom provider. + + // Simpler approach: drive it from the building block, not the full + // cancelAndAwaitTerminal. Use cancelAndAwaitNested directly via the + // factory to inject the stuck child. + + parent := &fakeDeployment{ + operationsFn: func(ctx context.Context) ([]*armresources.DeploymentOperation, error) { + return []*armresources.DeploymentOperation{makeNestedDeploymentOp(childID)}, nil + }, + } + + // Stuck child: Get always returns Running. + stuckChild := &fakeDeployment{ + deploymentUrl: "https://portal/stuck-child", + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateRunning, + }, nil + }, + } + + provider := newTestProvider(mockContext) + + // Drive cancelAndAwaitNested directly with our factory injection. + pollCtx, pollDone := context.WithTimeout(t.Context(), cancelOverallTimeout) + defer pollDone() + + descendants, err := provider.discoverDescendantDeployments(pollCtx, parent, + func(id *arm.ResourceID) infra.Deployment { + require.Equal(t, "stuck-child", id.Name) + return stuckChild + }) + require.NoError(t, err) + require.Len(t, descendants, 1) + + provider.cancelDescendants(pollCtx, descendants) + stuck := provider.pollDescendantsTerminal(pollCtx, descendants) + require.Len(t, stuck, 1, "stuck child must be reported") + + // nestedTimeoutOutcome composes the user-facing message and outcome. + outcome := provider.nestedTimeoutOutcome(t.Context(), stuck, "https://portal/parent") + require.ErrorIs(t, outcome.err, provisioning.ErrDeploymentCancelTimeout) + require.Equal(t, "cancel_timed_out_nested", outcome.telemetryValue) + // Best-effort cancel must have been issued on the stuck child. + require.GreaterOrEqual(t, stuckChild.cancelCalls.Load(), int32(1), + "stuck child must receive a best-effort cancel") +} + +func TestCancelDescendants_SkipsTerminal(t *testing.T) { + withFastInterruptPolling(t) + mockContext := mocks.NewMockContext(context.Background()) + provider := newTestProvider(mockContext) + + terminalChild := &fakeDeployment{ + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateCanceled, + }, nil + }, + } + runningChild := &fakeDeployment{ + getFn: func(ctx context.Context, n int32) (*azapi.ResourceDeployment, error) { + return &azapi.ResourceDeployment{ + ProvisioningState: azapi.DeploymentProvisioningStateRunning, + }, nil + }, + } + + provider.cancelDescendants(t.Context(), + []infra.Deployment{terminalChild, runningChild}) + + require.Equal(t, int32(0), terminalChild.cancelCalls.Load(), + "terminal child must not receive a Cancel") + require.GreaterOrEqual(t, runningChild.cancelCalls.Load(), int32(1), + "running child must receive a Cancel") +} diff --git a/cli/azd/pkg/infra/provisioning/cancel.go b/cli/azd/pkg/infra/provisioning/cancel.go new file mode 100644 index 00000000000..9e4df22a2e7 --- /dev/null +++ b/cli/azd/pkg/infra/provisioning/cancel.go @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package provisioning + +import "errors" + +// Cancellation sentinels surfaced by providers when the user interrupts a +// running deployment with Ctrl+C. These are sentinel errors so the action / +// error middleware can produce a friendly, non-zero exit (with the portal URL +// and a clear message) instead of treating the case as an unexpected failure. +var ( + // ErrDeploymentInterruptedLeaveRunning is returned when the user chose to + // stop azd but allow the in-flight Azure deployment to continue running. + ErrDeploymentInterruptedLeaveRunning = errors.New( + "azd was interrupted; the Azure deployment is still running") + + // ErrDeploymentCanceledByUser is returned when the user requested + // cancellation and Azure confirmed the deployment reached the Canceled + // terminal state. + ErrDeploymentCanceledByUser = errors.New( + "deployment was canceled by user request") + + // ErrDeploymentCancelTimeout is returned when azd asked Azure to cancel the + // deployment but the deployment had not reached a terminal state before + // the local wait budget expired. The cancellation is still in progress on + // Azure. + ErrDeploymentCancelTimeout = errors.New( + "deployment cancel request was submitted but did not complete before timeout") + + // ErrDeploymentCancelTooLate is returned when azd attempted to cancel the + // deployment but Azure had already moved it to a terminal state + // (Succeeded or Failed) before the cancel request could take effect. + ErrDeploymentCancelTooLate = errors.New( + "deployment finished before the cancel request could take effect") + + // ErrDeploymentCancelFailed is returned when the ARM Cancel request itself + // failed (network, permissions, etc.). The returned error includes the + // underlying cause so the caller can inspect it with errors.Is or + // errors.As. + ErrDeploymentCancelFailed = errors.New( + "deployment cancel request failed") +) diff --git a/cli/azd/pkg/infra/scope.go b/cli/azd/pkg/infra/scope.go index 303766d2d95..095dde9be09 100644 --- a/cli/azd/pkg/infra/scope.go +++ b/cli/azd/pkg/infra/scope.go @@ -54,6 +54,11 @@ type Deployment interface { options map[string]any, progress *async.Progress[azapi.DeleteDeploymentProgress], ) error + // Cancel requests Azure to cancel an in-flight deployment. Returns nil if + // the cancel request is accepted (the deployment will transition to the + // Canceling/Canceled state asynchronously). Callers should poll Get() to + // observe the terminal state. + Cancel(ctx context.Context) error // Deploy a given template with a set of parameters. DeployPreview( ctx context.Context, @@ -114,6 +119,12 @@ func (s *ResourceGroupDeployment) Delete( ) } +// Cancel requests Azure to cancel an in-flight resource-group-scoped deployment. +func (s *ResourceGroupDeployment) Cancel(ctx context.Context) error { + return s.deploymentService.CancelResourceGroupDeployment( + ctx, s.subscriptionId, s.resourceGroupName, s.name) +} + func (s *ResourceGroupDeployment) DeployPreview( ctx context.Context, template azure.RawArmTemplate, @@ -324,6 +335,11 @@ func (s *SubscriptionDeployment) Delete( return s.deploymentService.DeleteSubscriptionDeployment(ctx, s.subscriptionId, s.name, options, progress) } +// Cancel requests Azure to cancel an in-flight subscription-scoped deployment. +func (s *SubscriptionDeployment) Cancel(ctx context.Context) error { + return s.deploymentService.CancelSubscriptionDeployment(ctx, s.subscriptionId, s.name) +} + // Deploy a given template with a set of parameters. func (s *SubscriptionDeployment) DeployPreview( ctx context.Context, diff --git a/cli/azd/pkg/infra/scope_test.go b/cli/azd/pkg/infra/scope_test.go index 0ba0961b082..5137b806a66 100644 --- a/cli/azd/pkg/infra/scope_test.go +++ b/cli/azd/pkg/infra/scope_test.go @@ -297,6 +297,66 @@ func TestScopeGetResourceOperations(t *testing.T) { }) } +func TestScopeCancel(t *testing.T) { + t.Run("SubscriptionScopeSuccess", func(t *testing.T) { + mockContext := mocks.NewMockContext(t.Context()) + deploymentService := mockazapi.NewDeploymentsServiceFromMockContext(mockContext) + + called := false + mockContext.HttpClient.When(func(request *http.Request) bool { + return request.Method == http.MethodPost && strings.Contains( + request.URL.Path, + "/subscriptions/SUBSCRIPTION_ID/providers/Microsoft.Resources/deployments/DEPLOYMENT_NAME/cancel", + ) + }).RespondFn(func(request *http.Request) (*http.Response, error) { + called = true + return mocks.CreateEmptyHttpResponse(request, http.StatusNoContent) + }) + + scope := newSubscriptionScope(deploymentService, "SUBSCRIPTION_ID", "eastus2") + target := NewSubscriptionDeployment(scope, "DEPLOYMENT_NAME") + require.NoError(t, target.Cancel(*mockContext.Context)) + require.True(t, called, "expected ARM cancel endpoint to be called") + }) + + t.Run("ResourceGroupScopeSuccess", func(t *testing.T) { + mockContext := mocks.NewMockContext(t.Context()) + deploymentService := mockazapi.NewDeploymentsServiceFromMockContext(mockContext) + + called := false + mockContext.HttpClient.When(func(request *http.Request) bool { + return request.Method == http.MethodPost && strings.Contains( + request.URL.Path, + //nolint:lll + "/subscriptions/SUBSCRIPTION_ID/resourcegroups/RESOURCE_GROUP/providers/Microsoft.Resources/deployments/DEPLOYMENT_NAME/cancel", + ) + }).RespondFn(func(request *http.Request) (*http.Response, error) { + called = true + return mocks.CreateEmptyHttpResponse(request, http.StatusNoContent) + }) + + scope := newResourceGroupScope(deploymentService, "SUBSCRIPTION_ID", "RESOURCE_GROUP") + target := NewResourceGroupDeployment(scope, "DEPLOYMENT_NAME") + require.NoError(t, target.Cancel(*mockContext.Context)) + require.True(t, called, "expected ARM cancel endpoint to be called") + }) + + t.Run("PropagatesError", func(t *testing.T) { + mockContext := mocks.NewMockContext(t.Context()) + deploymentService := mockazapi.NewDeploymentsServiceFromMockContext(mockContext) + + mockContext.HttpClient.When(func(request *http.Request) bool { + return request.Method == http.MethodPost && strings.Contains(request.URL.Path, "/cancel") + }).RespondFn(func(request *http.Request) (*http.Response, error) { + return mocks.CreateEmptyHttpResponse(request, http.StatusConflict) + }) + + scope := newSubscriptionScope(deploymentService, "SUBSCRIPTION_ID", "eastus2") + target := NewSubscriptionDeployment(scope, "DEPLOYMENT_NAME") + require.Error(t, target.Cancel(*mockContext.Context)) + }) +} + var testArmParameters = azure.ArmParameters{ "location": { Value: "West US", diff --git a/cli/azd/pkg/input/console.go b/cli/azd/pkg/input/console.go index b30f253929b..b6bcb2deb2e 100644 --- a/cli/azd/pkg/input/console.go +++ b/cli/azd/pkg/input/console.go @@ -985,12 +985,59 @@ func watchTerminalInterrupt(c *AskerConsole) { signalChan := make(chan os.Signal, 1) signal.Notify(signalChan, os.Interrupt) go func() { - <-signalChan + for range signalChan { + // Reserve the running slot first so re-entrant Ctrl+C signals are + // suppressed even in the brief window where a handler has been + // popped from the stack but is still executing (e.g. a prompt is + // up while Deploy has already torn the registration down). + if !tryStartInterruptHandler() { + // A handler is already running. Subsequent Ctrl+C presses + // allow the user to force-exit if the handler appears stuck: + // the first additional press arms the force-exit latch and + // is suppressed; the next press triggers the exit (standard + // POSIX convention: kubectl, terraform, docker, etc.). + if incrementForceExitCounter() { + _ = c.spinner.Stop() + os.Exit(130) // 128 + SIGINT + } + continue + } - // unhide the cursor if applicable - _ = c.spinner.Stop() + handler := currentInterruptHandler() + if handler == nil { + // No handler registered → default behavior. Release the slot + // before exiting so future signals would behave correctly if + // we did not exit (defensive). + finishInterruptHandler() + _ = c.spinner.Stop() + os.Exit(1) + } - os.Exit(1) + // Run the handler inline on the signal goroutine so any "interrupt + // started" side-effects (e.g. closing a started channel, cancelling + // the deploy context) take effect synchronously after the signal + // is received — no scheduling window where the deploy goroutine + // could complete naturally and silently drop the Ctrl+C. + var handled bool + func() { + defer func() { + if r := recover(); r != nil { + buf := make([]byte, 4096) + n := runtime.Stack(buf, false) + log.Printf( + "interrupt handler panic: %v\n%s", r, buf[:n]) + } + }() + handled = handler() + }() + finishInterruptHandler() + if !handled { + // Handler declined to take ownership of shutdown — fall back + // to default behavior. + _ = c.spinner.Stop() + os.Exit(1) + } + } }() } diff --git a/cli/azd/pkg/input/interrupt.go b/cli/azd/pkg/input/interrupt.go new file mode 100644 index 00000000000..ac211edf430 --- /dev/null +++ b/cli/azd/pkg/input/interrupt.go @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package input + +import "sync" + +// InterruptHandler is invoked when the user presses Ctrl+C. +// +// Implementations are expected to drive any user interaction (such as +// confirming whether to abort an in-flight Azure operation) and return only +// after they have decided how to respond. The handler runs synchronously on a +// dedicated goroutine: any additional Ctrl+C signals received while the +// handler is running are ignored. +// +// If the returned bool is false, the default azd interrupt behavior runs after +// the handler returns (the spinner is stopped and the process exits with +// code 1). Returning true tells the runtime that the handler took ownership of +// the shutdown sequence. +type InterruptHandler func() (handled bool) + +var ( + interruptMu sync.Mutex + interruptStack []InterruptHandler + interruptRunning bool + // forceExitPending tracks whether one suppressed Ctrl+C has been + // received while a handler is running. A second suppressed press + // triggers a force-exit (standard POSIX convention). + forceExitPending bool +) + +// PushInterruptHandler registers a handler to be invoked on the next SIGINT +// (Ctrl+C). Handlers are stacked: the most recently pushed handler runs first. +// +// The returned function pops the handler from the stack and must be called to +// restore the previous interrupt behavior (typically with `defer`). +func PushInterruptHandler(h InterruptHandler) func() { + interruptMu.Lock() + interruptStack = append(interruptStack, h) + idx := len(interruptStack) - 1 + interruptMu.Unlock() + + return func() { + interruptMu.Lock() + defer interruptMu.Unlock() + // Only pop this handler if it is still the current top-of-stack + // entry. This enforces strict LIFO semantics and avoids accidentally + // removing unrelated newer handlers if pop functions are called out + // of order. + if len(interruptStack) == idx+1 { + // Clear the slot first so the GC can reclaim the popped handler + // (and anything it captured) even if the underlying array isn't + // reallocated for a while. + interruptStack[idx] = nil + interruptStack = interruptStack[:idx] + } + } +} + +// currentInterruptHandler returns the top-of-stack interrupt handler, or nil +// if no handler is registered. +func currentInterruptHandler() InterruptHandler { + interruptMu.Lock() + defer interruptMu.Unlock() + if len(interruptStack) == 0 { + return nil + } + return interruptStack[len(interruptStack)-1] +} + +// tryStartInterruptHandler returns true if no handler is currently running. +// On success the caller is responsible for calling finishInterruptHandler. +func tryStartInterruptHandler() bool { + interruptMu.Lock() + defer interruptMu.Unlock() + if interruptRunning { + return false + } + interruptRunning = true + forceExitPending = false // reset on new handler start + return true +} + +func finishInterruptHandler() { + interruptMu.Lock() + defer interruptMu.Unlock() + interruptRunning = false + forceExitPending = false +} + +// incrementForceExitCounter records a suppressed Ctrl+C while a handler is +// running. Returns true if this is the second suppressed press, indicating +// a force-exit should occur (standard POSIX convention). +func incrementForceExitCounter() bool { + interruptMu.Lock() + defer interruptMu.Unlock() + if !interruptRunning { + return false + } + if forceExitPending { + return true + } + forceExitPending = true + return false +} + +// SnapshotInterruptStack returns a copy of the current interrupt-handler stack +// in push order (oldest first). This exists exclusively to let tests in other +// packages observe and invoke the handler that they pushed via +// PushInterruptHandler without having to install the real OS signal pipeline. +// +// It is NOT a stable API and must not be used by production code. +func SnapshotInterruptStack() []InterruptHandler { + interruptMu.Lock() + defer interruptMu.Unlock() + out := make([]InterruptHandler, len(interruptStack)) + copy(out, interruptStack) + return out +} diff --git a/cli/azd/pkg/input/interrupt_test.go b/cli/azd/pkg/input/interrupt_test.go new file mode 100644 index 00000000000..7e9b534453f --- /dev/null +++ b/cli/azd/pkg/input/interrupt_test.go @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package input + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestPushInterruptHandler_LIFO(t *testing.T) { + require.Nil(t, currentInterruptHandler()) + + firstCalls := 0 + first := func() bool { + firstCalls++ + return true + } + pop1 := PushInterruptHandler(first) + t.Cleanup(pop1) + + cur := currentInterruptHandler() + require.NotNil(t, cur) + require.True(t, cur()) + require.Equal(t, 1, firstCalls) + + secondCalls := 0 + second := func() bool { + secondCalls++ + return true + } + pop2 := PushInterruptHandler(second) + t.Cleanup(pop2) + + // Top-of-stack should be `second` (most recently pushed). + cur = currentInterruptHandler() + require.NotNil(t, cur) + require.True(t, cur()) + require.Equal(t, 1, firstCalls, "pushing second must not invoke first") + require.Equal(t, 1, secondCalls) + + pop2() + // After popping `second`, current should be `first` again. + cur = currentInterruptHandler() + require.NotNil(t, cur) + require.True(t, cur()) + require.Equal(t, 2, firstCalls) + require.Equal(t, 1, secondCalls, "popping second must not re-invoke it") + + pop1() + require.Nil(t, currentInterruptHandler()) +} + +func TestTryStartInterruptHandler_PreventsConcurrent(t *testing.T) { + require.True(t, tryStartInterruptHandler()) + t.Cleanup(finishInterruptHandler) + + // While the first handler is "running", the second start should be + // rejected so additional Ctrl+C signals are ignored. + require.False(t, tryStartInterruptHandler()) +} + +func TestForceExitCounter(t *testing.T) { + require.True(t, tryStartInterruptHandler()) + t.Cleanup(finishInterruptHandler) + + // First suppressed Ctrl+C while handler is running — not yet force-exit. + require.False(t, incrementForceExitCounter()) + // Second suppressed Ctrl+C — should trigger force-exit. + require.True(t, incrementForceExitCounter()) +} + +func TestForceExitCounter_ResetsOnNewHandler(t *testing.T) { + require.True(t, tryStartInterruptHandler()) + require.False(t, incrementForceExitCounter()) + finishInterruptHandler() + + // After finishing and starting a new handler, the counter resets. + require.True(t, tryStartInterruptHandler()) + t.Cleanup(finishInterruptHandler) + + require.False(t, incrementForceExitCounter(), + "force-exit counter should reset when a new handler starts") +}