diff --git a/build/CITargets.cs b/build/CITargets.cs index 7886befb1..7ed84c154 100644 --- a/build/CITargets.cs +++ b/build/CITargets.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -24,11 +24,42 @@ void StartDockerServices(params string[] services) AwaitDockerServices(services); } + /// + /// How many times will try `docker compose up -d` before giving up. + /// + const int DockerComposeAttempts = 3; + /// /// Fires `docker compose up -d` for the services and returns as soon as the containers are /// launched — no readiness wait. Pair with . /// void LaunchDockerServices(params string[] services) + { + var serviceList = string.Join(" ", services); + Log.Information("Starting docker services: {Services}", serviceList); + + // Pass each service as a separate argument to avoid quoting issues + ComposeUp("compose up -d " + serviceList, serviceList); + } + + /// + /// Runs a `docker compose up` invocation, retrying a failed attempt. + /// + /// `docker compose up` contacts the registry for any image not already cached, and GitHub's + /// runners time out against Docker Hub often enough to redden main on their own. On 2026-08-03, + /// three of the four red main runs were exactly this and nothing else — CIOtel twice and CIMQTT5 + /// once, each dying in under a minute on + /// `Get "https://registry-1.docker.io/v2/": context deadline exceeded` before a single test ran. + /// A pull timeout is transient and says nothing about the commit that triggered it, so it should + /// cost a retry rather than a red build. + /// + /// Every failure is retried, not just registry-shaped ones: matching on the message text + /// would be guessing at a moving target, and a genuinely broken compose file still fails — it + /// just takes tries to get there, and the exception it throws + /// carries the real reason either way. Each failed attempt logs the tool's own output, so the + /// job log always says which it was instead of leaving the reader to infer it. + /// + void ComposeUp(string args, string serviceList) { bool IsToolAvailable(string toolName) { @@ -42,15 +73,40 @@ bool IsToolAvailable(string toolName) string toolName = new List { "docker", "podman" } .FirstOrDefault(IsToolAvailable) ?? "docker"; - var serviceList = string.Join(" ", services); - Log.Information("Starting docker services: {Services}", serviceList); + for (var attempt = 1;; attempt++) + { + var process = ProcessTasks + .StartProcess(toolName, args, logOutput: false, logInvocation: false) + .AssertWaitForExit(); - // Pass each service as a separate argument to avoid quoting issues - var args = "compose up -d " + serviceList; - ProcessTasks - .StartProcess(toolName, args, logOutput: false, logInvocation: false) - .AssertWaitForExit() - .AssertZeroExitCode(); + if (process.ExitCode == 0) + { + if (attempt > 1) + { + Log.Information("docker compose up succeeded for {Services} on attempt {Attempt}", + serviceList, attempt); + } + + return; + } + + var output = string.Join(Environment.NewLine, process.Output.Select(x => x.Text)); + + // Out of attempts: let AssertZeroExitCode throw with the tool's own message. + if (attempt >= DockerComposeAttempts) + { + Log.Error("docker compose up failed for {Services} after {Attempts} attempts:{NewLine}{Output}", + serviceList, DockerComposeAttempts, Environment.NewLine, output); + process.AssertZeroExitCode(); + return; + } + + var delay = TimeSpan.FromSeconds(5 * attempt); + Log.Warning( + "docker compose up failed for {Services} (attempt {Attempt} of {Attempts}), retrying in {Delay}s:{NewLine}{Output}", + serviceList, attempt, DockerComposeAttempts, delay.TotalSeconds, Environment.NewLine, output); + Thread.Sleep(delay); + } } /// diff --git a/build/build.cs b/build/build.cs index 9bcba3b5c..01973afe7 100644 --- a/build/build.cs +++ b/build/build.cs @@ -345,21 +345,9 @@ partial class Build : NukeBuild Target DockerUp => _ => _ .Executes(() => { - bool IsToolAvailable(string toolName) - { - try - { ToolPathResolver.GetPathExecutable(toolName); - return true; } - catch (ArgumentException) - { return false; } - } - - string toolName = new List { "docker", "podman" } - .FirstOrDefault(IsToolAvailable) ?? "docker"; - ProcessTasks - .StartProcess(toolName, "compose up -d", logOutput: false) - .AssertWaitForExit() - .AssertZeroExitCode(); + // Shares ComposeUp with the CI targets so this path gets the same registry-timeout + // retry — it pulls every image in the compose file, so it is the most exposed of all. + ComposeUp("compose up -d", "all services"); WaitForDatabaseToBeReady(); });