Copr API improvements to retries and diagnostics

ymir/agents/backport_agent.py

@@ -568,6 +568,11 @@ async def run_build_agent(state):
             if build_result.is_timeout:
                 logger.info(f"Build timed out for {state.jira_issue}, proceeding")
                 return "update_release"
+            if build_result.is_infra_error:
+                logger.error(f"Copr infrastructure error for {state.jira_issue}: {build_result.error}")
+                state.backport_result.success = False
+                state.backport_result.error = build_result.error or "Copr API infrastructure error"
+                return "comment_in_jira"
             state.attempts_remaining -= 1
             if state.attempts_remaining <= 0:
                 state.backport_result.success = False

ymir/agents/merge_request_agent.py

@@ -141,7 +141,6 @@ async def run_workflow(merge_request_url):
 
         async with mcp_tools(os.environ["MCP_GATEWAY_URL"]) as gateway_tools:
             merge_request_agent = create_merge_request_agent(gateway_tools, local_tool_options)
-            build_agent = create_build_agent(gateway_tools, local_tool_options)
 
             workflow = Workflow(State, name="MergeRequestWorkflow")
 
@@ -220,6 +219,7 @@ async def run_merge_request_agent(state):
                 return "comment_in_mr"
 
             async def run_build_agent(state):
+                build_agent = create_build_agent(gateway_tools, local_tool_options)
                 response = await build_agent.run(
                     render_template(
                         get_build_prompt(),
@@ -238,6 +238,11 @@ async def run_build_agent(state):
                 if build_result.is_timeout:
                     logger.info(f"Build timed out for {state.jira_issue}, proceeding")
                     return "stage_changes"
+                if build_result.is_infra_error:
+                    logger.error(f"Copr infrastructure error for {state.jira_issue}: {build_result.error}")
+                    state.mr_update_result.success = False
+                    state.mr_update_result.error = build_result.error or "Copr API infrastructure error"
+                    return "comment_in_mr"
                 state.attempts_remaining -= 1
                 if state.attempts_remaining <= 0:
                     state.mr_update_result.success = False

ymir/agents/prompts/build/instructions.j2

@@ -7,7 +7,9 @@ just return the error message. Otherwise, use the `extract_log_snippets` tool wi
 `log_path` pointing to the location of `builder-live.log` and try to identify
 the build failure. If not found, try the same with `root.log`. Summarize the findings
 and return them as `error`. Set `is_timeout` to the value of the `is_timeout` field
-from the `build_package` tool output.
+from the `build_package` tool output. If the build_package tool itself returned an error (e.g. Copr API error,
+project setup failure, or build submission failure) rather than a build failure with logs,
+set `is_infra_error` to `true`.
 
 General instructions:
 

ymir/agents/reasoning_agent/_runner.py

@@ -621,7 +621,17 @@ async def _run_unconstrained(self, evaluation: RequirementEvaluation) -> ChatMod
             self._tool_call_cycle_checker.register(tool_call_msg)
             if self._tool_call_cycle_checker.cycle_found:
                 self._tool_call_cycle_checker.reset()
-                await self._state.memory.add_many(response.output)
+                cycle_tool_results = [
+                    ToolMessage(
+                        MessageToolResultContent(
+                            tool_name=tc.tool_name,
+                            tool_call_id=tc.id,
+                            result="Tool call was not executed due to a cycle detection.",
+                        )
+                    )
+                    for tc in tool_calls
+                ]
+                await self._state.memory.add_many([*response.output, *cycle_tool_results])
 
                 await self._state.memory.add(
                     UserMessage(

ymir/agents/rebase_agent.py

@@ -158,7 +158,6 @@ async def run_workflow(
             os.environ["MCP_GATEWAY_URL"], call_meta={"jira_issue": jira_issue}
         ) as gateway_tools:
             rebase_agent = create_rebase_agent(gateway_tools, local_tool_options)
-            build_agent = create_build_agent(gateway_tools, local_tool_options)
             log_agent = create_log_agent(gateway_tools, local_tool_options)
 
             workflow = Workflow(State, name="RebaseWorkflow")
@@ -225,6 +224,7 @@ async def run_rebase_agent(state):
                 return "comment_in_jira"
 
             async def run_build_agent(state):
+                build_agent = create_build_agent(gateway_tools, local_tool_options)
                 response = await build_agent.run(
                     render_template(
                         get_build_prompt(),
@@ -243,6 +243,11 @@ async def run_build_agent(state):
                 if build_result.is_timeout:
                     logger.info(f"Build timed out for {state.jira_issue}, proceeding")
                     return "update_release"
+                if build_result.is_infra_error:
+                    logger.error(f"Copr infrastructure error for {state.jira_issue}: {build_result.error}")
+                    state.rebase_result.success = False
+                    state.rebase_result.error = build_result.error or "Copr API infrastructure error"
+                    return "comment_in_jira"
                 state.attempts_remaining -= 1
                 if state.attempts_remaining <= 0:
                     state.rebase_result.success = False

ymir/common/models.py

@@ -584,6 +584,10 @@ class BuildOutputSchema(BaseModel):
     success: bool = Field(description="Whether the build was successfully completed")
     error: str | None = Field(description="Specific details about an error")
     is_timeout: bool = Field(default=False, description="Whether the build failed due to a timeout")
+    is_infra_error: bool = Field(
+        default=False,
+        description="Whether the failure was caused by a Copr API or infrastructure error, not a build error",
+    )
 
 
 # ============================================================================

ymir/tools/privileged/copr.py

@@ -1,6 +1,7 @@
 import asyncio
 import gzip
 import logging
+import random
 import time
 from pathlib import Path
 from urllib.parse import urljoin, urlparse
@@ -16,6 +17,7 @@
     ToolRunOptions,
 )
 from copr.v3 import BuildProxy, ProjectChrootProxy, ProjectProxy
+from copr.v3.exceptions import CoprException
 from pydantic import BaseModel, Field
 
 from ymir.common import load_rhel_config
@@ -41,11 +43,43 @@
     "x86_64",
 }
 COPR_ARCH_PREFERENCE = ("x86_64", "aarch64", "s390x", "ppc64le")
+COPR_API_MAX_RETRIES = 3
+COPR_API_RETRY_BACKOFF_BASE = 5  # seconds
 
 
 logger = logging.getLogger(__name__)
 
 
+def _copr_error_detail(exc: Exception) -> str:
+    if isinstance(exc, CoprException):
+        result = getattr(exc, "result", None)
+        resp = getattr(result, "__response__", None) if result is not None else None
+        if resp is not None:
+            body = resp.text[:500] if resp.text else "(empty body)"
+            return f"HTTP {resp.status_code} from {resp.url}: {body}"
+    return str(exc)
+
+
+async def _copr_api_call(func, *args, **kwargs):
+    """Call a Copr API function with retries on transient errors."""
+    for attempt in range(COPR_API_MAX_RETRIES):
+        try:
+            return await asyncio.to_thread(func, *args, **kwargs)
+        except CoprException as e:
+            if attempt >= COPR_API_MAX_RETRIES - 1:
+                raise
+            delay = COPR_API_RETRY_BACKOFF_BASE * (2**attempt) + random.uniform(0, 1)  # noqa: S311
+            logger.warning(
+                "Copr API error (%s), retrying in %.1fs (attempt %d/%d)",
+                _copr_error_detail(e),
+                delay,
+                attempt + 1,
+                COPR_API_MAX_RETRIES,
+            )
+            await asyncio.sleep(delay)
+    raise RuntimeError("unreachable")
+
+
 class BuildResult(BaseModel):
     success: bool = Field(description="Whether the build succeeded")
     is_timeout: bool = Field(default=False, description="Whether the build failed due to a timeout")
@@ -121,13 +155,13 @@ async def _run(
         try:
             # if the project already exists, nothing is updated, so we have to edit it
             # afterwards unless our chroot is already enabled
-            project = await asyncio.to_thread(project_proxy.add, exist_ok=True, **kwargs)
+            project = await _copr_api_call(project_proxy.add, exist_ok=True, **kwargs)
             if chroot not in project.chroot_repos:
                 # make sure to preserve existing chroots
                 kwargs["chroots"] = sorted(set(project.chroot_repos.keys()) | {chroot})
-                await asyncio.to_thread(project_proxy.edit, **kwargs)
+                await _copr_api_call(project_proxy.edit, **kwargs)
         except Exception as e:
-            raise ToolError(f"Failed to create or update Copr project: {e}") from e
+            raise ToolError(f"Failed to create or update Copr project: {_copr_error_detail(e)}") from e
         if chroot.startswith("custom-"):
             # make sure the chroot has access to corresponding buildroot repository
             logger.info(f"Connecting to Copr API to update chroot configuration for {chroot}")
@@ -140,7 +174,7 @@ async def _run(
                 f"brewroot/repos/{dist_git_branch}-z-build/latest/{build_arch}",
             )
             try:
-                chroot_config = await asyncio.to_thread(
+                chroot_config = await _copr_api_call(
                     chroot_proxy.get,
                     ownername=copr_user,
                     projectname=jira_issue,
@@ -162,27 +196,27 @@ async def _run(
                         set(chroot_config.additional_packages) | {build_group}
                     )
                 if kwargs:
-                    await asyncio.to_thread(
+                    await _copr_api_call(
                         chroot_proxy.edit,
                         ownername=copr_user,
                         projectname=jira_issue,
                         chrootname=chroot,
                         **kwargs,
                     )
             except Exception as e:
-                raise ToolError(f"Failed to update Copr chroot: {e}") from e
+                raise ToolError(f"Failed to update Copr chroot: {_copr_error_detail(e)}") from e
         logger.info(f"Connecting to Copr API to submit build for {srpm_path}")
         build_proxy = BuildProxy({"username": copr_user, **COPR_CONFIG})
         try:
-            build = await asyncio.to_thread(
+            build = await _copr_api_call(
                 build_proxy.create_from_file,
                 ownername=copr_user,
                 projectname=jira_issue,
                 path=str(srpm_path),
                 buildopts={"chroots": [chroot], "timeout": COPR_BUILD_TIMEOUT},
             )
         except Exception as e:
-            raise ToolError(f"Failed to submit Copr build: {e}") from e
+            raise ToolError(f"Failed to submit Copr build: {_copr_error_detail(e)}") from e
         else:
             logger.info(f"{jira_issue}: build of {srpm_path} in {chroot} started: {build.id:08d}")
 

ymir/tools/privileged/tests/unit/test_copr.py

@@ -7,14 +7,18 @@
 import pytest
 from beeai_framework.tools import ToolError
 from copr.v3 import BuildProxy, ProjectChrootProxy, ProjectProxy
+from copr.v3.exceptions import CoprException
 from flexmock import flexmock
 
 from ymir.tools.privileged import copr as copr_tools
 from ymir.tools.privileged.copr import (
+    COPR_API_MAX_RETRIES,
     COPR_BUILD_TIMEOUT,
     COPR_PROJECT_LIFETIME,
     BuildPackageTool,
     DownloadArtifactsTool,
+    _copr_api_call,
+    _copr_error_detail,
 )
 
 
@@ -199,3 +203,75 @@ async def read():
         assert not path.is_file()
     else:
         assert path.read_bytes() == content
+
+
+def test_copr_error_detail_copr_exception_with_response():
+    resp = flexmock(status_code=500, url="https://copr.example.com/api/foo", text="Internal Server Error")
+    exc = CoprException()
+    exc.result = flexmock(__response__=resp)
+    assert _copr_error_detail(exc) == "HTTP 500 from https://copr.example.com/api/foo: Internal Server Error"
+
+
+def test_copr_error_detail_copr_exception_without_response():
+    exc = CoprException("something went wrong")
+    assert _copr_error_detail(exc) == "something went wrong"
+
+
+def test_copr_error_detail_generic_exception():
+    exc = RuntimeError("generic error")
+    assert _copr_error_detail(exc) == "generic error"
+
+
+@pytest.mark.asyncio
+async def test_copr_api_call_success():
+    result = await _copr_api_call(lambda x, key=None: (x, key), 1, key="val")
+    assert result == (1, "val")
+
+
+@pytest.mark.asyncio
+async def test_copr_api_call_retries_on_copr_exception():
+    async def noop_sleep(*_):
+        return
+
+    flexmock(asyncio).should_receive("sleep").replace_with(noop_sleep)
+
+    call_count = 0
+
+    def failing_then_ok():
+        nonlocal call_count
+        call_count += 1
+        if call_count < COPR_API_MAX_RETRIES:
+            raise CoprException("transient")
+        return "recovered"
+
+    result = await _copr_api_call(failing_then_ok)
+    assert result == "recovered"
+    assert call_count == COPR_API_MAX_RETRIES
+
+
+@pytest.mark.asyncio
+async def test_copr_api_call_raises_after_max_retries():
+    async def noop_sleep(*_):
+        return
+
+    flexmock(asyncio).should_receive("sleep").replace_with(noop_sleep)
+
+    def always_fails():
+        raise CoprException("persistent failure")
+
+    with pytest.raises(CoprException, match="persistent failure"):
+        await _copr_api_call(always_fails)
+
+
+@pytest.mark.asyncio
+async def test_copr_api_call_no_retry_on_non_copr_exception():
+    call_count = 0
+
+    def non_copr_error():
+        nonlocal call_count
+        call_count += 1
+        raise ValueError("not a copr error")
+
+    with pytest.raises(ValueError, match="not a copr error"):
+        await _copr_api_call(non_copr_error)
+    assert call_count == 1