diff --git a/src/conductor/providers/_schema.py b/src/conductor/providers/_schema.py new file mode 100644 index 00000000..dc85e2a1 --- /dev/null +++ b/src/conductor/providers/_schema.py @@ -0,0 +1,288 @@ +"""Shared, provider-neutral output-schema builders. + +This private module contains the core logic for turning +:class:`~conductor.config.schema.OutputField` definitions into +JSON-Schema fragments and prompt-facing schema fragments. Each provider +wraps these helpers with its own error type and message formatting. +""" + +from __future__ import annotations + +from typing import Any + +from conductor.config.schema import OutputField + + +class SchemaDepthError(Exception): + """Raised when output schema nesting exceeds the configured maximum depth.""" + + def __init__(self, depth: int, max_depth: int) -> None: + """Initialize with the depth that was exceeded. + + Args: + depth: The current nesting depth that triggered the limit. + max_depth: The maximum allowed nesting depth. + """ + super().__init__(f"Schema nesting depth {depth} exceeds maximum of {max_depth} levels") + self.depth = depth + self.max_depth = max_depth + + +def _check_depth(depth: int, max_depth: int) -> None: + """Raise :class:`SchemaDepthError` if ``depth > max_depth``. + + Args: + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + if depth > max_depth: + raise SchemaDepthError(depth, max_depth) + + +def build_json_schema_field( + field: OutputField, *, depth: int = 0, max_depth: int = 10 +) -> dict[str, Any]: + """Build a JSON-Schema fragment for a single ``OutputField``. + + The fragment contains ``type`` and optionally ``description``, + ``properties`` + ``required`` (for objects), or ``items`` (for arrays). + + Args: + field: The output field definition. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + + Returns: + A JSON-Schema fragment dictionary. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + _check_depth(depth, max_depth) + + schema: dict[str, Any] = {"type": field.type} + + if field.description: + schema["description"] = field.description + + if field.type == "object" and field.properties: + schema["properties"] = build_json_schema_properties( + field.properties, depth=depth + 1, max_depth=max_depth + ) + schema["required"] = list(field.properties.keys()) + + if field.type == "array" and field.items: + schema["items"] = build_json_schema_field(field.items, depth=depth + 1, max_depth=max_depth) + + return schema + + +def build_json_schema_properties( + fields: dict[str, OutputField], *, depth: int = 0, max_depth: int = 10 +) -> dict[str, Any]: + """Build a JSON-Schema ``properties`` mapping from named ``OutputField`` definitions. + + Args: + fields: Mapping from field name to output field definition. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + + Returns: + A JSON-Schema ``properties`` object. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + _check_depth(depth, max_depth) + + return { + name: build_json_schema_field(field, depth=depth, max_depth=max_depth) + for name, field in fields.items() + } + + +def build_prompt_schema_field( + field: OutputField, + *, + field_name: str | None, + depth: int = 0, + max_depth: int = 10, + description_fallback: bool = False, +) -> dict[str, Any]: + """Build a prompt-facing schema fragment for a single ``OutputField``. + + When ``description_fallback`` is true and the field has no explicit + description and ``field_name`` is provided, the description is filled + with ``"The {field_name} field"``. Array items are built with + ``field_name=None`` so they do not gain a fallback description. + + Args: + field: The output field definition. + field_name: The field name used for the fallback description, or + ``None`` to suppress the fallback. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + description_fallback: Whether to synthesize a description when one + is not explicitly set. + + Returns: + A prompt-facing schema fragment dictionary. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + _check_depth(depth, max_depth) + + description = field.description + if description_fallback and description is None and field_name is not None: + description = f"The {field_name} field" + + schema: dict[str, Any] = {"type": field.type} + if description: + schema["description"] = description + + if field.type == "object" and field.properties: + schema["properties"] = build_prompt_schema_properties( + field.properties, + depth=depth + 1, + max_depth=max_depth, + description_fallback=description_fallback, + ) + schema["required"] = list(field.properties.keys()) + + if field.type == "array" and field.items: + schema["items"] = build_prompt_schema_field( + field.items, + field_name=None, + depth=depth + 1, + max_depth=max_depth, + description_fallback=description_fallback, + ) + + return schema + + +def build_prompt_schema_properties( + fields: dict[str, OutputField], + *, + depth: int = 0, + max_depth: int = 10, + description_fallback: bool = False, +) -> dict[str, Any]: + """Build a prompt-facing schema mapping from named ``OutputField`` definitions. + + Args: + fields: Mapping from field name to output field definition. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + description_fallback: Whether to synthesize a description when one + is not explicitly set. + + Returns: + A prompt-facing schema mapping. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + _check_depth(depth, max_depth) + + return { + name: build_prompt_schema_field( + field, + field_name=name, + depth=depth, + max_depth=max_depth, + description_fallback=description_fallback, + ) + for name, field in fields.items() + } + + +def build_hermes_legacy_prompt_schema( + fields: dict[str, OutputField], *, depth: int = 0, max_depth: int = 10 +) -> dict[str, Any]: + """Build the Hermes legacy prompt-facing schema mapping. + + This matches the legacy Hermes provider behavior: descriptions fall back + to ``"The {field_name} field"`` at the top level, but array items do not + receive a fallback description. Unlike the generic prompt builder, object + items inside arrays are emitted with ``properties`` but no ``required`` + key, and array-of-array items collapse to ``{"type": "array"}`` without + further recursion (an explicit item description is still kept). + + Args: + fields: Mapping from field name to output field definition. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + + Returns: + The Hermes legacy prompt-facing schema mapping. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + _check_depth(depth, max_depth) + + result: dict[str, Any] = {} + for field_name, field_def in fields.items(): + field_schema: dict[str, Any] = {"type": field_def.type} + + if field_def.description: + field_schema["description"] = field_def.description + else: + field_schema["description"] = f"The {field_name} field" + + if field_def.type == "object" and field_def.properties: + field_schema["properties"] = build_hermes_legacy_prompt_schema( + field_def.properties, depth=depth + 1, max_depth=max_depth + ) + field_schema["required"] = list(field_def.properties.keys()) + + if field_def.type == "array" and field_def.items: + field_schema["items"] = _build_hermes_legacy_item_schema( + field_def.items, depth=depth + 1, max_depth=max_depth + ) + + result[field_name] = field_schema + + return result + + +def _build_hermes_legacy_item_schema( + field: OutputField, *, depth: int, max_depth: int +) -> dict[str, Any]: + """Build the Hermes legacy schema fragment for an array item. + + Object items include ``properties`` but no ``required``. Array items of + any kind collapse to the bare ``{"type": "array"}`` shape with no + inner recursion (an explicit item description is still kept). + + Args: + field: The array item output field definition. + depth: Current nesting depth. + max_depth: Maximum allowed nesting depth. + + Returns: + The Hermes legacy item schema fragment. + + Raises: + SchemaDepthError: When the depth limit is exceeded. + """ + item_schema: dict[str, Any] = {"type": field.type} + + if field.description: + item_schema["description"] = field.description + + if field.type == "object" and field.properties: + _check_depth(depth, max_depth) + # Pinned legacy counting: the array itself advanced depth by one when + # calling this helper, so the item's properties must recurse at the + # same depth (not depth + 1) to match pre-refactor Hermes behavior. + item_schema["properties"] = build_hermes_legacy_prompt_schema( + field.properties, depth=depth, max_depth=max_depth + ) + + return item_schema diff --git a/src/conductor/providers/claude.py b/src/conductor/providers/claude.py index c7083f10..ff1962cd 100644 --- a/src/conductor/providers/claude.py +++ b/src/conductor/providers/claude.py @@ -35,6 +35,7 @@ extract_tool_result_text, format_tool_arguments, ) +from conductor.providers._schema import SchemaDepthError, build_json_schema_properties from conductor.providers.base import ( AgentOutput, AgentProvider, @@ -2316,8 +2317,6 @@ def _build_json_schema_properties( ) -> dict[str, Any]: """Build JSON Schema properties from OutputField definitions. - Recursively handles nested objects and arrays with depth limiting. - Args: schema: Dictionary mapping field names to OutputField definitions. depth: Current nesting depth (for recursion safety). @@ -2328,97 +2327,15 @@ def _build_json_schema_properties( Raises: ValidationError: If schema nesting exceeds max depth. """ - if depth > self._max_schema_depth: - raise ValidationError( - f"Schema nesting depth exceeds maximum of {self._max_schema_depth} levels", - suggestion="Simplify your output schema to reduce nesting depth", + try: + return build_json_schema_properties( + schema, depth=depth, max_depth=self._max_schema_depth ) - - properties: dict[str, Any] = {} - - for field_name, field_def in schema.items(): - prop: dict[str, Any] = { - "type": self._map_type_to_json_schema(field_def.type), - } - - if field_def.description: - prop["description"] = field_def.description - - # Handle nested object schemas - if field_def.type == "object" and field_def.properties: - prop["properties"] = self._build_json_schema_properties( - field_def.properties, depth=depth + 1 - ) - # All properties in OutputField schemas are required - # (OutputField has no 'required' attribute, all fields are mandatory) - prop["required"] = list(field_def.properties.keys()) - - # Handle array schemas with item definitions - if field_def.type == "array" and field_def.items: - items_schema = self._build_single_field_schema(field_def.items, depth=depth + 1) - prop["items"] = items_schema - - properties[field_name] = prop - - return properties - - def _build_single_field_schema(self, field: OutputField, depth: int = 0) -> dict[str, Any]: - """Build JSON Schema for a single field (used for array items). - - Args: - field: The OutputField definition. - depth: Current nesting depth (for recursion safety). - - Returns: - JSON Schema definition for the field. - - Raises: - ValidationError: If schema nesting exceeds max depth. - """ - if depth > self._max_schema_depth: + except SchemaDepthError as exc: raise ValidationError( f"Schema nesting depth exceeds maximum of {self._max_schema_depth} levels", suggestion="Simplify your output schema to reduce nesting depth", - ) - - schema: dict[str, Any] = { - "type": self._map_type_to_json_schema(field.type), - } - - if field.description: - schema["description"] = field.description - - # Handle nested objects in array items - if field.type == "object" and field.properties: - schema["properties"] = self._build_json_schema_properties( - field.properties, depth=depth + 1 - ) - # All properties are required - schema["required"] = list(field.properties.keys()) - - # Handle nested arrays (array of arrays) - if field.type == "array" and field.items: - schema["items"] = self._build_single_field_schema(field.items, depth=depth + 1) - - return schema - - def _map_type_to_json_schema(self, field_type: str) -> str: - """Map OutputField type to JSON Schema type. - - Args: - field_type: The OutputField type string. - - Returns: - Corresponding JSON Schema type. - """ - type_mapping = { - "string": "string", - "number": "number", - "boolean": "boolean", - "array": "array", - "object": "object", - } - return type_mapping.get(field_type, "string") + ) from exc def _extract_output( self, response: Any, output_schema: dict[str, OutputField] | None diff --git a/src/conductor/providers/claude_agent_sdk.py b/src/conductor/providers/claude_agent_sdk.py index 566df5ae..7b7e3816 100644 --- a/src/conductor/providers/claude_agent_sdk.py +++ b/src/conductor/providers/claude_agent_sdk.py @@ -9,6 +9,11 @@ from typing import TYPE_CHECKING, Any, Final, cast from conductor.exceptions import ProviderError +from conductor.providers._schema import ( + SchemaDepthError, + build_json_schema_field, + build_json_schema_properties, +) from conductor.providers.base import AgentOutput, AgentProvider, EventCallback from conductor.providers.capabilities import ProviderCapabilities @@ -28,39 +33,26 @@ def _build_field_schema(field: OutputField, depth: int = 0) -> dict[str, Any]: - """Translate a single ``OutputField`` into a JSON-Schema fragment. + """Thin delegate to the shared JSON-Schema field builder. - Recursively descends into object properties and array items. The depth - cap (10) protects against pathological YAML that would otherwise blow - the Python recursion limit during schema construction. - - Args: - field: The output field definition from the workflow YAML. - depth: Current recursion depth (internal — do not pass). - - Returns: - A JSON-Schema fragment matching the field's type and constraints. - - Raises: - ProviderError: If recursion exceeds 10 nested levels. + Keep this entry point intact because tests import it directly. Depth + errors from the core are translated to the historical ProviderError + message so downstream assertions stay stable. """ - if depth > 10: - raise ProviderError("Output schema nesting exceeds 10 levels") - - schema: dict[str, Any] = {"type": field.type} - if field.description: - schema["description"] = field.description - if field.type == "object" and field.properties: - schema["properties"] = _build_properties(field.properties, depth + 1) - schema["required"] = list(field.properties.keys()) - if field.type == "array" and field.items: - schema["items"] = _build_field_schema(field.items, depth + 1) - return schema + try: + return build_json_schema_field(field, depth=depth, max_depth=10) + except SchemaDepthError as exc: + # Pinned message: downstream tests assert the exact text. + raise ProviderError("Output schema nesting exceeds 10 levels") from exc def _build_properties(fields: dict[str, OutputField], depth: int = 0) -> dict[str, Any]: - """Translate a mapping of named ``OutputField`` definitions into JSON-Schema properties.""" - return {name: _build_field_schema(field, depth) for name, field in fields.items()} + """Thin delegate to the shared JSON-Schema properties builder.""" + try: + return build_json_schema_properties(fields, depth=depth, max_depth=10) + except SchemaDepthError as exc: + # Pinned message: downstream tests assert the exact text. + raise ProviderError("Output schema nesting exceeds 10 levels") from exc def _build_output_format(output: dict[str, OutputField]) -> dict[str, Any]: diff --git a/src/conductor/providers/copilot.py b/src/conductor/providers/copilot.py index d9039380..6123a76f 100644 --- a/src/conductor/providers/copilot.py +++ b/src/conductor/providers/copilot.py @@ -23,6 +23,7 @@ extract_tool_result_text, format_tool_arguments, ) +from conductor.providers._schema import SchemaDepthError, build_prompt_schema_properties from conductor.providers.base import ( AgentOutput, AgentProvider, @@ -1526,59 +1527,18 @@ def _build_prompt_schema( self, schema: dict[str, OutputField], depth: int = 0 ) -> dict[str, Any]: """Build a prompt-facing schema description from OutputField definitions.""" - if depth > self._max_schema_depth: - raise ValidationError( - f"Schema nesting depth exceeds maximum of {self._max_schema_depth} levels", - suggestion="Simplify your output schema to reduce nesting depth", + try: + return build_prompt_schema_properties( + schema, + depth=depth, + max_depth=self._max_schema_depth, + description_fallback=True, ) - return { - field_name: self._build_prompt_field_schema(field_name, field_def, depth=depth) - for field_name, field_def in schema.items() - } - - def _build_prompt_field_schema( - self, - field_name: str, - field_def: OutputField, - depth: int = 0, - ) -> dict[str, Any]: - """Build a prompt-facing schema description for a named field.""" - schema: dict[str, Any] = { - "type": field_def.type, - "description": field_def.description or f"The {field_name} field", - } - - if field_def.type == "object" and field_def.properties: - schema["properties"] = self._build_prompt_schema(field_def.properties, depth=depth + 1) - schema["required"] = list(field_def.properties.keys()) - - if field_def.type == "array" and field_def.items: - schema["items"] = self._build_prompt_item_schema(field_def.items, depth=depth + 1) - - return schema - - def _build_prompt_item_schema(self, field_def: OutputField, depth: int = 0) -> dict[str, Any]: - """Build a prompt-facing schema description for an array item.""" - if depth > self._max_schema_depth: + except SchemaDepthError as exc: raise ValidationError( f"Schema nesting depth exceeds maximum of {self._max_schema_depth} levels", suggestion="Simplify your output schema to reduce nesting depth", - ) - schema: dict[str, Any] = { - "type": field_def.type, - } - - if field_def.description: - schema["description"] = field_def.description - - if field_def.type == "object" and field_def.properties: - schema["properties"] = self._build_prompt_schema(field_def.properties, depth=depth + 1) - schema["required"] = list(field_def.properties.keys()) - - if field_def.type == "array" and field_def.items: - schema["items"] = self._build_prompt_item_schema(field_def.items, depth=depth + 1) - - return schema + ) from exc def _log_event_verbose( self, diff --git a/src/conductor/providers/hermes.py b/src/conductor/providers/hermes.py index 9a695486..b219f65f 100644 --- a/src/conductor/providers/hermes.py +++ b/src/conductor/providers/hermes.py @@ -23,6 +23,10 @@ from conductor.exceptions import ProviderError, ValidationError from conductor.executor.output import parse_json_output, validate_output +from conductor.providers._schema import ( + SchemaDepthError, + build_hermes_legacy_prompt_schema, +) from conductor.providers.base import AgentOutput, AgentProvider, EventCallback from conductor.providers.capabilities import ProviderCapabilities from conductor.providers.reasoning import ReasoningEffort, resolve_reasoning_effort @@ -639,33 +643,18 @@ async def _wait_for_event(event: asyncio.Event) -> None: def _build_prompt_schema(schema: dict[str, Any], depth: int = 0) -> dict[str, Any]: - """Build a prompt-facing schema description from OutputField definitions.""" - if depth > _MAX_SCHEMA_DEPTH: + """Build a prompt-facing schema description from OutputField definitions. + + Wraps the shared Hermes legacy builder, converting core depth errors into + the provider's ValidationError with the exact legacy message and suggestion. + """ + try: + return build_hermes_legacy_prompt_schema(schema, depth=depth, max_depth=_MAX_SCHEMA_DEPTH) + except SchemaDepthError as exc: raise ValidationError( f"Schema nesting depth exceeds maximum of {_MAX_SCHEMA_DEPTH} levels", suggestion="Simplify your output schema to reduce nesting depth", - ) - result: dict[str, Any] = {} - for field_name, field_def in schema.items(): - field_schema: dict[str, Any] = {"type": field_def.type} - if field_def.description: - field_schema["description"] = field_def.description - else: - field_schema["description"] = f"The {field_name} field" - if field_def.type == "object" and field_def.properties: - field_schema["properties"] = _build_prompt_schema(field_def.properties, depth + 1) - field_schema["required"] = list(field_def.properties.keys()) - if field_def.type == "array" and field_def.items: - item_schema: dict[str, Any] = {"type": field_def.items.type} - if field_def.items.description: - item_schema["description"] = field_def.items.description - if field_def.items.type == "object" and field_def.items.properties: - item_schema["properties"] = _build_prompt_schema( - field_def.items.properties, depth + 1 - ) - field_schema["items"] = item_schema - result[field_name] = field_schema - return result + ) from exc def _build_recovery_prompt(parse_error: str, original_response: str, schema: dict[str, Any]) -> str: diff --git a/tests/test_providers/test_claude.py b/tests/test_providers/test_claude.py index 672f5668..3be52ccd 100644 --- a/tests/test_providers/test_claude.py +++ b/tests/test_providers/test_claude.py @@ -648,6 +648,39 @@ def test_build_tools_for_simple_schema( assert tools[0]["input_schema"]["properties"]["score"]["type"] == "number" assert set(tools[0]["input_schema"]["required"]) == {"result", "score"} + @patch("conductor.providers.claude.ANTHROPIC_SDK_AVAILABLE", True) + @patch("conductor.providers.claude.AsyncAnthropic") + @patch("conductor.providers.claude.anthropic") + def test_build_tools_schema_depth_limit_direct( + self, mock_anthropic_module: Mock, mock_anthropic_class: Mock + ) -> None: + """Directly assert the private builder raises ValidationError at 11-deep nesting. + + Requirement: schema nesting beyond the configured max depth (10 levels) must + surface a ValidationError with the exact message "Schema nesting depth exceeds + maximum of 10 levels" so callers cannot construct unbounded recursive schemas. + """ + mock_anthropic_module.__version__ = "0.77.0" + mock_client = Mock() + mock_client.models.list = AsyncMock(return_value=Mock(data=[])) + mock_anthropic_class.return_value = mock_client + + provider = ClaudeProvider() + + def nested(level: int) -> OutputField: + if level == 0: + return OutputField(type="string") + return OutputField( + type="object", + properties={"nested": nested(level - 1)}, + ) + + with pytest.raises( + ValidationError, + match="Schema nesting depth exceeds maximum of 10 levels", + ): + provider._build_json_schema_properties({"root": nested(11)}) + class TestConcurrentExecution: """Tests for concurrent execution scenarios.""" diff --git a/tests/test_providers/test_claude_agent_sdk.py b/tests/test_providers/test_claude_agent_sdk.py index 563194b0..d655630f 100644 --- a/tests/test_providers/test_claude_agent_sdk.py +++ b/tests/test_providers/test_claude_agent_sdk.py @@ -408,6 +408,8 @@ def test_array_of_objects_schema(self) -> None: def test_depth_limit_raises(self) -> None: from conductor.providers.claude_agent_sdk import _build_field_schema + # Pinned requirement: the exact ProviderError message must stay + # "nesting exceeds 10" because the downstream contract is fixed. field_def = OutputField(type="string") for _ in range(12): field_def = OutputField(type="object", properties={"nested": field_def}) diff --git a/tests/test_providers/test_copilot.py b/tests/test_providers/test_copilot.py index 658fb4f5..f3f706cb 100644 --- a/tests/test_providers/test_copilot.py +++ b/tests/test_providers/test_copilot.py @@ -441,6 +441,29 @@ def test_build_prompt_schema_recurses_through_nested_fields(self) -> None: assert schema["plan"]["required"] == ["questions", "areas", "sources"] assert schema["summary"]["description"] == "The summary field" + def test_build_prompt_schema_depth_limit_enforced(self) -> None: + """Excessively nested schemas raise ValidationError with the pinned message. + + The Copilot provider must convert the shared core's SchemaDepthError + into the exact existing ValidationError message and suggestion so + callers see stable behavior. + """ + from conductor.config.schema import OutputField + from conductor.exceptions import ValidationError + + provider = CopilotProvider(mock_handler=stub_handler) + + # Build an 11-level nested object chain to exceed the default depth limit of 10. + inner: OutputField = OutputField(type="string") + for _ in range(11): + inner = OutputField(type="object", properties={"nested": inner}) + + with pytest.raises(ValidationError) as exc_info: + provider._build_prompt_schema({"root": inner}) + + assert "Schema nesting depth exceeds maximum of 10 levels" in str(exc_info.value) + assert exc_info.value.suggestion == "Simplify your output schema to reduce nesting depth" + @pytest.mark.asyncio async def test_execute_appends_nested_schema_to_prompt( self, monkeypatch: pytest.MonkeyPatch diff --git a/tests/test_providers/test_hermes.py b/tests/test_providers/test_hermes.py index 9beb9a11..a74360fa 100644 --- a/tests/test_providers/test_hermes.py +++ b/tests/test_providers/test_hermes.py @@ -10,7 +10,7 @@ from conductor.config.schema import AgentDef, OutputField, ReasoningConfig from conductor.exceptions import ProviderError, ValidationError -from conductor.providers.hermes import HermesProvider +from conductor.providers.hermes import _MAX_SCHEMA_DEPTH, HermesProvider, _build_prompt_schema def _make_agent( @@ -834,3 +834,119 @@ def test_real_capabilities_tuple_excludes_max(self) -> None: """Guard against an accidental future widening of the real CAPABILITIES declaration (as opposed to a test mock).""" assert "max" not in HermesProvider.CAPABILITIES.reasoning_effort + + +class TestHermesBuildPromptSchema: + """Tests for the Hermes prompt-schema builder wrapper. + + The wrapper delegates to conductor.providers._schema.build_hermes_legacy_prompt_schema + and converts SchemaDepthError into ValidationError with the exact legacy message and + suggestion. These tests pin the legacy behavior (description fallback, no required + inside array-item objects, collapsed array-of-array items) and the depth boundary. + """ + + def _chain_schema(self, levels: int) -> dict[str, OutputField]: + """Build a chain of nested objects `levels` deep.""" + schema: dict[str, OutputField] = {"leaf": OutputField(type="string")} + for _ in range(levels): + schema = {"nested": OutputField(type="object", properties=schema)} + return schema + + def test_build_prompt_schema_top_level_description_fallback(self) -> None: + """Top-level fields without explicit descriptions get the legacy fallback.""" + schema = {"answer": OutputField(type="string")} + result = _build_prompt_schema(schema) + assert result == {"answer": {"type": "string", "description": "The answer field"}} + + def test_build_prompt_schema_array_item_object_has_no_required(self) -> None: + """Legacy Hermes: array object items include properties but no required.""" + schema = { + "items": OutputField( + type="array", + items=OutputField( + type="object", + properties={ + "key": OutputField(type="string"), + "value": OutputField(type="number"), + }, + ), + ) + } + result = _build_prompt_schema(schema) + assert "required" not in result["items"]["items"] + assert "properties" in result["items"]["items"] + + def test_build_prompt_schema_array_of_arrays_collapsed(self) -> None: + """Legacy Hermes behavior: array-of-array items collapse to bare {type: array}.""" + schema = { + "matrix": OutputField( + type="array", + items=OutputField(type="array", items=OutputField(type="number")), + ) + } + result = _build_prompt_schema(schema) + assert result["matrix"]["items"] == {"type": "array"} + + def test_build_prompt_schema_exceeds_max_depth(self) -> None: + """Depths above _MAX_SCHEMA_DEPTH raise ValidationError with the exact + legacy message and suggestion.""" + # A 12-level object chain reaches depth 11, which exceeds the default max depth of 10. + overly_nested = self._chain_schema(_MAX_SCHEMA_DEPTH + 2) + with pytest.raises(ValidationError) as exc_info: + _build_prompt_schema(overly_nested) + + error = exc_info.value + expected = f"Schema nesting depth exceeds maximum of {_MAX_SCHEMA_DEPTH} levels" + assert error.args[0] == expected + assert error.suggestion == "Simplify your output schema to reduce nesting depth" + + def test_build_prompt_schema_array_item_depth_parity(self) -> None: + """Array object items must not consume an extra depth level. + + Legacy Hermes passed the item's properties recursion the same depth as the + array field itself (depth+1 total). This pins that a chain reaching depth 10 + inside an array item is accepted, while a chain reaching depth 11 raises. + """ + + def chain_in_array_item(levels: int) -> dict[str, OutputField]: + return { + "arr": OutputField( + type="array", + items=OutputField( + type="object", + properties=self._chain_schema(levels), + ), + ) + } + + # _chain_schema(9) reaches depth 10 inside the array item: accepted. + _build_prompt_schema(chain_in_array_item(9)) + + # _chain_schema(10) reaches depth 11: one level too deep. + with pytest.raises(ValidationError, match="exceeds maximum"): + _build_prompt_schema(chain_in_array_item(10)) + + def test_build_prompt_schema_non_object_array_item_at_boundary_accepted(self) -> None: + """Non-object array items must not consume a depth level. + + Requirement: legacy Hermes checked depth only when recursing into object + properties, so a chain reaching exactly _MAX_SCHEMA_DEPTH whose leaf is an + array of scalars (or an array of arrays) was accepted pre-refactor. The + shared builder must preserve that boundary: no error at depth 10, while a + chain one object level deeper still raises. + """ + # 10 nested objects (depths 0..9) ending in non-object array fields at depth 10. + inner: dict[str, OutputField] = { + "tags": OutputField(type="array", items=OutputField(type="string")), + "matrix": OutputField( + type="array", items=OutputField(type="array", items=OutputField(type="number")) + ), + } + for _ in range(_MAX_SCHEMA_DEPTH): + inner = {"nested": OutputField(type="object", properties=inner)} + _build_prompt_schema(inner) + + # One more object level pushes the object chain to depth 11: must raise. + too_deep = {"nested": OutputField(type="object", properties=inner)} + with pytest.raises(ValidationError, match="exceeds maximum"): + _build_prompt_schema(too_deep) diff --git a/tests/test_providers/test_output_schema.py b/tests/test_providers/test_output_schema.py new file mode 100644 index 00000000..ae061fc3 --- /dev/null +++ b/tests/test_providers/test_output_schema.py @@ -0,0 +1,805 @@ +"""Golden regression tests for provider output-schema wrappers. + +These tests pin the exact full-wrapper output of each provider's schema builder +against pre-refactor literals. Any behavioral change in the +shared schema builder or a provider wrapper that alters the serialized JSON will +cause these tests to fail. +""" + +from __future__ import annotations + +import json +from typing import Any +from unittest.mock import AsyncMock, Mock, patch + +from conductor.config.schema import OutputField +from conductor.providers.claude import ClaudeProvider +from conductor.providers.claude_agent_sdk import _build_output_format +from conductor.providers.copilot import CopilotProvider +from conductor.providers.hermes import _build_prompt_schema + +# Shared schema definitions used by every provider baseline below. +rich_schema = { + "string_scalar": OutputField( + type="string", + description="A string scalar field with a description", + ), + "number_scalar": OutputField( + type="number", + ), + "nested_object": OutputField( + type="object", + description="A nested object with properties", + properties={ + "nested_string": OutputField( + type="string", + description="Nested string field", + ), + "nested_number": OutputField( + type="number", + ), + }, + ), + "array_of_scalars": OutputField( + type="array", + description="An array of strings", + items=OutputField( + type="string", + description="A string item", + ), + ), + "array_of_objects": OutputField( + type="array", + description="An array of objects", + items=OutputField( + type="object", + properties={ + "obj_key": OutputField( + type="string", + description="The key", + ), + "obj_val": OutputField( + type="number", + description="The value", + ), + }, + ), + ), + "array_of_arrays": OutputField( + type="array", + description="An array of arrays", + items=OutputField( + type="array", + items=OutputField( + type="number", + description="A number in nested array", + ), + ), + ), +} + +missing_descriptions_schema = { + "string_scalar": OutputField( + type="string", + ), + "nested_object": OutputField( + type="object", + properties={ + "nested_string": OutputField( + type="string", + ), + "nested_number": OutputField( + type="number", + ), + }, + ), + "array_of_scalars": OutputField( + type="array", + items=OutputField( + type="string", + ), + ), + "array_of_objects": OutputField( + type="array", + items=OutputField( + type="object", + properties={ + "obj_key": OutputField( + type="string", + ), + "obj_val": OutputField( + type="number", + ), + }, + ), + ), + "array_of_arrays": OutputField( + type="array", + items=OutputField( + type="array", + items=OutputField( + type="number", + ), + ), + ), +} + + +def _serialize(actual: Any) -> str: + """Serialize the wrapper output using the exact golden format.""" + return json.dumps(actual, indent=2, sort_keys=False) + + +# Expected output for ClaudeProvider._build_tools_for_structured_output(rich_schema). +EXPECTED_CLAUDE_RICH_SCHEMA = """[ + { + "name": "emit_output", + "description": "Emit the structured output for this task", + "input_schema": { + "type": "object", + "properties": { + "string_scalar": { + "type": "string", + "description": "A string scalar field with a description" + }, + "number_scalar": { + "type": "number" + }, + "nested_object": { + "type": "object", + "description": "A nested object with properties", + "properties": { + "nested_string": { + "type": "string", + "description": "Nested string field" + }, + "nested_number": { + "type": "number" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "An array of strings", + "items": { + "type": "string", + "description": "A string item" + } + }, + "array_of_objects": { + "type": "array", + "description": "An array of objects", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The key" + }, + "obj_val": { + "type": "number", + "description": "The value" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "description": "An array of arrays", + "items": { + "type": "array", + "items": { + "type": "number", + "description": "A number in nested array" + } + } + } + }, + "required": [ + "string_scalar", + "number_scalar", + "nested_object", + "array_of_scalars", + "array_of_objects", + "array_of_arrays" + ] + } + } +]""" + +# Expected output for ClaudeProvider._build_tools_for_structured_output( +# missing_descriptions_schema). +EXPECTED_CLAUDE_MISSING_SCHEMA = """[ + { + "name": "emit_output", + "description": "Emit the structured output for this task", + "input_schema": { + "type": "object", + "properties": { + "string_scalar": { + "type": "string" + }, + "nested_object": { + "type": "object", + "properties": { + "nested_string": { + "type": "string" + }, + "nested_number": { + "type": "number" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "items": { + "type": "string" + } + }, + "array_of_objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string" + }, + "obj_val": { + "type": "number" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "required": [ + "string_scalar", + "nested_object", + "array_of_scalars", + "array_of_objects", + "array_of_arrays" + ] + } + } +]""" + +# Expected output for CopilotProvider._build_prompt_schema(rich_schema). +EXPECTED_COPILOT_RICH_SCHEMA = """{ + "string_scalar": { + "type": "string", + "description": "A string scalar field with a description" + }, + "number_scalar": { + "type": "number", + "description": "The number_scalar field" + }, + "nested_object": { + "type": "object", + "description": "A nested object with properties", + "properties": { + "nested_string": { + "type": "string", + "description": "Nested string field" + }, + "nested_number": { + "type": "number", + "description": "The nested_number field" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "An array of strings", + "items": { + "type": "string", + "description": "A string item" + } + }, + "array_of_objects": { + "type": "array", + "description": "An array of objects", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The key" + }, + "obj_val": { + "type": "number", + "description": "The value" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "description": "An array of arrays", + "items": { + "type": "array", + "items": { + "type": "number", + "description": "A number in nested array" + } + } + } +}""" + +# Expected output for CopilotProvider._build_prompt_schema(missing_descriptions_schema). +EXPECTED_COPILOT_MISSING_SCHEMA = """{ + "string_scalar": { + "type": "string", + "description": "The string_scalar field" + }, + "nested_object": { + "type": "object", + "description": "The nested_object field", + "properties": { + "nested_string": { + "type": "string", + "description": "The nested_string field" + }, + "nested_number": { + "type": "number", + "description": "The nested_number field" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "The array_of_scalars field", + "items": { + "type": "string" + } + }, + "array_of_objects": { + "type": "array", + "description": "The array_of_objects field", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The obj_key field" + }, + "obj_val": { + "type": "number", + "description": "The obj_val field" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "description": "The array_of_arrays field", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } +}""" + +# Expected output for Hermes _build_prompt_schema(rich_schema). +EXPECTED_HERMES_RICH_SCHEMA = """{ + "string_scalar": { + "type": "string", + "description": "A string scalar field with a description" + }, + "number_scalar": { + "type": "number", + "description": "The number_scalar field" + }, + "nested_object": { + "type": "object", + "description": "A nested object with properties", + "properties": { + "nested_string": { + "type": "string", + "description": "Nested string field" + }, + "nested_number": { + "type": "number", + "description": "The nested_number field" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "An array of strings", + "items": { + "type": "string", + "description": "A string item" + } + }, + "array_of_objects": { + "type": "array", + "description": "An array of objects", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The key" + }, + "obj_val": { + "type": "number", + "description": "The value" + } + } + } + }, + "array_of_arrays": { + "type": "array", + "description": "An array of arrays", + "items": { + "type": "array" + } + } +}""" + +# Expected output for Hermes _build_prompt_schema(missing_descriptions_schema). +EXPECTED_HERMES_MISSING_SCHEMA = """{ + "string_scalar": { + "type": "string", + "description": "The string_scalar field" + }, + "nested_object": { + "type": "object", + "description": "The nested_object field", + "properties": { + "nested_string": { + "type": "string", + "description": "The nested_string field" + }, + "nested_number": { + "type": "number", + "description": "The nested_number field" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "The array_of_scalars field", + "items": { + "type": "string" + } + }, + "array_of_objects": { + "type": "array", + "description": "The array_of_objects field", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The obj_key field" + }, + "obj_val": { + "type": "number", + "description": "The obj_val field" + } + } + } + }, + "array_of_arrays": { + "type": "array", + "description": "The array_of_arrays field", + "items": { + "type": "array" + } + } +}""" + +# Expected output for Claude Agent SDK _build_output_format(rich_schema). +EXPECTED_CLAUDE_AGENT_SDK_RICH_SCHEMA = """{ + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "string_scalar": { + "type": "string", + "description": "A string scalar field with a description" + }, + "number_scalar": { + "type": "number" + }, + "nested_object": { + "type": "object", + "description": "A nested object with properties", + "properties": { + "nested_string": { + "type": "string", + "description": "Nested string field" + }, + "nested_number": { + "type": "number" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "description": "An array of strings", + "items": { + "type": "string", + "description": "A string item" + } + }, + "array_of_objects": { + "type": "array", + "description": "An array of objects", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string", + "description": "The key" + }, + "obj_val": { + "type": "number", + "description": "The value" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "description": "An array of arrays", + "items": { + "type": "array", + "items": { + "type": "number", + "description": "A number in nested array" + } + } + } + }, + "required": [ + "string_scalar", + "number_scalar", + "nested_object", + "array_of_scalars", + "array_of_objects", + "array_of_arrays" + ] + } +}""" + +# Expected output for Claude Agent SDK _build_output_format(missing_descriptions_schema). +EXPECTED_CLAUDE_AGENT_SDK_MISSING_SCHEMA = """{ + "type": "json_schema", + "schema": { + "type": "object", + "properties": { + "string_scalar": { + "type": "string" + }, + "nested_object": { + "type": "object", + "properties": { + "nested_string": { + "type": "string" + }, + "nested_number": { + "type": "number" + } + }, + "required": [ + "nested_string", + "nested_number" + ] + }, + "array_of_scalars": { + "type": "array", + "items": { + "type": "string" + } + }, + "array_of_objects": { + "type": "array", + "items": { + "type": "object", + "properties": { + "obj_key": { + "type": "string" + }, + "obj_val": { + "type": "number" + } + }, + "required": [ + "obj_key", + "obj_val" + ] + } + }, + "array_of_arrays": { + "type": "array", + "items": { + "type": "array", + "items": { + "type": "number" + } + } + } + }, + "required": [ + "string_scalar", + "nested_object", + "array_of_scalars", + "array_of_objects", + "array_of_arrays" + ] + } +}""" + + +def _make_copilot_provider() -> CopilotProvider: + """Return a CopilotProvider instance wired to a no-op stub handler.""" + + def stub_handler(agent: Any, prompt: str, context: dict[str, Any]) -> dict[str, Any]: + return {"result": "stub"} + + return CopilotProvider(mock_handler=stub_handler) + + +class TestClaudeOutputSchemaGolden: + """Golden tests for ClaudeProvider's structured-output tool wrapper.""" + + @patch("conductor.providers.claude.ANTHROPIC_SDK_AVAILABLE", True) + @patch("conductor.providers.claude.AsyncAnthropic") + @patch("conductor.providers.claude.anthropic") + def test_build_tools_rich_schema_matches_baseline( + self, mock_anthropic_module: Mock, mock_anthropic_class: Mock + ) -> None: + """Claude tool wrapper output must be byte-for-byte identical to the pre-refactor + baseline for a schema with descriptions, nested objects, and arrays.""" + mock_anthropic_module.__version__ = "0.77.0" + mock_client = Mock() + mock_client.models.list = AsyncMock(return_value=Mock(data=[])) + mock_anthropic_class.return_value = mock_client + + provider = ClaudeProvider() + actual = _serialize(provider._build_tools_for_structured_output(rich_schema)) + assert actual == EXPECTED_CLAUDE_RICH_SCHEMA + + @patch("conductor.providers.claude.ANTHROPIC_SDK_AVAILABLE", True) + @patch("conductor.providers.claude.AsyncAnthropic") + @patch("conductor.providers.claude.anthropic") + def test_build_tools_missing_descriptions_matches_baseline( + self, mock_anthropic_module: Mock, mock_anthropic_class: Mock + ) -> None: + """Claude tool wrapper output must be byte-for-byte identical to the pre-refactor + baseline for a schema without explicit descriptions.""" + mock_anthropic_module.__version__ = "0.77.0" + mock_client = Mock() + mock_client.models.list = AsyncMock(return_value=Mock(data=[])) + mock_anthropic_class.return_value = mock_client + + provider = ClaudeProvider() + actual = _serialize( + provider._build_tools_for_structured_output(missing_descriptions_schema) + ) + assert actual == EXPECTED_CLAUDE_MISSING_SCHEMA + + +class TestCopilotOutputSchemaGolden: + """Golden tests for CopilotProvider's prompt schema wrapper.""" + + def test_build_prompt_schema_rich_matches_baseline(self) -> None: + """Copilot prompt schema wrapper must be byte-for-byte identical to the pre-refactor + baseline for a schema with descriptions and nested structure.""" + provider = _make_copilot_provider() + actual = _serialize(provider._build_prompt_schema(rich_schema)) + assert actual == EXPECTED_COPILOT_RICH_SCHEMA + + def test_build_prompt_schema_missing_descriptions_matches_baseline(self) -> None: + """Copilot prompt schema wrapper must be byte-for-byte identical to the pre-refactor + baseline for a schema without descriptions, including description fallbacks.""" + provider = _make_copilot_provider() + actual = _serialize(provider._build_prompt_schema(missing_descriptions_schema)) + assert actual == EXPECTED_COPILOT_MISSING_SCHEMA + + +class TestHermesOutputSchemaGolden: + """Golden tests for Hermes' module-level prompt schema wrapper.""" + + def test_build_prompt_schema_rich_matches_baseline(self) -> None: + """Hermes prompt schema wrapper must be byte-for-byte identical to the pre-refactor + baseline for a schema with descriptions, including legacy collapsed array-of-arrays.""" + actual = _serialize(_build_prompt_schema(rich_schema)) + assert actual == EXPECTED_HERMES_RICH_SCHEMA + + def test_build_prompt_schema_missing_descriptions_matches_baseline(self) -> None: + """Hermes prompt schema wrapper must be byte-for-byte identical to the pre-refactor + baseline for a schema without descriptions, including description fallbacks.""" + actual = _serialize(_build_prompt_schema(missing_descriptions_schema)) + assert actual == EXPECTED_HERMES_MISSING_SCHEMA + + +class TestClaudeAgentSdkOutputSchemaGolden: + """Golden tests for Claude Agent SDK's output_format wrapper.""" + + def test_build_output_format_rich_matches_baseline(self) -> None: + """Claude Agent SDK output_format wrapper must be byte-for-byte identical to the + pre-refactor baseline for a schema with descriptions and nested structure.""" + actual = _serialize(_build_output_format(rich_schema)) + assert actual == EXPECTED_CLAUDE_AGENT_SDK_RICH_SCHEMA + + def test_build_output_format_missing_descriptions_matches_baseline(self) -> None: + """Claude Agent SDK output_format wrapper must be byte-for-byte identical to the + pre-refactor baseline for a schema without descriptions.""" + actual = _serialize(_build_output_format(missing_descriptions_schema)) + assert actual == EXPECTED_CLAUDE_AGENT_SDK_MISSING_SCHEMA + + +class TestGoldenMutationGuard: + """Negative guard proving the golden tests are sensitive to behavioral changes.""" + + def test_mutated_copilot_literal_does_not_match(self) -> None: + """If the builder output is mutated (e.g., a description fallback is changed), the + golden assertion must fail so regressions are caught.""" + provider = _make_copilot_provider() + actual = _serialize(provider._build_prompt_schema(rich_schema)) + mutated = EXPECTED_COPILOT_RICH_SCHEMA.replace( + '"description": "The number_scalar field"', + '"description": "The number_scalar field (mutated)"', + ) + assert actual != mutated