diff --git a/mkdocs/site/packages/evo-compute/ComputeClient.html b/mkdocs/site/packages/evo-compute/ComputeClient.html index f1031751..a71b92e7 100644 --- a/mkdocs/site/packages/evo-compute/ComputeClient.html +++ b/mkdocs/site/packages/evo-compute/ComputeClient.html @@ -137,6 +137,41 @@

DEFAULT_CACHE_TTL_SECONDS + + + validate + + + bool + + +
+

Validate parameters against the task schema before submitting +(required-field presence). Defaults to True. This is the master switch: +False turns off deep validation too, whatever deep_validation says.

+
+ + + True + + + + + deep_validation + + + bool + + +
+

Additionally run full JSON Schema Draft 2020-12 validation. +Defaults to False. Only consulted when validate is True.

+
+ + + False + + @@ -173,7 +208,14 @@

-
arun(topic: str, task: str, parameters: dict[str, Any]) -> dict
+
arun(
+    topic: str,
+    task: str,
+    parameters: dict[str, Any],
+    *,
+    validate: bool | None = None,
+    deep_validation: bool | None = None,
+) -> dict
 
@@ -242,6 +284,40 @@

required + + + validate + + + bool | None + + +
+

Override the client's shallow-validation setting for this call. +The master switch: False skips deep validation too.

+
+ + + None + + + + + deep_validation + + + bool | None + + +
+

Override the client's deep-validation setting for this call. +Only consulted when validation is enabled.

+
+ + + None + + @@ -290,7 +366,7 @@

- TypeError + ParameterValidationError
diff --git a/packages/evo-compute/pyproject.toml b/packages/evo-compute/pyproject.toml index 22480499..12b6f641 100644 --- a/packages/evo-compute/pyproject.toml +++ b/packages/evo-compute/pyproject.toml @@ -12,6 +12,7 @@ dependencies = [ "evo-blockmodels[utils]", "evo-objects[utils,blockmodels]", "evo-sdk-common", + "jsonschema>=4.18", "pydantic>=2", "typing_extensions>=4.0", ] diff --git a/packages/evo-compute/src/evo/compute/__init__.py b/packages/evo-compute/src/evo/compute/__init__.py index a2c1764b..3605ea56 100644 --- a/packages/evo-compute/src/evo/compute/__init__.py +++ b/packages/evo-compute/src/evo/compute/__init__.py @@ -15,6 +15,7 @@ from .discovery import DiscoveryClient from .endpoints.models import TaskResource from .engine import ComputeClient +from .exceptions import ParameterValidationError __all__ = [ "ComputeClient", @@ -22,6 +23,7 @@ "JobClient", "JobProgress", "JobStatusEnum", + "ParameterValidationError", "TaskResource", "tasks", ] diff --git a/packages/evo-compute/src/evo/compute/engine.py b/packages/evo-compute/src/evo/compute/engine.py index 33cda3aa..5531bee6 100644 --- a/packages/evo-compute/src/evo/compute/engine.py +++ b/packages/evo-compute/src/evo/compute/engine.py @@ -41,6 +41,8 @@ from .client import JobClient from .discovery import DEFAULT_CACHE_TTL_SECONDS, DiscoveryClient from .endpoints.models import TaskResource +from .exceptions import ParameterValidationError +from .validation import validate_parameters __all__ = [ "ComputeClient", @@ -136,10 +138,12 @@ def _check_fallback_parameters(spec: TaskResource, parameters: dict[str, Any], l properties: dict[str, Any] = schema.get("properties", {}) for name in schema.get("required", []): if name not in parameters: - raise TypeError(f"{label}.run(): missing a required keyword-only argument: {name!r}") + detail = f"missing a required keyword-only argument: {name!r}" + raise ParameterValidationError(f"{label}.run(): {detail}", task=label, errors=[detail]) for name in parameters: if name not in properties: - raise TypeError(f"{label}.run(): got an unexpected keyword argument {name!r}") + detail = f"got an unexpected keyword argument {name!r}" + raise ParameterValidationError(f"{label}.run(): {detail}", task=label, errors=[detail]) class ComputeClient: @@ -150,12 +154,26 @@ class ComputeClient: :param context: An authenticated Evo context. :param cache_ttl_seconds: How long a discovered task catalogue is cached. + :param validate: Validate parameters against the task schema before submitting + (required-field presence). Defaults to ``True``. This is the master switch: + ``False`` turns off deep validation too, whatever ``deep_validation`` says. + :param deep_validation: Additionally run full JSON Schema Draft 2020-12 validation. + Defaults to ``False``. Only consulted when ``validate`` is ``True``. """ - def __init__(self, context: IContext, *, cache_ttl_seconds: float = DEFAULT_CACHE_TTL_SECONDS) -> None: + def __init__( + self, + context: IContext, + *, + cache_ttl_seconds: float = DEFAULT_CACHE_TTL_SECONDS, + validate: bool = True, + deep_validation: bool = False, + ) -> None: self._org_id: UUID = context.get_org_id() self._connector: APIConnector = context.get_connector() self._discovery = DiscoveryClient(self._connector, self._org_id, cache_ttl_seconds=cache_ttl_seconds) + self._validate = validate + self._deep_validation = deep_validation # -- dynamic namespace ------------------------------------------------- # @@ -184,7 +202,15 @@ def _peek_spec(self, topic: str, task: str) -> TaskResource | None: # -- execution --------------------------------------------------------- # - async def arun(self, topic: str, task: str, parameters: dict[str, Any]) -> dict: + async def arun( + self, + topic: str, + task: str, + parameters: dict[str, Any], + *, + validate: bool | None = None, + deep_validation: bool | None = None, + ) -> dict: """Run a task by name and wait for its results. The same as ``await client...run(**parameters)``, for names held as @@ -193,11 +219,15 @@ async def arun(self, topic: str, task: str, parameters: dict[str, Any]) -> dict: :param topic: The task topic. :param task: The task name. :param parameters: The task parameters, plus ``preview=True`` to run a preview task. + :param validate: Override the client's shallow-validation setting for this call. + The master switch: ``False`` skips deep validation too. + :param deep_validation: Override the client's deep-validation setting for this call. + Only consulted when validation is enabled. :return: The results of the finished job. :raises AttributeError: If the topic does not advertise the task. - :raises TypeError: If the parameters do not match the task's schema. + :raises ParameterValidationError: If the parameters do not match the task's schema. """ spec = await self._resolve_spec(topic, task) label = f"{topic}.{_normalise(task)}" @@ -205,7 +235,7 @@ async def arun(self, topic: str, task: str, parameters: dict[str, Any]) -> dict: try: bound = signature.bind(**parameters) except TypeError as error: - raise TypeError(f"{label}.run(): {error}") from None + raise ParameterValidationError(f"{label}.run(): {error}", task=label, errors=[str(error)]) from None # Forward only what the caller actually passed, so unset optionals fall back to the # platform's own defaults while an explicit ``None`` still reaches the wire. @@ -219,6 +249,14 @@ async def arun(self, topic: str, task: str, parameters: dict[str, Any]) -> dict: if (spec.parameters or {}).get("properties"): _check_fallback_parameters(spec, wire_parameters, label) + if validate is None: + validate = self._validate + if deep_validation is None: + deep_validation = self._deep_validation + # ``validate`` is the master switch; deep validation only runs underneath it. + if validate: + validate_parameters(spec, wire_parameters, deep=deep_validation, task_label=label) + job: JobClient[dict] = await JobClient.submit( connector=self._connector, org_id=self._org_id, diff --git a/packages/evo-compute/src/evo/compute/exceptions.py b/packages/evo-compute/src/evo/compute/exceptions.py index 240e3636..e285a094 100644 --- a/packages/evo-compute/src/evo/compute/exceptions.py +++ b/packages/evo-compute/src/evo/compute/exceptions.py @@ -67,3 +67,16 @@ class JobPendingError(EvoClientException): def __init__(self, url: str, status: str) -> None: super().__init__(f"Job at {url} is still pending with status: {status}") + + +class ParameterValidationError(ValueError): + """Raised when task parameters fail validation against the task's schema. + + Subclasses :class:`ValueError`. :attr:`errors` holds one actionable message per + individual validation failure; :attr:`task` is the ``topic.task`` label when known. + """ + + def __init__(self, message: str, *, task: str | None = None, errors: list[str] | None = None) -> None: + super().__init__(message) + self.task = task + self.errors = errors or [] diff --git a/packages/evo-compute/src/evo/compute/validation.py b/packages/evo-compute/src/evo/compute/validation.py new file mode 100644 index 00000000..84e115bb --- /dev/null +++ b/packages/evo-compute/src/evo/compute/validation.py @@ -0,0 +1,417 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Schema-driven validation of compute-task parameters before submission. + +The generic engine shapes a task's ``run(...)`` call from its published JSON Schema. +This module validates the resolved payload against that schema before it is submitted, +so mistakes surface locally with actionable messages instead of as opaque platform +errors: + +* **Shallow** (always on): required-field presence on the resolved payload. Signature + binding in :mod:`evo.compute.engine` already rejects unknown/missing keyword + arguments; this adds the schema-driven guard that a required field wasn't supplied + as ``None`` unless its schema permits null. +* **Deep** (opt-in): full JSON Schema Draft 2020-12 validation, including ``$ref`` / + ``$defs`` resolution and discriminated unions, via :mod:`jsonschema`. + +Reference leaves (properties carrying a ``reference_to`` annotation) are relaxed, because +the concrete object/attribute/file shape the schema declares is produced by the +reference-resolution layer rather than supplied by the caller. The relaxation is only as +wide as it has to be: a leaf transmitted as a scalar -- every one in the published +catalogue is a ``string`` URL -- keeps its declared type, so static hints and runtime +checks agree. A leaf declared as its resolved object or array form is relaxed to accept +any value, since the caller legitimately passes a bare reference instead. Everything else +in the schema is validated normally. + +:func:`unknown_annotation_keys` backs the annotation-conformance test that fails CI if +the platform introduces a schema annotation the engine does not know how to handle. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any + +from jsonschema import Draft202012Validator, ValidationError + +from .endpoints.models import TaskResource +from .exceptions import ParameterValidationError + +__all__ = [ + "KNOWN_SCHEMA_ANNOTATIONS", + "unknown_annotation_keys", + "validate_parameters", +] + + +KNOWN_SCHEMA_ANNOTATIONS: frozenset[str] = frozenset( + { + "reference_to", + "output", + "supported_schemas", + "attribute_from", + "attribute_path", + "target", + "discriminator", + "composite", + } +) +"""The closed set of non-JSON-Schema annotation keys the engine understands. + +The annotation-conformance test fails if a schema uses any annotation outside this set, +flagging that the engine needs updating before the new task can be handled generically. + +``composite`` names a sub-object the platform composes from several fields (for example, +the published ``kriging`` schema tags ``source.filter`` and ``target.filter`` with +``composite: filter``). It is descriptive only: the node it marks is an ordinary subschema +that resolution and validation already walk, so the engine records the key as understood +rather than reporting every discovery of a filtered task as vocabulary drift. +""" + + +# Standard JSON Schema Draft 2020-12 keywords. Any dict key at a schema position that is +# neither one of these nor a KNOWN_SCHEMA_ANNOTATIONS entry is an unrecognised annotation. +_JSON_SCHEMA_KEYWORDS: frozenset[str] = frozenset( + { + # Core + "$schema", + "$id", + "$ref", + "$anchor", + "$dynamicRef", + "$dynamicAnchor", + "$vocabulary", + "$comment", + "$defs", + "definitions", + # Applicators + "allOf", + "anyOf", + "oneOf", + "not", + "if", + "then", + "else", + "dependentSchemas", + "prefixItems", + "items", + "additionalItems", + "contains", + "properties", + "patternProperties", + "additionalProperties", + "propertyNames", + "unevaluatedItems", + "unevaluatedProperties", + # Validation + "type", + "enum", + "const", + "multipleOf", + "maximum", + "exclusiveMaximum", + "minimum", + "exclusiveMinimum", + "maxLength", + "minLength", + "pattern", + "maxItems", + "minItems", + "uniqueItems", + "maxContains", + "minContains", + "maxProperties", + "minProperties", + "required", + "dependentRequired", + # Meta-data + "title", + "description", + "default", + "deprecated", + "readOnly", + "writeOnly", + "examples", + # Format and content + "format", + "contentEncoding", + "contentMediaType", + "contentSchema", + } +) + +# Keywords whose value is itself a subschema. +_SUBSCHEMA_KEYS: frozenset[str] = frozenset( + { + "additionalProperties", + "unevaluatedProperties", + "unevaluatedItems", + "additionalItems", + "contains", + "propertyNames", + "not", + "if", + "then", + "else", + "contentSchema", + } +) +# Keywords whose value is a list of subschemas. +_SUBSCHEMA_LIST_KEYS: frozenset[str] = frozenset({"allOf", "anyOf", "oneOf", "prefixItems"}) +# Keywords whose value maps arbitrary names to subschemas (the names are not keywords). +_SUBSCHEMA_MAP_KEYS: frozenset[str] = frozenset( + {"properties", "patternProperties", "dependentSchemas", "$defs", "definitions"} +) + + +def _iter_schema_nodes(node: Any) -> Iterator[dict[str, Any]]: + """Yield every dict that occupies a *schema* position within ``node``. + + Recurses only through keywords that hold subschemas, so arbitrary names (property + names, ``$defs`` keys) and data values (``enum``, ``default``) are never mistaken + for schema keywords. + """ + if not isinstance(node, dict): + return + yield node + for key in _SUBSCHEMA_KEYS: + if key in node: + yield from _iter_schema_nodes(node[key]) + items = node.get("items") + if isinstance(items, list): + for sub in items: + yield from _iter_schema_nodes(sub) + elif isinstance(items, dict): + yield from _iter_schema_nodes(items) + for key in _SUBSCHEMA_LIST_KEYS: + value = node.get(key) + if isinstance(value, list): + for sub in value: + yield from _iter_schema_nodes(sub) + for key in _SUBSCHEMA_MAP_KEYS: + value = node.get(key) + if isinstance(value, dict): + for sub in value.values(): + yield from _iter_schema_nodes(sub) + + +def unknown_annotation_keys(schema: dict[str, Any] | None) -> set[str]: + """Return schema keys that are neither standard JSON Schema nor a known annotation. + + :param schema: A task ``parameters`` or ``results`` JSON Schema (or ``None``). + + :return: The set of unrecognised annotation keys; empty when the schema only uses + standard keywords and the closed annotation vocabulary. + """ + unknown: set[str] = set() + for node in _iter_schema_nodes(schema): + for key in node: + if key in _JSON_SCHEMA_KEYWORDS or key in KNOWN_SCHEMA_ANNOTATIONS: + continue + unknown.add(key) + return unknown + + +# Types a reference is actually transmitted as, so its declared type still holds when the +# caller passes the reference itself. ``null`` is not among them: it says the leaf may be +# omitted, not that a reference travels as one. +_TRANSMITTED_REFERENCE_TYPES: frozenset[str] = frozenset({"string", "integer", "number", "boolean"}) + + +def _relaxed_reference(node: dict[str, Any]) -> dict[str, Any]: + """The strongest constraint a ``reference_to`` leaf can still be held to. + + Keeps the declared type as soon as one of its members is a form the reference travels + as, and drops every constraint only when the leaf describes nothing but the resolved + object the caller does not pass. A nullable declaration is kept whole either way, so + ``["string", "null"]`` still admits both while ``["object", "null"]`` relaxes: after + resolution that leaf holds a URL, which its declared object type would reject. + """ + declared = node.get("type") + types = [declared] if isinstance(declared, str) else declared if isinstance(declared, list) else [] + if not any(entry in _TRANSMITTED_REFERENCE_TYPES for entry in types): + return {} + return {"type": types[0] if len(types) == 1 else types} + + +def _relax_reference_leaves(node: Any) -> Any: + """Deep-copy ``node``, relaxing any subschema carrying ``reference_to``. + + A ``reference_to`` leaf describes the resolved object/attribute/file form the schema + wants, which the reference-resolution layer produces. The caller passes a reference, so + the leaf keeps only what still applies to the reference itself (see + :func:`_relaxed_reference`); presence is still enforced by the parent's ``required``. + + Recurses through the same schema positions as :func:`_iter_schema_nodes`, so a literal + inside ``const``/``enum``/``default`` is copied untouched rather than rewritten. + """ + if not isinstance(node, dict): + return node + if "reference_to" in node: + return _relaxed_reference(node) + relaxed = dict(node) + for key in (*_SUBSCHEMA_KEYS, *_SUBSCHEMA_LIST_KEYS, *_SUBSCHEMA_MAP_KEYS, "items"): + if key not in relaxed: + continue + value = relaxed[key] + if key in _SUBSCHEMA_MAP_KEYS and isinstance(value, dict): + relaxed[key] = {name: _relax_reference_leaves(sub) for name, sub in value.items()} + elif isinstance(value, list): + relaxed[key] = [_relax_reference_leaves(sub) for sub in value] + else: + relaxed[key] = _relax_reference_leaves(value) + return relaxed + + +def _location(error: ValidationError) -> str: + return ".".join(str(part) for part in error.absolute_path) or "parameters" + + +def _describe_union(error: ValidationError) -> str | None: + """Describe a *discriminated* ``anyOf``/``oneOf`` failure, or ``None`` if it isn't discriminated. + + :mod:`jsonschema` only reports that the value matched no branch, and its own + best-match heuristic routinely picks the wrong one -- telling someone building a + spherical variogram that ``'scale' is a required property``. A ``discriminator`` + makes the intended branch knowable: it is the one whose sub-errors do not fault the + discriminating property, so report that branch's real errors instead. + """ + schema = error.schema if isinstance(error.schema, dict) else {} + discriminator = schema.get("discriminator") + name = discriminator.get("propertyName") if isinstance(discriminator, dict) else None + if not isinstance(name, str) or not isinstance(error.instance, dict) or name not in error.instance: + return None + + tag_path = (*error.absolute_path, name) + branches: dict[Any, list[ValidationError]] = {} + for sub in error.context: + branches.setdefault(sub.schema_path[0], []).append(sub) + matched = [subs for subs in branches.values() if all(tuple(sub.absolute_path) != tag_path for sub in subs)] + if len(matched) == 1: + return "; ".join(_describe(sub) for sub in matched[0]) + if matched: + # Two or more branches accept the discriminator value (the tags aren't mutually + # exclusive), so the intended one can't be singled out. Decline, and let the generic + # "not valid under any of the given schemas" message stand rather than guess. + return None + + # No branch accepted the tag, so the tag itself is the problem. + allowed: list[Any] = [] + for sub in error.context: + if tuple(sub.absolute_path) != tag_path: + continue + if sub.validator == "const": + allowed.append(sub.validator_value) + elif sub.validator == "enum": + allowed.extend(sub.validator_value) + if not allowed: + return None + values = ", ".join(repr(value) for value in dict.fromkeys(allowed)) + return f"{'.'.join(str(part) for part in tag_path)}: must be one of [{values}], got {error.instance[name]!r}" + + +def _describe(error: ValidationError) -> str: + """Map a :class:`jsonschema.ValidationError` to a short, actionable message.""" + location = _location(error) + if error.validator == "required": + instance = error.instance if isinstance(error.instance, dict) else {} + missing = [name for name in (error.validator_value or []) if name not in instance] + names = ", ".join(repr(name) for name in missing) or error.message + # Unprefixed at the top level, where the location adds nothing. + prefix = f"{location}: " if error.absolute_path else "" + return f"{prefix}missing required parameter(s): {names}" + if error.validator in {"anyOf", "oneOf"} and error.context: + if (described := _describe_union(error)) is not None: + return described + if error.validator == "type": + expected = error.validator_value + expected_str = expected if isinstance(expected, str) else " or ".join(expected) + return f"{location}: expected type {expected_str}, got {type(error.instance).__name__}" + if error.validator == "enum": + allowed = ", ".join(repr(value) for value in error.validator_value) + return f"{location}: must be one of [{allowed}], got {error.instance!r}" + if error.validator == "additionalProperties": + return f"{location}: {error.message}" + return f"{location}: {error.message}" + + +def _default_label(spec: TaskResource) -> str: + return f"{spec.topic}.{spec.name.replace('-', '_')}" + + +def _permits_null(root: dict[str, Any], prop_schema: Any) -> bool: + """Whether a property's subschema accepts ``None``. + + Evaluated in the root schema's scope so ``$ref``, ``allOf`` and bare schemas are + judged exactly as deep validation would judge them. A required-but-nullable field + (``Optional[X]`` with no default) has ``None`` as a real value, not a missing one. + """ + return Draft202012Validator(root).evolve(schema=prop_schema).is_valid(None) + + +def _validate_required(spec: TaskResource, parameters: dict[str, Any], label: str) -> None: + """Shallow check: every required field is supplied, and not as ``None`` unless nullable.""" + schema = spec.parameters or {} + properties = schema.get("properties", {}) or {} + required = schema.get("required", []) or [] + missing = sorted( + name + for name in required + if name not in parameters or (parameters[name] is None and not _permits_null(schema, properties.get(name, {}))) + ) + if missing: + errors = [f"missing required parameter {name!r}" for name in missing] + raise ParameterValidationError( + f"{label}.run(): missing required parameter(s): {', '.join(missing)}", + task=label, + errors=errors, + ) + + +def _validate_deep(spec: TaskResource, parameters: dict[str, Any], label: str) -> None: + """Deep check: validate the payload against the task schema (reference leaves relaxed).""" + schema = _relax_reference_leaves(spec.parameters or {}) + validator = Draft202012Validator(schema) + # Rendered parts keep the key uniformly typed: raw paths mix str and int and won't sort. + errors = sorted(validator.iter_errors(parameters), key=lambda error: [str(part) for part in error.absolute_path]) + if errors: + messages = [_describe(error) for error in errors] + detail = "\n - ".join(messages) + raise ParameterValidationError( + f"{label}.run(): parameters do not match the task schema:\n - {detail}", + task=label, + errors=messages, + ) + + +def validate_parameters( + spec: TaskResource, + parameters: dict[str, Any], + *, + deep: bool = False, + task_label: str | None = None, +) -> None: + """Validate a resolved parameter payload against a task's schema. + + :param spec: The discovered task, carrying its ``parameters`` JSON Schema. + :param parameters: The resolved payload about to be submitted. + :param deep: When ``True``, also run full JSON Schema Draft 2020-12 validation. + :param task_label: A ``topic.task`` label for error messages; derived from ``spec`` + when omitted. + + :raises ParameterValidationError: If the payload is missing a required field or, when + ``deep`` is set, fails schema validation. + """ + label = task_label or _default_label(spec) + _validate_required(spec, parameters, label) + if deep: + _validate_deep(spec, parameters, label) diff --git a/packages/evo-compute/tests/data/discovery-validation.json b/packages/evo-compute/tests/data/discovery-validation.json new file mode 100644 index 00000000..8143c156 --- /dev/null +++ b/packages/evo-compute/tests/data/discovery-validation.json @@ -0,0 +1,25 @@ +{ + "limit": 100, + "offset": 0, + "total": 1, + "count": 1, + "results": [ + { + "key": "demo/widget", + "topic": "demo", + "name": "widget", + "display_name": "Widget", + "version": "1.0.0", + "description": "A demo task carrying non-reference constraints for validation tests.", + "parameters": { + "type": "object", + "properties": { + "source": {"type": "object", "reference_to": "geoscience-object"}, + "mode": {"type": "string", "enum": ["fast", "accurate"]}, + "iterations": {"type": "integer", "minimum": 1, "default": null} + }, + "required": ["source", "mode"] + } + } + ] +} diff --git a/packages/evo-compute/tests/test_engine.py b/packages/evo-compute/tests/test_engine.py index 5976fa8a..2faac28a 100644 --- a/packages/evo-compute/tests/test_engine.py +++ b/packages/evo-compute/tests/test_engine.py @@ -22,7 +22,7 @@ from evo.common.test_tools import TestWithConnector from data import load_test_data -from evo.compute import ComputeClient +from evo.compute import ComputeClient, ParameterValidationError def _task(topic: str, name: str, properties: dict[str, Any], required: list[str] | None = None) -> dict[str, Any]: @@ -152,13 +152,13 @@ async def test_fallback_parameters_still_reject_missing_and_unexpected_names(sel ), ] with self.catalogue_response(), self.mock_job_client() as submit: - with self.assertRaisesRegex(TypeError, r"mesh_tools\.export\.run\(\):.*'from'"): + with self.assertRaisesRegex(ParameterValidationError, r"mesh_tools\.export\.run\(\):.*'from'"): await self.client.mesh_tools.export.run() - with self.assertRaisesRegex(TypeError, r"mesh_tools\.export\.run\(\):.*'bogus'"): + with self.assertRaisesRegex(ParameterValidationError, r"mesh_tools\.export\.run\(\):.*'bogus'"): await self.client.mesh_tools.export.run(**{"from": "mesh-1", "bogus": 1}) - with self.assertRaisesRegex(TypeError, r"mesh-tools\.merge\.run\(\):.*'user-info'"): + with self.assertRaisesRegex(ParameterValidationError, r"mesh-tools\.merge\.run\(\):.*'user-info'"): await self.client.arun("mesh-tools", "merge", {"user_info": "b"}) - with self.assertRaisesRegex(TypeError, r"mesh-tools\.merge\.run\(\):.*'bogus'"): + with self.assertRaisesRegex(ParameterValidationError, r"mesh-tools\.merge\.run\(\):.*'bogus'"): await self.client.arun("mesh-tools", "merge", {"user-info": "a", "bogus": 1}) submit.assert_not_awaited() @@ -245,22 +245,24 @@ async def test_unknown_task_raises_attribute_error(self) -> None: await self.client.geostatistics.does_not_exist.run() self.assertIn("declustering", str(ctx.exception)) - async def test_missing_required_parameter_raises_type_error(self) -> None: + async def test_missing_required_parameter_raises_validation_error(self) -> None: """A missing required parameter is rejected before submission.""" with self.catalogue_response(), self.mock_job_client() as submit: - with self.assertRaises(TypeError) as ctx: + with self.assertRaises(ParameterValidationError) as ctx: await self.client.geostatistics.declustering.run() self.assertIn("source", str(ctx.exception)) submit.assert_not_awaited() - async def test_unexpected_parameter_raises_type_error(self) -> None: + async def test_unexpected_parameter_raises_validation_error(self) -> None: """An unknown keyword argument is rejected, and the error names the task.""" with self.catalogue_response(), self.mock_job_client() as submit: - with self.assertRaises(TypeError) as ctx: + with self.assertRaises(ParameterValidationError) as ctx: await self.client.geostatistics.declustering.run(source="obj-1", bogus=1) self.assertIn("geostatistics.declustering.run():", str(ctx.exception)) self.assertIn("bogus", str(ctx.exception)) + # Binding failures populate ``errors`` too, so callers never have to parse the message. + self.assertEqual(["got an unexpected keyword argument 'bogus'"], ctx.exception.errors) submit.assert_not_awaited() # -- progressive ergonomics (cache-backed, no forced discovery) -------- # diff --git a/packages/evo-compute/tests/test_schema_conformance.py b/packages/evo-compute/tests/test_schema_conformance.py new file mode 100644 index 00000000..a8020537 --- /dev/null +++ b/packages/evo-compute/tests/test_schema_conformance.py @@ -0,0 +1,148 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Annotation-conformance tests for the generic engine's closed schema vocabulary. + +The engine handles a task generically only because every task schema is built from +standard JSON Schema plus a small, closed set of custom annotations. These tests fail +if a schema uses an annotation the engine does not know about, flagging that the engine +needs updating before that task can be handled generically. + +Two variants: + +* Fixture-based (always runs): checks the bundled catalogue fixtures and the vocabulary + constant. +* Live (opt-in): checks the organization's real, live catalogue. Enable by setting + ``EVO_COMPUTE_LIVE_DISCOVERY=1`` along with ``EVO_ACCESS_TOKEN``, ``EVO_HUB_URL`` and + ``EVO_ORG_ID``. +""" + +from __future__ import annotations + +import os +import unittest +from uuid import UUID + +from data import load_test_data +from evo.compute import TaskResource +from evo.compute.validation import KNOWN_SCHEMA_ANNOTATIONS, unknown_annotation_keys + +_EXPECTED_VOCABULARY = frozenset( + { + "reference_to", + "output", + "supported_schemas", + "attribute_from", + "attribute_path", + "target", + "discriminator", + "composite", + } +) + + +def _unknown_keys_for_task(task: TaskResource) -> set[str]: + return unknown_annotation_keys(task.parameters) | unknown_annotation_keys(task.results) + + +class TestSchemaAnnotationConformance(unittest.TestCase): + def test_vocabulary_matches_expected_set(self) -> None: + """Guard the closed vocabulary against accidental drift.""" + self.assertEqual(_EXPECTED_VOCABULARY, KNOWN_SCHEMA_ANNOTATIONS) + + def test_bundled_catalogue_uses_only_known_annotations(self) -> None: + catalogue = load_test_data("discovery-tasks.json") + offenders: dict[str, set[str]] = {} + for result in catalogue["results"]: + task = TaskResource.model_validate(result) + unknown = _unknown_keys_for_task(task) + if unknown: + offenders[task.name] = unknown + self.assertEqual({}, offenders, f"unknown schema annotations found: {offenders}") + + def test_unknown_annotation_is_flagged(self) -> None: + schema = { + "type": "object", + "properties": {"source": {"type": "object", "made_up_annotation": True}}, + } + self.assertEqual({"made_up_annotation"}, unknown_annotation_keys(schema)) + + def test_known_annotations_are_not_flagged(self) -> None: + schema = { + "type": "object", + "properties": { + "source": { + "type": "object", + "reference_to": "geoscience-object", + "supported_schemas": ["x"], + }, + "attr": {"type": "string", "reference_to": "attribute", "attribute_from": "0/source"}, + "filter": {"type": "object", "composite": "filter"}, + }, + "$defs": { + "Union": { + "discriminator": {"propertyName": "kind"}, + "oneOf": [{"type": "object"}], + } + }, + } + self.assertEqual(set(), unknown_annotation_keys(schema)) + + def test_property_named_like_keyword_is_not_flagged(self) -> None: + """A property whose *name* collides with a keyword must not be treated as one.""" + schema = { + "type": "object", + "properties": { + "type": {"type": "string"}, + "reference_to": {"type": "string"}, + "output": {"type": "number"}, + }, + } + self.assertEqual(set(), unknown_annotation_keys(schema)) + + def test_data_values_are_not_walked(self) -> None: + """Arbitrary data in enum/default/const must not be mistaken for schema keywords.""" + schema = { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": ["fast"], "default": "fast"}, + "meta": {"const": {"made_up_annotation": 1}}, + }, + } + self.assertEqual(set(), unknown_annotation_keys(schema)) + + def test_output_annotation_in_results_is_known(self) -> None: + results = {"type": "object", "properties": {"target": {"type": "object", "output": "geoscience-object"}}} + self.assertEqual(set(), unknown_annotation_keys(results)) + + +class TestLiveCatalogueConformance(unittest.IsolatedAsyncioTestCase): + @unittest.skipUnless( + os.getenv("EVO_COMPUTE_LIVE_DISCOVERY"), + "set EVO_COMPUTE_LIVE_DISCOVERY=1 (+ EVO_ACCESS_TOKEN, EVO_HUB_URL, EVO_ORG_ID) to run the live check", + ) + async def test_live_catalogue_uses_only_known_annotations(self) -> None: + from evo.aio import AioTransport + from evo.common import APIConnector + from evo.oauth import AccessTokenAuthorizer + + from evo.compute import DiscoveryClient + + transport = AioTransport(user_agent="evo-compute-conformance-test") + authorizer = AccessTokenAuthorizer(access_token=os.environ["EVO_ACCESS_TOKEN"]) + connector = APIConnector(base_url=os.environ["EVO_HUB_URL"], transport=transport, authorizer=authorizer) + async with connector: + client = DiscoveryClient(connector, UUID(os.environ["EVO_ORG_ID"])) + tasks = await client.list_tasks() + + self.assertTrue(tasks, "live discovery returned no tasks") + offenders = {task.name: unknown for task in tasks if (unknown := _unknown_keys_for_task(task))} + self.assertEqual({}, offenders, f"unknown schema annotations in live catalogue: {offenders}") diff --git a/packages/evo-compute/tests/test_validation.py b/packages/evo-compute/tests/test_validation.py new file mode 100644 index 00000000..744a2c9a --- /dev/null +++ b/packages/evo-compute/tests/test_validation.py @@ -0,0 +1,526 @@ +# Copyright © 2026 Bentley Systems, Incorporated +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import unittest +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any +from unittest import mock + +from evo.common.test_tools import ORG as TEST_ORG +from evo.common.test_tools import TestWithConnector + +from data import load_test_data +from evo.compute import ComputeClient, ParameterValidationError, TaskResource +from evo.compute.validation import validate_parameters + + +def make_spec(parameters: dict[str, Any], *, topic: str = "demo", name: str = "widget") -> TaskResource: + """Build a TaskResource carrying the given parameter schema.""" + return TaskResource.model_validate({"topic": topic, "name": name, "parameters": parameters}) + + +class TestValidateParameters(unittest.TestCase): + """Unit tests for the schema-driven parameter validator (no network).""" + + # -- shallow ----------------------------------------------------------- # + + def test_shallow_passes_when_required_present(self) -> None: + spec = make_spec({"type": "object", "properties": {"source": {"type": "object"}}, "required": ["source"]}) + validate_parameters(spec, {"source": "obj-1"}) # does not raise + + def test_shallow_rejects_missing_required(self) -> None: + spec = make_spec({"type": "object", "properties": {"source": {"type": "object"}}, "required": ["source"]}) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {}) + self.assertIn("source", str(ctx.exception)) + self.assertEqual(["missing required parameter 'source'"], ctx.exception.errors) + + def test_shallow_rejects_required_set_to_none(self) -> None: + """A required field set to None (when its schema is non-nullable) is caught.""" + spec = make_spec({"type": "object", "properties": {"source": {"type": "object"}}, "required": ["source"]}) + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {"source": None}) + + def test_shallow_accepts_none_for_a_nullable_required_field(self) -> None: + """``Optional[X]`` with no default is required *and* nullable, so None is a real value.""" + union = make_spec( + { + "type": "object", + "properties": {"seed": {"anyOf": [{"type": "integer"}, {"type": "null"}]}}, + "required": ["seed"], + } + ) + validate_parameters(union, {"seed": None}, deep=True) # does not raise + + type_list = make_spec( + {"type": "object", "properties": {"seed": {"type": ["integer", "null"]}}, "required": ["seed"]} + ) + validate_parameters(type_list, {"seed": None}, deep=True) # does not raise + + def test_shallow_defers_to_the_schema_for_less_obvious_nullability(self) -> None: + """Nullability is whatever Draft 2020-12 says it is, not just an explicit ``"null"`` type.""" + bare = make_spec({"type": "object", "properties": {"options": {"default": None}}, "required": ["options"]}) + validate_parameters(bare, {"options": None}, deep=True) # unconstrained, so null is allowed + + behind_ref = make_spec( + { + "type": "object", + "properties": {"seed": {"$ref": "#/$defs/MaybeSeed"}}, + "required": ["seed"], + "$defs": {"MaybeSeed": {"type": ["integer", "null"]}}, + } + ) + validate_parameters(behind_ref, {"seed": None}, deep=True) # does not raise + + const_null = make_spec({"type": "object", "properties": {"unset": {"const": None}}, "required": ["unset"]}) + validate_parameters(const_null, {"unset": None}, deep=True) # does not raise + + def test_shallow_still_rejects_an_omitted_nullable_required_field(self) -> None: + spec = make_spec( + {"type": "object", "properties": {"seed": {"type": ["integer", "null"]}}, "required": ["seed"]} + ) + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {}) + + def test_shallow_ignores_type_mismatches(self) -> None: + """Shallow validation only checks presence, not types.""" + spec = make_spec({"type": "object", "properties": {"count": {"type": "integer"}}}) + validate_parameters(spec, {"count": "not-an-int"}) # does not raise without deep + + # -- deep -------------------------------------------------------------- # + + def test_deep_rejects_bad_enum(self) -> None: + spec = make_spec({"type": "object", "properties": {"mode": {"type": "string", "enum": ["fast", "accurate"]}}}) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"mode": "turbo"}, deep=True) + message = str(ctx.exception) + self.assertIn("mode", message) + self.assertIn("fast", message) + + def test_deep_rejects_wrong_scalar_type(self) -> None: + spec = make_spec({"type": "object", "properties": {"power": {"type": "number"}}}) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"power": "abc"}, deep=True) + self.assertIn("power", str(ctx.exception)) + self.assertIn("number", str(ctx.exception)) + + def test_deep_rejects_missing_nested_required(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "config": { + "type": "object", + "properties": {"seed": {"type": "integer"}}, + "required": ["seed"], + } + }, + "required": ["config"], + } + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"config": {}}, deep=True) + self.assertIn("seed", str(ctx.exception)) + + def test_deep_resolves_internal_ref(self) -> None: + """``$ref``/``$defs`` are resolved so nested constraints are enforced.""" + spec = make_spec( + { + "type": "object", + "properties": {"variogram": {"$ref": "#/$defs/Variogram"}}, + "required": ["variogram"], + "$defs": { + "Variogram": { + "type": "object", + "properties": {"nugget": {"type": "number"}}, + "required": ["nugget"], + } + }, + } + ) + validate_parameters(spec, {"variogram": {"nugget": 0.1}}, deep=True) # does not raise + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"variogram": {}}, deep=True) + self.assertIn("nugget", str(ctx.exception)) + + def test_deep_accepts_valid_payload(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": ["fast", "accurate"]}, + "iterations": {"type": "integer", "minimum": 1}, + }, + "required": ["mode"], + } + ) + validate_parameters(spec, {"mode": "fast", "iterations": 5}, deep=True) # does not raise + + def test_deep_accepts_nullable_union(self) -> None: + spec = make_spec({"type": "object", "properties": {"power": {"type": ["number", "null"]}}}) + validate_parameters(spec, {"power": None}, deep=True) # does not raise + + def test_deep_accepts_anyof_union(self) -> None: + spec = make_spec( + {"type": "object", "properties": {"shape": {"anyOf": [{"type": "string"}, {"type": "integer"}]}}} + ) + validate_parameters(spec, {"shape": "circle"}, deep=True) + validate_parameters(spec, {"shape": 3}, deep=True) + + # -- reference-leaf relaxation ----------------------------------------- # + + def test_deep_relaxes_a_reference_leaf_declared_as_its_resolved_form(self) -> None: + """A leaf typed as the object it resolves to accepts the plain reference instead.""" + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": "object", "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + validate_parameters(spec, {"source": "https://example/objects/abc"}, deep=True) # does not raise + + def test_deep_holds_a_scalar_reference_to_its_declared_type(self) -> None: + """A reference transmitted as a string is still checked against its scalar type.""" + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": "string", "format": "uri", "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + validate_parameters(spec, {"source": "https://example/objects/abc"}, deep=True) # does not raise + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"source": 12345}, deep=True) + self.assertEqual(["source: expected type string, got int"], ctx.exception.errors) + + def test_deep_allows_null_for_a_nullable_scalar_reference(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": ["string", "null"], "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + validate_parameters(spec, {"source": None}, deep=True) # does not raise + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {"source": 12345}, deep=True) + + def test_deep_keeps_the_transmitted_branch_of_a_mixed_declared_type(self) -> None: + """Part scalar, part resolved form: the value still has to be one of the two.""" + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": ["string", "object"], "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + validate_parameters(spec, {"source": "https://example/objects/abc"}, deep=True) # does not raise + validate_parameters(spec, {"source": {"object": "abc"}}, deep=True) # nor the resolved form + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {"source": 12345}, deep=True) + + def test_deep_relaxes_a_nullable_resolved_form(self) -> None: + """``null`` says the leaf may be omitted, not that a reference travels as one.""" + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": ["object", "null"], "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + validate_parameters(spec, {"source": "https://example/objects/abc"}, deep=True) # does not raise + validate_parameters(spec, {"source": None}, deep=True) # nor the null the schema allows + + def test_deep_still_requires_reference_presence(self) -> None: + """Relaxation loosens the value's shape but does not waive the required check.""" + spec = make_spec( + { + "type": "object", + "properties": {"source": {"type": "object", "reference_to": "geoscience-object"}}, + "required": ["source"], + } + ) + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {}, deep=True) + + def test_relaxation_leaves_literal_values_alone(self) -> None: + """``reference_to`` inside a ``const`` literal is data, not a subschema to relax.""" + spec = make_spec( + {"type": "object", "properties": {"meta": {"const": {"reference_to": "literal"}}}}, + ) + validate_parameters(spec, {"meta": {"reference_to": "literal"}}, deep=True) # does not raise + with self.assertRaises(ParameterValidationError): + validate_parameters(spec, {"meta": {}}, deep=True) + + # -- friendly messages ------------------------------------------------- # + + def test_nested_required_message_names_its_location(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "config": {"type": "object", "properties": {"seed": {"type": "integer"}}, "required": ["seed"]} + }, + } + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"config": {}}, deep=True) + self.assertEqual(["config: missing required parameter(s): 'seed'"], ctx.exception.errors) + + def test_top_level_required_message_is_unprefixed(self) -> None: + """A root-level ``required`` the shallow pass can't see, reported without a location.""" + spec = make_spec( + {"type": "object", "properties": {"mode": {"type": "string"}}, "allOf": [{"required": ["mode"]}]} + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {}, deep=True) + self.assertEqual(["missing required parameter(s): 'mode'"], ctx.exception.errors) + + def test_error_aggregates_multiple_failures(self) -> None: + spec = make_spec( + { + "type": "object", + "properties": { + "mode": {"type": "string", "enum": ["fast"]}, + "power": {"type": "number"}, + }, + } + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"mode": "turbo", "power": "x"}, deep=True) + self.assertEqual(2, len(ctx.exception.errors)) + + def test_errors_at_mixed_type_keys_are_still_reported(self) -> None: + """Sibling failures under int and str keys must not collide while being ordered.""" + spec = make_spec({"type": "object", "additionalProperties": {"type": "string"}}) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {0: 1, "name": 2}, deep=True) + self.assertEqual(2, len(ctx.exception.errors)) + + # -- discriminated unions ---------------------------------------------- # + + @staticmethod + def _variogram_spec() -> TaskResource: + """A Pydantic-style discriminated union whose branches are ``$ref``s.""" + return make_spec( + { + "type": "object", + "properties": { + "variogram": { + "discriminator": {"propertyName": "kind"}, + "oneOf": [{"$ref": "#/$defs/Spherical"}, {"$ref": "#/$defs/Exponential"}], + } + }, + "required": ["variogram"], + "$defs": { + "Spherical": { + "type": "object", + "properties": {"kind": {"const": "spherical"}, "range": {"type": "number"}}, + "required": ["kind", "range"], + }, + "Exponential": { + "type": "object", + "properties": {"kind": {"const": "exponential"}, "scale": {"type": "number"}}, + "required": ["kind", "scale"], + }, + }, + } + ) + + def test_deep_accepts_a_valid_union_branch(self) -> None: + validate_parameters(self._variogram_spec(), {"variogram": {"kind": "spherical", "range": 12.0}}, deep=True) + + def test_union_error_reports_the_selected_branch(self) -> None: + """The discriminator picks the branch, so the message is about *that* branch.""" + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters( + self._variogram_spec(), {"variogram": {"kind": "spherical", "range": "wide"}}, deep=True + ) + self.assertEqual(["variogram.range: expected type number, got str"], ctx.exception.errors) + + def test_union_error_reports_a_missing_field_of_the_selected_branch(self) -> None: + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(self._variogram_spec(), {"variogram": {"kind": "spherical"}}, deep=True) + self.assertEqual(["variogram: missing required parameter(s): 'range'"], ctx.exception.errors) + + def test_union_error_reports_allowed_discriminator_values(self) -> None: + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(self._variogram_spec(), {"variogram": {"kind": "gaussian"}}, deep=True) + self.assertEqual( + ["variogram.kind: must be one of ['spherical', 'exponential'], got 'gaussian'"], ctx.exception.errors + ) + + def test_undiscriminated_union_keeps_the_generic_message(self) -> None: + """Without a discriminator no branch can be singled out, so no branch is guessed at.""" + spec = make_spec( + {"type": "object", "properties": {"shape": {"anyOf": [{"type": "string"}, {"type": "integer"}]}}} + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"shape": 1.5}, deep=True) + self.assertIn("is not valid under any of the given schemas", ctx.exception.errors[0]) + + def test_union_with_several_matching_branches_keeps_the_generic_message(self) -> None: + """When more than one branch accepts the tag, no branch is guessed at either.""" + spec = make_spec( + { + "type": "object", + "properties": { + "variogram": { + "discriminator": {"propertyName": "kind"}, + "oneOf": [{"$ref": "#/$defs/A"}, {"$ref": "#/$defs/B"}], + } + }, + "required": ["variogram"], + "$defs": { + # Neither branch pins ``kind`` with a const, so both accept any kind. + "A": { + "type": "object", + "properties": {"kind": {"type": "string"}, "range": {"type": "number"}}, + "required": ["kind", "range"], + }, + "B": { + "type": "object", + "properties": {"kind": {"type": "string"}, "scale": {"type": "number"}}, + "required": ["kind", "scale"], + }, + }, + } + ) + with self.assertRaises(ParameterValidationError) as ctx: + validate_parameters(spec, {"variogram": {"kind": "spherical", "range": "wide"}}, deep=True) + self.assertIn("is not valid under any of the given schemas", ctx.exception.errors[0]) + + +class TestEngineValidation(TestWithConnector): + """Engine-level tests for the validation toggles (shallow default, deep opt-in).""" + + def setUp(self) -> None: + super().setUp() + self.context = FakeContext(self.connector, TEST_ORG.id) + self.catalogue = load_test_data("discovery-validation.json") + + @contextmanager + def catalogue_response(self) -> Iterator[None]: + with self.transport.set_http_response( + status_code=200, + content=json.dumps(self.catalogue), + headers={"Content-Type": "application/json"}, + ): + yield + + @contextmanager + def mock_job_client(self) -> Iterator[mock.AsyncMock]: + job = mock.Mock() + job.wait_for_results = mock.AsyncMock(return_value={"ok": True}) + submit = mock.AsyncMock(return_value=job) + with mock.patch("evo.compute.engine.JobClient") as mock_job_client: + mock_job_client.submit = submit + yield submit + + async def test_shallow_is_on_by_default_deep_is_off(self) -> None: + """By default an enum violation slips through (deep off) and the job is submitted.""" + client = ComputeClient(self.context) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.demo.widget.run(source="obj-1", mode="turbo") + submit.assert_awaited_once() + + async def test_deep_validation_rejects_bad_enum(self) -> None: + client = ComputeClient(self.context, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaises(ParameterValidationError) as ctx: + await client.demo.widget.run(source="obj-1", mode="turbo") + self.assertIn("mode", str(ctx.exception)) + submit.assert_not_awaited() + + async def test_deep_validation_rejects_constraint_violation(self) -> None: + client = ComputeClient(self.context, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaises(ParameterValidationError): + await client.demo.widget.run(source="obj-1", mode="fast", iterations=0) + submit.assert_not_awaited() + + async def test_deep_validation_rejects_explicit_none_for_a_non_nullable_field(self) -> None: + """An explicit ``None`` reaches validation rather than being silently dropped.""" + client = ComputeClient(self.context, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaises(ParameterValidationError) as ctx: + await client.demo.widget.run(source="obj-1", mode="fast", iterations=None) + self.assertIn("iterations", str(ctx.exception)) + submit.assert_not_awaited() + + async def test_deep_validation_accepts_reference_string(self) -> None: + """Deep validation passes when the required reference is supplied as a string.""" + client = ComputeClient(self.context, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.demo.widget.run(source="obj-1", mode="fast") + submit.assert_awaited_once() + + async def test_deep_validation_can_be_forced_per_call(self) -> None: + client = ComputeClient(self.context) # deep off at the client level + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaises(ParameterValidationError): + await client.arun("demo", "widget", {"source": "obj-1", "mode": "turbo"}, deep_validation=True) + submit.assert_not_awaited() + + async def test_validation_can_be_disabled_per_call(self) -> None: + """With validation off, a required field set to ``None`` goes out for the platform to reject.""" + client = ComputeClient(self.context) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.arun("demo", "widget", {"source": "obj-1", "mode": None}, validate=False) + submit.assert_awaited_once() + self.assertIsNone(submit.await_args.kwargs["parameters"]["mode"]) + + async def test_validation_on_catches_required_set_to_none(self) -> None: + client = ComputeClient(self.context) + with self.catalogue_response(), self.mock_job_client() as submit: + with self.assertRaises(ParameterValidationError): + await client.arun("demo", "widget", {"source": "obj-1", "mode": None}) + submit.assert_not_awaited() + + async def test_validate_off_at_the_client_disables_deep_validation_too(self) -> None: + """``validate`` is the master switch, so ``deep_validation=True`` under it does nothing.""" + client = ComputeClient(self.context, validate=False, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.demo.widget.run(source="obj-1", mode="turbo") + submit.assert_awaited_once() + + async def test_validate_off_per_call_disables_deep_validation_too(self) -> None: + client = ComputeClient(self.context, deep_validation=True) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.arun("demo", "widget", {"source": "obj-1", "mode": "turbo"}, validate=False) + submit.assert_awaited_once() + + async def test_validate_off_per_call_overrides_deep_validation_on_per_call(self) -> None: + client = ComputeClient(self.context) + with self.catalogue_response(), self.mock_job_client() as submit: + await client.arun( + "demo", "widget", {"source": "obj-1", "mode": "turbo"}, validate=False, deep_validation=True + ) + submit.assert_awaited_once() + + +class FakeContext: + """Minimal duck-typed IContext exposing just what ComputeClient uses.""" + + def __init__(self, connector, org_id) -> None: + self._connector = connector + self._org_id = org_id + + def get_connector(self): + return self._connector + + def get_org_id(self): + return self._org_id diff --git a/uv.lock b/uv.lock index c7e5174e..b45d254b 100644 --- a/uv.lock +++ b/uv.lock @@ -1056,6 +1056,7 @@ dependencies = [ { name = "evo-blockmodels", extra = ["utils"] }, { name = "evo-objects", extra = ["blockmodels", "utils"] }, { name = "evo-sdk-common" }, + { name = "jsonschema" }, { name = "pydantic" }, { name = "typing-extensions" }, ] @@ -1090,6 +1091,7 @@ requires-dist = [ { name = "evo-sdk-common", editable = "packages/evo-sdk-common" }, { name = "evo-sdk-common", extras = ["aiohttp"], marker = "extra == 'aiohttp'", editable = "packages/evo-sdk-common" }, { name = "evo-sdk-common", extras = ["notebooks"], marker = "extra == 'notebooks'", editable = "packages/evo-sdk-common" }, + { name = "jsonschema", specifier = ">=4.18" }, { name = "pydantic", specifier = ">=2" }, { name = "typing-extensions", specifier = ">=4.0" }, ]