feat: Add support for referencing registered metrics by resource name in evaluation run API

PiperOrigin-RevId: 878604099

Author: vertex-sdk-bot

vertexai/_genai/_evals_common.py

@@ -45,6 +45,7 @@
 from . import _gcs_utils
 from . import evals
 from . import types
+from . import _transformers as t
 
 logger = logging.getLogger(__name__)
 
@@ -1328,7 +1329,7 @@ def _resolve_dataset_inputs(
 
 
 def _resolve_evaluation_run_metrics(
-    metrics: list[types.EvaluationRunMetric], api_client: Any
+    metrics: list[types.EvaluationRunMetric]|list[types.Metric], api_client: Any
 ) -> list[types.EvaluationRunMetric]:
     """Resolves a list of evaluation run metric instances, loading RubricMetric if necessary."""
     if not metrics:
@@ -1361,6 +1362,16 @@ def _resolve_evaluation_run_metrics(
                     e,
                 )
                 raise
+        elif isinstance(metric_instance, types.Metric):
+            config_dict = t.t_metrics([metric_instance])[0]
+            res_name = config_dict.pop("metric_resource_name", None)
+            resolved_metrics_list.append(
+                types.EvaluationRunMetric(
+                    metric=metric_instance.name,
+                    metric_config=config_dict if config_dict else None,
+                    metric_resource_name=res_name,
+                )
+            )
         else:
             try:
                 metric_name_str = str(metric_instance)

vertexai/_genai/_transformers.py

@@ -38,6 +38,8 @@ def t_metrics(
 
     for metric in metrics:
         metric_payload_item: dict[str, Any] = {}
+        if hasattr(metric, "metric_resource_name") and metric.metric_resource_name:
+            metric_payload_item["metric_resource_name"] = metric.metric_resource_name
 
         metric_name = getv(metric, ["name"]).lower()
 
@@ -79,6 +81,9 @@ def t_metrics(
                     "return_raw_output": return_raw_output
                 }
             metric_payload_item["pointwise_metric_spec"] = pointwise_spec
+        elif "metric_resource_name" in metric_payload_item:
+            # Valid case: Metric is identified by resource name; no inline spec required.
+            pass
         else:
             raise ValueError(
                 f"Unsupported metric type or invalid metric name: {metric_name}"

vertexai/_genai/evals.py

@@ -399,6 +399,13 @@ def _EvaluationRunMetric_from_vertex(
             _UnifiedMetric_from_vertex(getv(from_object, ["metricConfig"]), to_object),
         )
 
+    if getv(from_object, ["metricResourceName"]) is not None:
+        setv(
+            to_object,
+            ["metric_resource_name"],
+            getv(from_object, ["metricResourceName"]),
+        )
+
     return to_object
 
 
@@ -417,6 +424,13 @@ def _EvaluationRunMetric_to_vertex(
             _UnifiedMetric_to_vertex(getv(from_object, ["metric_config"]), to_object),
         )
 
+    if getv(from_object, ["metric_resource_name"]) is not None:
+        setv(
+            to_object,
+            ["metricResourceName"],
+            getv(from_object, ["metric_resource_name"]),
+        )
+
     return to_object
 
 
@@ -2307,6 +2321,7 @@ async def _generate_rubrics(
             types.PredefinedMetricSpecOrDict
         ] = None,
         rubric_generation_spec: Optional[types.RubricGenerationSpecOrDict] = None,
+        metric_resource_name: Optional[str] = None,
         config: Optional[types.RubricGenerationConfigOrDict] = None,
     ) -> types.GenerateInstanceRubricsResponse:
         """
@@ -2317,6 +2332,7 @@ async def _generate_rubrics(
             contents=contents,
             predefined_rubric_generation_spec=predefined_rubric_generation_spec,
             rubric_generation_spec=rubric_generation_spec,
+            metric_resource_name=metric_resource_name,
             config=config,
         )
 

vertexai/_genai/types/common.py

@@ -2235,6 +2235,9 @@ class PredefinedMetricSpec(_common.BaseModel):
         default=None,
         description="""The parameters needed to run the pre-defined metric.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None, description="""The resource name of the metric definition."""
+    )
 
 
 class PredefinedMetricSpecDict(TypedDict, total=False):
@@ -2247,6 +2250,9 @@ class PredefinedMetricSpecDict(TypedDict, total=False):
     metric_spec_parameters: Optional[dict[str, Any]]
     """The parameters needed to run the pre-defined metric."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition."""
+
 
 PredefinedMetricSpecOrDict = Union[PredefinedMetricSpec, PredefinedMetricSpecDict]
 
@@ -2326,6 +2332,10 @@ class LLMBasedMetricSpec(_common.BaseModel):
         default=None,
         description="""Dynamically generate rubrics using this specification.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}""",
+    )
 
 
 class LLMBasedMetricSpecDict(TypedDict, total=False):
@@ -2350,6 +2360,9 @@ class LLMBasedMetricSpecDict(TypedDict, total=False):
     rubric_generation_spec: Optional[RubricGenerationSpecDict]
     """Dynamically generate rubrics using this specification."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 LLMBasedMetricSpecOrDict = Union[LLMBasedMetricSpec, LLMBasedMetricSpecDict]
 
@@ -2482,6 +2495,10 @@ class EvaluationRunMetric(_common.BaseModel):
     metric_config: Optional[UnifiedMetric] = Field(
         default=None, description="""The unified metric used for evaluation run."""
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}""",
+    )
 
 
 class EvaluationRunMetricDict(TypedDict, total=False):
@@ -2493,6 +2510,9 @@ class EvaluationRunMetricDict(TypedDict, total=False):
     metric_config: Optional[UnifiedMetricDict]
     """The unified metric used for evaluation run."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 EvaluationRunMetricOrDict = Union[EvaluationRunMetric, EvaluationRunMetricDict]
 
@@ -4439,6 +4459,10 @@ class Metric(_common.BaseModel):
         default=None,
         description="""Optional steering instruction parameters for the automated predefined metric.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}""",
+    )
 
     # Allow extra fields to support metric-specific config fields.
     model_config = ConfigDict(extra="allow")
@@ -4643,6 +4667,9 @@ class MetricDict(TypedDict, total=False):
     metric_spec_parameters: Optional[dict[str, Any]]
     """Optional steering instruction parameters for the automated predefined metric."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition. Example: projects/{project}/locations/{location}/evaluationMetrics/{evaluation_metric_id}"""
+
 
 MetricOrDict = Union[Metric, MetricDict]
 
@@ -5354,6 +5381,10 @@ class _GenerateInstanceRubricsRequest(_common.BaseModel):
         default=None,
         description="""Specification for how the rubrics should be generated.""",
     )
+    metric_resource_name: Optional[str] = Field(
+        default=None,
+        description="""The resource name of the metric definition to use for rubric generation.""",
+    )
     config: Optional[RubricGenerationConfig] = Field(default=None, description="""""")
 
 
@@ -5374,6 +5405,9 @@ class _GenerateInstanceRubricsRequestDict(TypedDict, total=False):
     rubric_generation_spec: Optional[RubricGenerationSpecDict]
     """Specification for how the rubrics should be generated."""
 
+    metric_resource_name: Optional[str]
+    """The resource name of the metric definition to use for rubric generation."""
+
     config: Optional[RubricGenerationConfigDict]
     """"""