From b8ff3a8ac735dee0d92b3ed5098f2f1effb71c3d Mon Sep 17 00:00:00 2001 From: A Vertex SDK engineer Date: Thu, 23 Jul 2026 10:18:13 -0700 Subject: [PATCH] feat: Add code_execution_region to SDK custom code execution metric. FUTURE_COPYBARA_INTEGRATE_REVIEW=https://github.com/googleapis/python-aiplatform/pull/7027 from googleapis:release-please--branches--main a5a0f275b96a8b39e606f3dd03a9d68d85b100b3 PiperOrigin-RevId: 952828943 --- agentplatform/_genai/_transformers.py | 7 ++-- .../test_custom_code_execution_metric.py | 41 +++++++++++++++++++ vertexai/_genai/_transformers.py | 7 ++-- vertexai/_genai/evals.py | 14 +++++++ vertexai/_genai/types/common.py | 9 ++++ 5 files changed, 72 insertions(+), 6 deletions(-) diff --git a/agentplatform/_genai/_transformers.py b/agentplatform/_genai/_transformers.py index 7f58b961c6..6e7ee0a42d 100644 --- a/agentplatform/_genai/_transformers.py +++ b/agentplatform/_genai/_transformers.py @@ -82,9 +82,10 @@ def t_metrics( and isinstance(getattr(metric, "custom_function", None), str) ) ) and getattr(metric, "custom_function", None): - metric_payload_item["custom_code_execution_spec"] = { - "evaluation_function": metric.custom_function - } + spec = {"evaluation_function": metric.custom_function} + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec # LLM-based metrics elif hasattr(metric, "prompt_template") and metric.prompt_template: llm_based_spec: dict[str, Any] = { diff --git a/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py b/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py index 34aed0c68d..48ec8f94d8 100644 --- a/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py +++ b/tests/unit/agentplatform/genai/replays/test_custom_code_execution_metric.py @@ -79,6 +79,47 @@ def test_custom_code_execution(client, custom_metric): assert case_result.response_candidate_results is not None +def test_custom_code_execution_with_region(client): + """Tests that code_execution_region is included in the custom code execution spec.""" + + prompts_df = pd.DataFrame({ + "prompt": ["What is 2+2?", "What is 3+3?"], + "response": ["4", "5"], + "reference": ["4", "6"], + }) + + eval_dataset = types.EvaluationDataset( + eval_dataset_df=prompts_df, + candidate_name="test_model", + ) + + metric = types.CodeExecutionMetric( + name="my_custom_code_metric", + custom_function=CODE_SNIPPET, + code_execution_region="europe-west3", + ) + + evaluation_result = client.evals.evaluate( + dataset=eval_dataset, + metrics=[metric], + ) + + assert isinstance(evaluation_result, types.EvaluationResult) + + assert evaluation_result.summary_metrics is not None + assert evaluation_result.summary_metrics + for summary in evaluation_result.summary_metrics: + assert isinstance(summary, types.AggregatedMetricResult) + assert summary.metric_name == "my_custom_code_metric" + + assert evaluation_result.eval_case_results is not None + assert evaluation_result.eval_case_results + for case_result in evaluation_result.eval_case_results: + assert isinstance(case_result, types.EvalCaseResult) + assert case_result.eval_case_index is not None + assert case_result.response_candidate_results is not None + + @pytest.mark.parametrize( "custom_metric", [ diff --git a/vertexai/_genai/_transformers.py b/vertexai/_genai/_transformers.py index fb6f477cf8..eac32096aa 100644 --- a/vertexai/_genai/_transformers.py +++ b/vertexai/_genai/_transformers.py @@ -82,9 +82,10 @@ def t_metrics( and isinstance(getattr(metric, "custom_function", None), str) ) ) and getattr(metric, "custom_function", None): - metric_payload_item["custom_code_execution_spec"] = { - "evaluation_function": metric.custom_function - } + spec = {"evaluation_function": metric.custom_function} + if getattr(metric, "code_execution_region", None): + spec["code_execution_region"] = metric.code_execution_region + metric_payload_item["custom_code_execution_spec"] = spec # LLM-based metrics elif hasattr(metric, "prompt_template") and metric.prompt_template: llm_based_spec: dict[str, Any] = { diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index 3715571d96..fe7b75a812 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -206,6 +206,13 @@ def _CustomCodeExecutionSpec_from_vertex( getv(from_object, ["evaluation_function"]), ) + if getv(from_object, ["codeExecutionRegion"]) is not None: + setv( + to_object, + ["code_execution_region"], + getv(from_object, ["codeExecutionRegion"]), + ) + return to_object @@ -228,6 +235,13 @@ def _CustomCodeExecutionSpec_to_vertex( getv(from_object, ["remote_custom_function"]), ) + if getv(from_object, ["code_execution_region"]) is not None: + setv( + to_object, + ["codeExecutionRegion"], + getv(from_object, ["code_execution_region"]), + ) + return to_object diff --git a/vertexai/_genai/types/common.py b/vertexai/_genai/types/common.py index 5bb3b8811b..8b1acbf5b1 100644 --- a/vertexai/_genai/types/common.py +++ b/vertexai/_genai/types/common.py @@ -1905,6 +1905,11 @@ class CodeExecutionMetric(Metric): description="""The Python function code to be executed on the server side.""", ) + code_execution_region: Optional[str] = Field( + default=None, + description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. If unset, the request's originating region is used.""", + ) + # You can also add hand-written validators or methods here @field_validator("custom_function") @classmethod @@ -2164,6 +2169,10 @@ def evaluate(instance: dict[str, Any]) -> float: Instance is the evaluation instance, any fields populated in the instance are available to the function as instance[field_name].""", ) + code_execution_region: Optional[str] = Field( + default=None, + description="""Optional. The region to use for code execution. If set, the Code Execution Sandbox will be invoked in the specified region regardless of the request's originating region. Supported regions: us-central1, us-east1, us-east4, us-west1, us-west4, southamerica-east1, europe-west2, europe-west3, asia-east1, asia-south1, asia-southeast1. If unset, the request's originating region is used.""", + ) class CustomCodeExecutionSpecDict(TypedDict, total=False):