Skip to content

fix: GenAI Client(evals) - Save Gemini agent run_inference results to dest #7189

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
copybara-service merged 1 commit into from
Oct 7, 2026
+80 −42
Merged

fix: GenAI Client(evals) - Save Gemini agent run_inference results to dest #7189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .release-please-manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
{
".": "2.4.0"
".": "2.3.0"
}
15 changes: 0 additions & 15 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,20 +1,5 @@
# Changelog

## [2.4.0](https://github.com/googleapis/python-aiplatform/compare/v2.3.0...v2.4.0) (2026-10-06)


### Features

* Add per-step judges and cross-region routing to evals ([5fa1455](https://github.com/googleapis/python-aiplatform/commit/5fa14552cd06179e8ea08f0ea0d29ea6cafb1ca0))
* GenAI Client(evals) - add EvaluationDataset.load_from_adk_eval_set ([4c045e1](https://github.com/googleapis/python-aiplatform/commit/4c045e16c2459d0708bca77c2a82a3c1daff0409))
* Surface the LRO name in Agent Engine create and update requests ([b1378bb](https://github.com/googleapis/python-aiplatform/commit/b1378bb54566396506dfe17041f4e8d3fb715573))
* Update discovery doc ([39693a9](https://github.com/googleapis/python-aiplatform/commit/39693a9d956449f810c18d340eb57526faa9cb00))


### Bug Fixes

* Replay session_events into a scratch session and delete it afterwards ([3647753](https://github.com/googleapis/python-aiplatform/commit/364775307aa9c0b4d30844da4c7f23c05221dfcd))

## [2.3.0](https://github.com/googleapis/python-aiplatform/compare/v2.2.0...v2.3.0) (2026-09-25)


Expand Down
9 changes: 7 additions & 2 deletions agentplatform/_genai/_evals_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2654,7 +2654,7 @@ def _execute_inference(
"Gemini Agent inference completed in %.2f seconds.",
end_time - start_time,
)
return types.EvaluationDataset(
evaluation_dataset = types.EvaluationDataset(
eval_dataset_df=results_df,
candidate_name=gemini_agent.split("/")[-1],
)
Expand Down Expand Up @@ -2759,7 +2759,12 @@ def _execute_inference(
results_df.to_json(full_dest_path, orient="records", lines=True)
logger.info("Results saved locally to: %s", full_dest_path)
except Exception as e: # pylint: disable=broad-exception-caught
logger.error("Failed to save results to %s. Error: %s", full_dest_path, e)
logger.error(
"Failed to save results to %s. Error: %s",
full_dest_path,
e,
exc_info=True,
)

return evaluation_dataset

Expand Down
2 changes: 1 addition & 1 deletion agentplatform/version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,4 @@
#
"""Version file for agentplatform."""

__version__ = "2.4.0"
__version__ = "2.3.0"
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/instance/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/instance_v1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/params/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/params_v1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/prediction/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/predict/prediction_v1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/trainingjob/definition/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1/schema/trainingjob/definition_v1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/instance/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/instance_v1beta1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/params/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/params_v1beta1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/prediction/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/predict/prediction_v1beta1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/trainingjob/definition/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/v1beta1/schema/trainingjob/definition_v1beta1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform/version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@
# limitations under the License.
#

__version__ = "2.4.0"
__version__ = "2.3.0"
2 changes: 1 addition & 1 deletion google/cloud/aiplatform_v1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion google/cloud/aiplatform_v1beta1/gapic_version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
__version__ = "2.4.0" # {x-release-please-version}
__version__ = "2.3.0" # {x-release-please-version}
2 changes: 1 addition & 1 deletion pypi/_vertex_ai_placeholder/version.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,4 @@
# limitations under the License.
#

__version__ = "2.4.0"
__version__ = "2.3.0"
2 changes: 1 addition & 1 deletion samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
],
"language": "PYTHON",
"name": "google-cloud-aiplatform",
"version": "2.4.0"
"version": "2.3.0"
},
"snippets": [
{
Expand Down
2 changes: 1 addition & 1 deletion samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1beta1.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
],
"language": "PYTHON",
"name": "google-cloud-aiplatform",
"version": "2.4.0"
"version": "2.3.0"
},
"snippets": [
{
Expand Down
48 changes: 48 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4308,6 +4308,54 @@ def test_run_inference_gemini_agent_continues_on_failure(
assert result_df["agent_data"].tolist()[0] == {}
assert mock_interactions.create.call_count == 2

@mock.patch.object(_evals_common, "_fetch_agent_config_dict")
@mock.patch.object(_evals_common, "_get_interactions_client")
@mock.patch.object(_evals_utils, "EvalDatasetLoader")
def test_run_inference_gemini_agent_saves_to_dest(
self,
mock_eval_dataset_loader,
mock_get_interactions_client,
mock_fetch_agent_config,
):
mock_fetch_agent_config.return_value = (
agentplatform_genai_types.evals.AgentConfig(agent_id="test-agent")
)
mock_df = pd.DataFrame({"prompt": ["p1"]})
mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict(
orient="records"
)
mock_interactions = mock.Mock()
mock_interactions.create.return_value = {
"id": "interaction-1",
"status": "completed",
"steps": [
{
"type": "model_output",
"content": [{"type": "text", "text": "response 1"}],
}
],
}
mock_get_interactions_client.return_value = mock_interactions

with tempfile.TemporaryDirectory() as local_dest_dir:
inference_result = self.client.evals.run_inference(
src=mock_df,
agent=_TEST_GEMINI_AGENT,
config=agentplatform_genai_types.EvalRunInferenceConfig(
dest=local_dest_dir
),
)

saved_file_path = os.path.join(local_dest_dir, "agent_run_results.jsonl")
with open(saved_file_path, "r") as f:
saved_records = [json.loads(line) for line in f]

assert saved_records == inference_result.eval_dataset_df.to_dict(
orient="records"
)
assert saved_records[0]["interaction_id"] == "interaction-1"
assert inference_result.candidate_name == "test-agent"

@mock.patch.object(_evals_common, "_get_interactions_client")
@mock.patch.object(_evals_utils, "EvalDatasetLoader")
@mock.patch.object(_evals_common.agentplatform, "Client")
Expand Down
Loading