Skip to content

Commit bd71da5

Browse files
jsondaicopybara-github
jsondai
authored and
copybara-github
committed
fix: GenAI Client(evals) - Parse JSON-string and array conversation history in evaluate()
PiperOrigin-RevId: 992464091
1 parent efa3980 commit bd71da5

3 files changed

Lines changed: 251 additions & 4 deletions

File tree

‎agentplatform/_genai/_evals_data_converters.py‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222

2323
from google.genai import _common
2424
from google.genai import types as genai_types
25+
import numpy as np
26+
import pandas as pd
2527
from pydantic import alias_generators
2628
from pydantic import ValidationError
2729
from typing_extensions import override
@@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
194196
% (i, type(item_dict).__name__, item_dict)
195197
)
196198
item = copy.deepcopy(item_dict)
199+
for column in ("conversation_history", "history"):
200+
value = item.get(column)
201+
if (isinstance(value, str) and not value.strip()) or (
202+
pd.api.types.is_scalar(value) and pd.isna(value)
203+
):
204+
item.pop(column, None)
197205
eval_case_id = "eval_case_%s" % i
198206
prompt_data = item.pop("prompt", None)
199207
if not prompt_data:
200208
prompt_data = item.pop("source", None)
201209

202-
conversation_history_data = item.pop("conversation_history", None)
210+
history_column = "conversation_history"
211+
conversation_history_data = item.pop(history_column, None)
203212
if conversation_history_data is None:
204-
conversation_history_data = item.pop("history", None)
213+
history_column = "history"
214+
conversation_history_data = item.pop(history_column, None)
205215
response_data = item.pop("response", None)
206216
reference_data = item.pop("reference", None)
207217
system_instruction_data = item.pop("instruction", None)
@@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
230240
"Invalid prompt type for case %s: %s" % (i, type(prompt_data))
231241
)
232242

243+
if isinstance(conversation_history_data, np.ndarray):
244+
conversation_history_data = conversation_history_data.tolist()
245+
elif isinstance(conversation_history_data, tuple):
246+
conversation_history_data = list(conversation_history_data)
247+
elif isinstance(conversation_history_data, str):
248+
try:
249+
conversation_history_data = json.loads(conversation_history_data)
250+
except json.JSONDecodeError as e:
251+
logger.warning(
252+
"Could not decode JSON string in '%s' column for case %s:"
253+
" %s. Skipping conversation history.",
254+
history_column,
255+
eval_case_id,
256+
e,
257+
)
258+
conversation_history_data = None
259+
if conversation_history_data is not None and not isinstance(
260+
conversation_history_data, list
261+
):
262+
logger.warning(
263+
"Invalid type in '%s' column for case %s. Expected a list or a"
264+
" JSON string of a list, but got %s. Skipping conversation"
265+
" history.",
266+
history_column,
267+
eval_case_id,
268+
type(conversation_history_data),
269+
)
233270
conversation_history: Optional[list[types.evals.Message]] = None
234271
if isinstance(conversation_history_data, list):
235272
conversation_history = []

‎tests/unit/agentplatform/genai/test_evals.py‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import asyncio
1919
import enum
2020
import importlib
21+
import io
2122
import json
2223
import os
2324
import re
@@ -45,9 +46,13 @@
4546
types as agentplatform_genai_types,
4647
)
4748
from agentplatform._genai.types import common as common_types
49+
from vertexai._genai import (
50+
_evals_data_converters as vertexai_evals_data_converters,
51+
)
4852
from google.genai import client
4953
from google.genai import errors as genai_errors
5054
from google.genai import types as genai_types
55+
import numpy as np
5156
import pandas as pd
5257
import pydantic
5358
import pytest
@@ -61,6 +66,12 @@
6166

6267
pytestmark = pytest.mark.usefixtures("google_auth_mock")
6368

69+
_CONVERTER_MODULES = pytest.mark.parametrize(
70+
"converters",
71+
[_evals_data_converters, vertexai_evals_data_converters],
72+
ids=["agent_platform", "vertexai"],
73+
)
74+
6475

6576
class TestDropEmptyColumns:
6677
"""Unit tests for the _drop_empty_columns function."""
@@ -5671,6 +5682,16 @@ def test_convert_with_raw_string_response(self):
56715682
)
56725683

56735684

5685+
_GEMINI_HISTORY = [
5686+
{"role": "user", "parts": [{"text": "My code word is BLUE."}]},
5687+
{"role": "model", "parts": [{"text": "Noted."}]},
5688+
]
5689+
5690+
5691+
def _csv_round_trip(row: dict[str, object]) -> pd.DataFrame:
5692+
return pd.read_csv(io.StringIO(pd.DataFrame([row]).to_csv(index=False)))
5693+
5694+
56745695
class TestFlattenEvalDataConverter:
56755696
"""Unit tests for the _FlattenEvalDataConverter class."""
56765697

@@ -5892,6 +5913,158 @@ def test_convert_with_intermediate_events_as_event_objects(self):
58925913
== "intermediate event"
58935914
)
58945915

5916+
@_CONVERTER_MODULES
5917+
@pytest.mark.parametrize("column", ["conversation_history", "history"])
5918+
def test_convert_json_string_history_from_csv(self, converters, column):
5919+
raw_data_df = _csv_round_trip(
5920+
{
5921+
"prompt": "Code word?",
5922+
"response": "BLUE",
5923+
column: json.dumps(_GEMINI_HISTORY),
5924+
}
5925+
)
5926+
5927+
result_dataset = converters._FlattenEvalDataConverter().convert(
5928+
raw_data_df.to_dict(orient="records")
5929+
)
5930+
5931+
assert [
5932+
message.content
5933+
for message in result_dataset.eval_cases[0].conversation_history
5934+
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]
5935+
5936+
@_CONVERTER_MODULES
5937+
@pytest.mark.parametrize(
5938+
"history",
5939+
[
5940+
np.array(
5941+
[
5942+
{"role": turn["role"], "parts": np.array(turn["parts"])}
5943+
for turn in _GEMINI_HISTORY
5944+
]
5945+
),
5946+
tuple(_GEMINI_HISTORY),
5947+
],
5948+
ids=["ndarray", "tuple"],
5949+
)
5950+
def test_convert_array_history(self, converters, history):
5951+
result_dataset = converters._FlattenEvalDataConverter().convert(
5952+
[{"prompt": "Code word?", "response": "BLUE", "history": history}]
5953+
)
5954+
5955+
assert [
5956+
message.content
5957+
for message in result_dataset.eval_cases[0].conversation_history
5958+
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]
5959+
5960+
@_CONVERTER_MODULES
5961+
@pytest.mark.parametrize(
5962+
"column,value,expected_warning",
5963+
[
5964+
(
5965+
"conversation_history",
5966+
"[not json",
5967+
"Could not decode JSON string in 'conversation_history' column for"
5968+
" case eval_case_0",
5969+
),
5970+
(
5971+
"history",
5972+
json.dumps(_GEMINI_HISTORY[0]),
5973+
"Invalid type in 'history' column for case eval_case_0. Expected",
5974+
),
5975+
(
5976+
"history",
5977+
np.array(1.5),
5978+
"Invalid type in 'history' column for case eval_case_0. Expected",
5979+
),
5980+
],
5981+
ids=["invalid_json", "json_object", "zero_dim_array"],
5982+
)
5983+
def test_convert_invalid_history_value_logs_warning(
5984+
self, converters, column, value, expected_warning, caplog
5985+
):
5986+
with caplog.at_level("WARNING", logger=converters.logger.name):
5987+
result_dataset = converters._FlattenEvalDataConverter().convert(
5988+
[{"prompt": "Hello", "response": "Hi", column: value}]
5989+
)
5990+
5991+
assert result_dataset.eval_cases[0].conversation_history is None
5992+
assert len(caplog.records) == 1
5993+
assert expected_warning in caplog.text
5994+
5995+
@_CONVERTER_MODULES
5996+
@pytest.mark.parametrize(
5997+
"value",
5998+
[None, float("nan"), np.float32("nan"), pd.NA, pd.NaT, " "],
5999+
ids=["none", "nan", "float32_nan", "pd_na", "pd_nat", "blank"],
6000+
)
6001+
def test_convert_empty_history_value_without_warning(
6002+
self, converters, value, caplog
6003+
):
6004+
with caplog.at_level("WARNING", logger=converters.logger.name):
6005+
result_dataset = converters._FlattenEvalDataConverter().convert(
6006+
[{"prompt": "Hello", "response": "Hi", "conversation_history": value}]
6007+
)
6008+
6009+
assert result_dataset.eval_cases[0].conversation_history is None
6010+
assert not caplog.records
6011+
6012+
@_CONVERTER_MODULES
6013+
def test_convert_missing_conversation_history_falls_back_to_history(
6014+
self, converters
6015+
):
6016+
raw_data_df = _csv_round_trip(
6017+
{
6018+
"prompt": "Code word?",
6019+
"response": "BLUE",
6020+
"conversation_history": None,
6021+
"history": json.dumps(_GEMINI_HISTORY),
6022+
}
6023+
)
6024+
6025+
result_dataset = converters._FlattenEvalDataConverter().convert(
6026+
raw_data_df.to_dict(orient="records")
6027+
)
6028+
6029+
assert [
6030+
message.content
6031+
for message in result_dataset.eval_cases[0].conversation_history
6032+
] == [genai_types.Content.model_validate(turn) for turn in _GEMINI_HISTORY]
6033+
6034+
@pytest.mark.usefixtures("mock_eval_dependencies")
6035+
def test_evaluate_sends_csv_history_to_multi_turn_metric(
6036+
self, mock_api_client_fixture
6037+
):
6038+
dataset_df = _csv_round_trip(
6039+
{
6040+
"prompt": "Code word?",
6041+
"response": "BLUE",
6042+
"conversation_history": json.dumps(_GEMINI_HISTORY),
6043+
}
6044+
)
6045+
6046+
with mock.patch.object(
6047+
evals.Evals, "_evaluate_instances"
6048+
) as mock_evaluate_instances:
6049+
mock_evaluate_instances.return_value = (
6050+
agentplatform_genai_types.EvaluateInstancesResponse(
6051+
metric_results=[agentplatform_genai_types.MetricResult(score=1.0)]
6052+
)
6053+
)
6054+
evals.Evals(api_client_=mock_api_client_fixture).evaluate(
6055+
dataset=dataset_df,
6056+
metrics=[
6057+
agentplatform_genai_types.Metric(
6058+
name="multi_turn_general_quality_v1"
6059+
)
6060+
],
6061+
)
6062+
6063+
instance = mock_evaluate_instances.call_args.kwargs["instance"]
6064+
assert [
6065+
content.parts[0].text for content in instance.prompt.contents.contents
6066+
] == ["My code word is BLUE.", "Noted.", "Code word?"]
6067+
58956068

58966069
class TestOpenAIDataConverter:
58976070
"""Unit tests for the _OpenAIDataConverter class."""

‎vertexai/_genai/_evals_data_converters.py‎

Lines changed: 39 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222

2323
from google.genai import _common
2424
from google.genai import types as genai_types
25+
import numpy as np
26+
import pandas as pd
2527
from pydantic import alias_generators
2628
from pydantic import ValidationError
2729
from typing_extensions import override
@@ -194,14 +196,22 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
194196
% (i, type(item_dict).__name__, item_dict)
195197
)
196198
item = copy.deepcopy(item_dict)
199+
for column in ("conversation_history", "history"):
200+
value = item.get(column)
201+
if (isinstance(value, str) and not value.strip()) or (
202+
pd.api.types.is_scalar(value) and pd.isna(value)
203+
):
204+
item.pop(column, None)
197205
eval_case_id = "eval_case_%s" % i
198206
prompt_data = item.pop("prompt", None)
199207
if not prompt_data:
200208
prompt_data = item.pop("source", None)
201209

202-
conversation_history_data = item.pop("conversation_history", None)
210+
history_column = "conversation_history"
211+
conversation_history_data = item.pop(history_column, None)
203212
if conversation_history_data is None:
204-
conversation_history_data = item.pop("history", None)
213+
history_column = "history"
214+
conversation_history_data = item.pop(history_column, None)
205215
response_data = item.pop("response", None)
206216
reference_data = item.pop("reference", None)
207217
system_instruction_data = item.pop("instruction", None)
@@ -230,6 +240,33 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
230240
"Invalid prompt type for case %s: %s" % (i, type(prompt_data))
231241
)
232242

243+
if isinstance(conversation_history_data, np.ndarray):
244+
conversation_history_data = conversation_history_data.tolist()
245+
elif isinstance(conversation_history_data, tuple):
246+
conversation_history_data = list(conversation_history_data)
247+
elif isinstance(conversation_history_data, str):
248+
try:
249+
conversation_history_data = json.loads(conversation_history_data)
250+
except json.JSONDecodeError as e:
251+
logger.warning(
252+
"Could not decode JSON string in '%s' column for case %s:"
253+
" %s. Skipping conversation history.",
254+
history_column,
255+
eval_case_id,
256+
e,
257+
)
258+
conversation_history_data = None
259+
if conversation_history_data is not None and not isinstance(
260+
conversation_history_data, list
261+
):
262+
logger.warning(
263+
"Invalid type in '%s' column for case %s. Expected a list or a"
264+
" JSON string of a list, but got %s. Skipping conversation"
265+
" history.",
266+
history_column,
267+
eval_case_id,
268+
type(conversation_history_data),
269+
)
233270
conversation_history: Optional[list[types.evals.Message]] = None
234271
if isinstance(conversation_history_data, list):
235272
conversation_history = []

0 commit comments

Comments
 (0)