Skip to content

fix: GenAI Client(evals) - Accept OpenAI-style and Message items in conversation history #7206

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
copybara-service wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: GenAI Client(evals) - Accept OpenAI-style and Message items in conversation history #7206

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 35 additions & 17 deletions agentplatform/_genai/_evals_data_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,20 @@ def _create_placeholder_response_candidate(
)


def _openai_message_to_eval_message(
turn_id: int, message: dict[str, Any]
) -> types.evals.Message:
"""Converts an OpenAI chat message into a conversation history message."""
role = message.get("role", "user")
return types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=message.get("content", ""))], role=role
),
author=role,
)


class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter):
"""Converter for dataset in the Gemini format."""

Expand Down Expand Up @@ -199,9 +213,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
if not prompt_data:
prompt_data = item.pop("source", None)

conversation_history_data = item.pop("conversation_history", None)
history_column = "conversation_history"
conversation_history_data = item.pop(history_column, None)
if conversation_history_data is None:
conversation_history_data = item.pop("history", None)
history_column = "history"
conversation_history_data = item.pop(history_column, None)
response_data = item.pop("response", None)
reference_data = item.pop("reference", None)
system_instruction_data = item.pop("instruction", None)
Expand Down Expand Up @@ -241,6 +257,16 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
content=content,
)
)
elif isinstance(content, types.evals.Message):
conversation_history.append(content)
elif (
isinstance(content, dict)
and isinstance(content.get("content"), str)
and isinstance(content.get("role", "user"), str)
):
conversation_history.append(
_openai_message_to_eval_message(turn_id, content)
)
elif isinstance(content, dict):
try:
validated_content = genai_types.Content.model_validate(
Expand All @@ -254,18 +280,20 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
)
except ValidationError as e:
logger.warning(
"Item at index %s in 'history' column for case "
"Item at index %s in '%s' column for case"
" %s is a dict but could not be validated as"
" genai_types.Content: %s",
turn_id,
history_column,
eval_case_id,
e,
)
else:
logger.warning(
"Invalid type in 'history' column for case %s at index %s. "
"Expected genai_types.Content or dict, but got %s. "
"Skipping this history item.",
"Invalid type in '%s' column for case %s at index %s. "
"Expected genai_types.Content, types.evals.Message or "
"dict, but got %s. Skipping this history item.",
history_column,
eval_case_id,
turn_id,
type(content),
Expand Down Expand Up @@ -492,17 +520,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[
messages = messages[1:]

for turn_id, msg in enumerate(messages):
role = msg.get("role", "user")
content = msg.get("content", "")
conversation_history.append(
types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=content)], role=role
),
author=role,
)
)
conversation_history.append(_openai_message_to_eval_message(turn_id, msg))

if conversation_history:
last_message = conversation_history.pop()
Expand Down
101 changes: 101 additions & 0 deletions tests/unit/agentplatform/genai/test_evals.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,9 @@
types as agentplatform_genai_types,
)
from agentplatform._genai.types import common as common_types
from vertexai._genai import (
_evals_data_converters as vertexai_evals_data_converters,
)
from google.genai import client
from google.genai import errors as genai_errors
from google.genai import types as genai_types
Expand All @@ -61,6 +64,12 @@

pytestmark = pytest.mark.usefixtures("google_auth_mock")

_CONVERTER_MODULES = pytest.mark.parametrize(
"converters",
[_evals_data_converters, vertexai_evals_data_converters],
ids=["agent_platform", "vertexai"],
)


class TestDropEmptyColumns:
"""Unit tests for the _drop_empty_columns function."""
Expand Down Expand Up @@ -5778,6 +5787,84 @@ def test_convert_with_conversation_history_column_name(self):
eval_case.conversation_history[1].content.parts[0].text == "Old model msg"
)

@_CONVERTER_MODULES
def test_convert_openai_style_history(self, converters):
result_dataset = converters._FlattenEvalDataConverter().convert(
[
{
"prompt": "Code word?",
"response": "BLUE",
"conversation_history": [
{"content": "My code word is BLUE."},
{"role": "assistant", "content": "Noted."},
],
}
]
)

assert result_dataset.eval_cases[0].conversation_history == [
converters.types.evals.Message(
turn_id="0",
content=genai_types.Content(
parts=[genai_types.Part(text="My code word is BLUE.")], role="user"
),
author="user",
),
converters.types.evals.Message(
turn_id="1",
content=genai_types.Content(
parts=[genai_types.Part(text="Noted.")], role="assistant"
),
author="assistant",
),
]

@_CONVERTER_MODULES
def test_convert_message_history_items(self, converters):
history = [
converters.types.evals.Message(
turn_id="turn-0",
content=genai_types.Content(
parts=[genai_types.Part(text="My code word is BLUE.")], role="user"
),
author="user",
)
]

result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Code word?", "response": "BLUE", "history": history}]
)

assert result_dataset.eval_cases[0].conversation_history == history

@_CONVERTER_MODULES
@pytest.mark.parametrize("column", ["conversation_history", "history"])
@pytest.mark.parametrize(
"item,expected_warning",
[
(
{"role": "user", "text": "Hi"},
"Item at index 0 in '{column}' column for case eval_case_0 is a dict",
),
(
{"role": 1, "content": "Hi"},
"Item at index 0 in '{column}' column for case eval_case_0 is a dict",
),
(42, "Invalid type in '{column}' column for case eval_case_0 at index 0."),
],
ids=["invalid_dict_item", "non_str_role", "invalid_item_type"],
)
def test_convert_invalid_history_item_logs_warning(
self, converters, column, item, expected_warning, caplog
):
with caplog.at_level("WARNING", logger=converters.logger.name):
result_dataset = converters._FlattenEvalDataConverter().convert(
[{"prompt": "Hello", "response": "Hi", column: [item]}]
)

assert result_dataset.eval_cases[0].conversation_history == []
assert expected_warning.format(column=column) in caplog.text

def test_convert_missing_response_raises_value_error(self):
raw_data_df = pd.DataFrame({"prompt": ["Hello"]}) # Missing response
raw_data = raw_data_df.to_dict(orient="records")
Expand Down Expand Up @@ -6032,6 +6119,20 @@ def test_convert_skips_missing_request_or_response(self):
result_dataset = self.converter.convert(raw_data)
assert len(result_dataset.eval_cases) == 0

@pytest.mark.parametrize(
"message,role,text",
[({"role": "assistant", "content": "Hi"}, "assistant", "Hi"), ({}, "user", "")],
ids=["role_and_content", "defaults"],
)
def test_openai_message_to_eval_message(self, message, role, text):
assert _evals_data_converters._openai_message_to_eval_message(
1, message
) == agentplatform_genai_types.evals.Message(
turn_id="1",
content=genai_types.Content(parts=[genai_types.Part(text=text)], role=role),
author=role,
)


class TestObservabilityDataConverter:
"""Unit tests for the ObservabilityDataConverter class."""
Expand Down
52 changes: 35 additions & 17 deletions vertexai/_genai/_evals_data_converters.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,20 @@ def _create_placeholder_response_candidate(
)


def _openai_message_to_eval_message(
turn_id: int, message: dict[str, Any]
) -> types.evals.Message:
"""Converts an OpenAI chat message into a conversation history message."""
role = message.get("role", "user")
return types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=message.get("content", ""))], role=role
),
author=role,
)


class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter):
"""Converter for dataset in the Gemini format."""

Expand Down Expand Up @@ -199,9 +213,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
if not prompt_data:
prompt_data = item.pop("source", None)

conversation_history_data = item.pop("conversation_history", None)
history_column = "conversation_history"
conversation_history_data = item.pop(history_column, None)
if conversation_history_data is None:
conversation_history_data = item.pop("history", None)
history_column = "history"
conversation_history_data = item.pop(history_column, None)
response_data = item.pop("response", None)
reference_data = item.pop("reference", None)
system_instruction_data = item.pop("instruction", None)
Expand Down Expand Up @@ -241,6 +257,16 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
content=content,
)
)
elif isinstance(content, types.evals.Message):
conversation_history.append(content)
elif (
isinstance(content, dict)
and isinstance(content.get("content"), str)
and isinstance(content.get("role", "user"), str)
):
conversation_history.append(
_openai_message_to_eval_message(turn_id, content)
)
elif isinstance(content, dict):
try:
validated_content = genai_types.Content.model_validate(
Expand All @@ -254,18 +280,20 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset:
)
except ValidationError as e:
logger.warning(
"Item at index %s in 'history' column for case "
"Item at index %s in '%s' column for case"
" %s is a dict but could not be validated as"
" genai_types.Content: %s",
turn_id,
history_column,
eval_case_id,
e,
)
else:
logger.warning(
"Invalid type in 'history' column for case %s at index %s. "
"Expected genai_types.Content or dict, but got %s. "
"Skipping this history item.",
"Invalid type in '%s' column for case %s at index %s. "
"Expected genai_types.Content, types.evals.Message or "
"dict, but got %s. Skipping this history item.",
history_column,
eval_case_id,
turn_id,
type(content),
Expand Down Expand Up @@ -492,17 +520,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[
messages = messages[1:]

for turn_id, msg in enumerate(messages):
role = msg.get("role", "user")
content = msg.get("content", "")
conversation_history.append(
types.evals.Message(
turn_id=str(turn_id),
content=genai_types.Content(
parts=[genai_types.Part(text=content)], role=role
),
author=role,
)
)
conversation_history.append(_openai_message_to_eval_message(turn_id, msg))

if conversation_history:
last_message = conversation_history.pop()
Expand Down
Loading