diff --git a/agentplatform/_genai/_evals_data_converters.py b/agentplatform/_genai/_evals_data_converters.py index 2934ba92ea..308e6402ed 100644 --- a/agentplatform/_genai/_evals_data_converters.py +++ b/agentplatform/_genai/_evals_data_converters.py @@ -57,6 +57,20 @@ def _create_placeholder_response_candidate( ) +def _openai_message_to_eval_message( + turn_id: int, message: dict[str, Any] +) -> types.evals.Message: + """Converts an OpenAI chat message into a conversation history message.""" + role = message.get("role", "user") + return types.evals.Message( + turn_id=str(turn_id), + content=genai_types.Content( + parts=[genai_types.Part(text=message.get("content", ""))], role=role + ), + author=role, + ) + + class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter): """Converter for dataset in the Gemini format.""" @@ -199,9 +213,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -241,6 +257,16 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: content=content, ) ) + elif isinstance(content, types.evals.Message): + conversation_history.append(content) + elif ( + isinstance(content, dict) + and isinstance(content.get("content"), str) + and isinstance(content.get("role", "user"), str) + ): + conversation_history.append( + _openai_message_to_eval_message(turn_id, content) + ) elif isinstance(content, dict): try: validated_content = genai_types.Content.model_validate( @@ -254,18 +280,20 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: ) except ValidationError as e: logger.warning( - "Item at index %s in 'history' column for case " + "Item at index %s in '%s' column for case" " %s is a dict but could not be validated as" " genai_types.Content: %s", turn_id, + history_column, eval_case_id, e, ) else: logger.warning( - "Invalid type in 'history' column for case %s at index %s. " - "Expected genai_types.Content or dict, but got %s. " - "Skipping this history item.", + "Invalid type in '%s' column for case %s at index %s. " + "Expected genai_types.Content, types.evals.Message or " + "dict, but got %s. Skipping this history item.", + history_column, eval_case_id, turn_id, type(content), @@ -492,17 +520,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[ messages = messages[1:] for turn_id, msg in enumerate(messages): - role = msg.get("role", "user") - content = msg.get("content", "") - conversation_history.append( - types.evals.Message( - turn_id=str(turn_id), - content=genai_types.Content( - parts=[genai_types.Part(text=content)], role=role - ), - author=role, - ) - ) + conversation_history.append(_openai_message_to_eval_message(turn_id, msg)) if conversation_history: last_message = conversation_history.pop() diff --git a/tests/unit/agentplatform/genai/test_evals.py b/tests/unit/agentplatform/genai/test_evals.py index 492d9cb429..e529a9775d 100644 --- a/tests/unit/agentplatform/genai/test_evals.py +++ b/tests/unit/agentplatform/genai/test_evals.py @@ -45,6 +45,9 @@ types as agentplatform_genai_types, ) from agentplatform._genai.types import common as common_types +from vertexai._genai import ( + _evals_data_converters as vertexai_evals_data_converters, +) from google.genai import client from google.genai import errors as genai_errors from google.genai import types as genai_types @@ -61,6 +64,12 @@ pytestmark = pytest.mark.usefixtures("google_auth_mock") +_CONVERTER_MODULES = pytest.mark.parametrize( + "converters", + [_evals_data_converters, vertexai_evals_data_converters], + ids=["agent_platform", "vertexai"], +) + class TestDropEmptyColumns: """Unit tests for the _drop_empty_columns function.""" @@ -5778,6 +5787,84 @@ def test_convert_with_conversation_history_column_name(self): eval_case.conversation_history[1].content.parts[0].text == "Old model msg" ) + @_CONVERTER_MODULES + def test_convert_openai_style_history(self, converters): + result_dataset = converters._FlattenEvalDataConverter().convert( + [ + { + "prompt": "Code word?", + "response": "BLUE", + "conversation_history": [ + {"content": "My code word is BLUE."}, + {"role": "assistant", "content": "Noted."}, + ], + } + ] + ) + + assert result_dataset.eval_cases[0].conversation_history == [ + converters.types.evals.Message( + turn_id="0", + content=genai_types.Content( + parts=[genai_types.Part(text="My code word is BLUE.")], role="user" + ), + author="user", + ), + converters.types.evals.Message( + turn_id="1", + content=genai_types.Content( + parts=[genai_types.Part(text="Noted.")], role="assistant" + ), + author="assistant", + ), + ] + + @_CONVERTER_MODULES + def test_convert_message_history_items(self, converters): + history = [ + converters.types.evals.Message( + turn_id="turn-0", + content=genai_types.Content( + parts=[genai_types.Part(text="My code word is BLUE.")], role="user" + ), + author="user", + ) + ] + + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Code word?", "response": "BLUE", "history": history}] + ) + + assert result_dataset.eval_cases[0].conversation_history == history + + @_CONVERTER_MODULES + @pytest.mark.parametrize("column", ["conversation_history", "history"]) + @pytest.mark.parametrize( + "item,expected_warning", + [ + ( + {"role": "user", "text": "Hi"}, + "Item at index 0 in '{column}' column for case eval_case_0 is a dict", + ), + ( + {"role": 1, "content": "Hi"}, + "Item at index 0 in '{column}' column for case eval_case_0 is a dict", + ), + (42, "Invalid type in '{column}' column for case eval_case_0 at index 0."), + ], + ids=["invalid_dict_item", "non_str_role", "invalid_item_type"], + ) + def test_convert_invalid_history_item_logs_warning( + self, converters, column, item, expected_warning, caplog + ): + with caplog.at_level("WARNING", logger=converters.logger.name): + result_dataset = converters._FlattenEvalDataConverter().convert( + [{"prompt": "Hello", "response": "Hi", column: [item]}] + ) + + assert result_dataset.eval_cases[0].conversation_history == [] + assert expected_warning.format(column=column) in caplog.text + def test_convert_missing_response_raises_value_error(self): raw_data_df = pd.DataFrame({"prompt": ["Hello"]}) # Missing response raw_data = raw_data_df.to_dict(orient="records") @@ -6032,6 +6119,20 @@ def test_convert_skips_missing_request_or_response(self): result_dataset = self.converter.convert(raw_data) assert len(result_dataset.eval_cases) == 0 + @pytest.mark.parametrize( + "message,role,text", + [({"role": "assistant", "content": "Hi"}, "assistant", "Hi"), ({}, "user", "")], + ids=["role_and_content", "defaults"], + ) + def test_openai_message_to_eval_message(self, message, role, text): + assert _evals_data_converters._openai_message_to_eval_message( + 1, message + ) == agentplatform_genai_types.evals.Message( + turn_id="1", + content=genai_types.Content(parts=[genai_types.Part(text=text)], role=role), + author=role, + ) + class TestObservabilityDataConverter: """Unit tests for the ObservabilityDataConverter class.""" diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index f293152cbe..cda2c36ca8 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -57,6 +57,20 @@ def _create_placeholder_response_candidate( ) +def _openai_message_to_eval_message( + turn_id: int, message: dict[str, Any] +) -> types.evals.Message: + """Converts an OpenAI chat message into a conversation history message.""" + role = message.get("role", "user") + return types.evals.Message( + turn_id=str(turn_id), + content=genai_types.Content( + parts=[genai_types.Part(text=message.get("content", ""))], role=role + ), + author=role, + ) + + class _GeminiEvalDataConverter(_evals_utils.EvalDataConverter): """Converter for dataset in the Gemini format.""" @@ -199,9 +213,11 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: if not prompt_data: prompt_data = item.pop("source", None) - conversation_history_data = item.pop("conversation_history", None) + history_column = "conversation_history" + conversation_history_data = item.pop(history_column, None) if conversation_history_data is None: - conversation_history_data = item.pop("history", None) + history_column = "history" + conversation_history_data = item.pop(history_column, None) response_data = item.pop("response", None) reference_data = item.pop("reference", None) system_instruction_data = item.pop("instruction", None) @@ -241,6 +257,16 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: content=content, ) ) + elif isinstance(content, types.evals.Message): + conversation_history.append(content) + elif ( + isinstance(content, dict) + and isinstance(content.get("content"), str) + and isinstance(content.get("role", "user"), str) + ): + conversation_history.append( + _openai_message_to_eval_message(turn_id, content) + ) elif isinstance(content, dict): try: validated_content = genai_types.Content.model_validate( @@ -254,18 +280,20 @@ def convert(self, raw_data: list[dict[str, Any]]) -> types.EvaluationDataset: ) except ValidationError as e: logger.warning( - "Item at index %s in 'history' column for case " + "Item at index %s in '%s' column for case" " %s is a dict but could not be validated as" " genai_types.Content: %s", turn_id, + history_column, eval_case_id, e, ) else: logger.warning( - "Invalid type in 'history' column for case %s at index %s. " - "Expected genai_types.Content or dict, but got %s. " - "Skipping this history item.", + "Invalid type in '%s' column for case %s at index %s. " + "Expected genai_types.Content, types.evals.Message or " + "dict, but got %s. Skipping this history item.", + history_column, eval_case_id, turn_id, type(content), @@ -492,17 +520,7 @@ def _parse_messages(self, messages: list[dict[str, Any]]) -> tuple[ messages = messages[1:] for turn_id, msg in enumerate(messages): - role = msg.get("role", "user") - content = msg.get("content", "") - conversation_history.append( - types.evals.Message( - turn_id=str(turn_id), - content=genai_types.Content( - parts=[genai_types.Part(text=content)], role=role - ), - author=role, - ) - ) + conversation_history.append(_openai_message_to_eval_message(turn_id, msg)) if conversation_history: last_message = conversation_history.pop()