diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index f7c07cec73..dac924f72b 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -14,6 +14,7 @@ from __future__ import annotations +import re from typing import Optional from google.genai import types as genai_types @@ -96,6 +97,30 @@ def _get_eval_status(score: float, threshold: float) -> EvalStatus: return EvalStatus.PASSED if score >= threshold else EvalStatus.FAILED +class _UnicodeTokenizer(rouge_scorer.tokenizers.Tokenizer): + """A tokenizer that handles non-ASCII text (e.g. Thai, Chinese, Japanese). + + The default rouge_score tokenizer strips all non-ASCII characters, which + causes every non-English token to be discarded, resulting in a score of 0.0 + even for identical non-English strings. This tokenizer preserves Unicode + characters by splitting non-ASCII tokens into individual characters so that + character-level overlap can be measured correctly. + """ + + def tokenize(self, text): + text = text.lower() + tokens = re.split(r"\s+", text) + result = [] + for token in tokens: + if re.match(r"^[a-z0-9]+$", token): + result.append(token) + elif token: + # For non-ASCII tokens (e.g. Thai, Chinese), fall back to character- + # level tokenization so that identical strings score 1.0. + result.extend(list(token)) + return [t for t in result if t.strip()] + + def _calculate_rouge_1_scores(candidate: str, reference: str): """Calculates the ROUGE-1 score between a candidate and reference text. @@ -114,7 +139,9 @@ def _calculate_rouge_1_scores(candidate: str, reference: str): Returns: A dictionary containing the ROUGE-1 precision, recall, and f-measure. """ - scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True) + scorer = rouge_scorer.RougeScorer( + ["rouge1"], tokenizer=_UnicodeTokenizer() + ) # The score method returns a dictionary where keys are the ROUGE types # and values are Score objects (tuples) with precision, recall, and fmeasure. diff --git a/tests/unittests/evaluation/test_response_evaluator.py b/tests/unittests/evaluation/test_response_evaluator.py index 5d093ca08f..873c1bf812 100644 --- a/tests/unittests/evaluation/test_response_evaluator.py +++ b/tests/unittests/evaluation/test_response_evaluator.py @@ -70,6 +70,44 @@ def test_evaluate_invocations_rouge_metric(self, mocker): assert evaluation_result.overall_eval_status == EvalStatus.FAILED mock_perform_eval.assert_not_called() # Ensure _perform_eval was not called + def test_evaluate_invocations_rouge_metric_non_english(self, mocker): + """Test that ROUGE metric correctly scores non-English (Thai) text.""" + mocker.patch( + "google.adk.evaluation.vertex_ai_eval_facade._VertexAiEvalFacade._perform_eval" + ) + thai_text = "สวัสดี" + actual_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="สวัสดี?")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text=thai_text)] + ), + ) + ] + expected_invocations = [ + Invocation( + user_content=genai_types.Content( + parts=[genai_types.Part(text="สวัสดี?")] + ), + final_response=genai_types.Content( + parts=[genai_types.Part(text=thai_text)] + ), + ) + ] + evaluator = ResponseEvaluator( + threshold=0.5, metric_name="response_match_score" + ) + + evaluation_result = evaluator.evaluate_invocations( + actual_invocations, expected_invocations + ) + + # Identical Thai strings should score 1.0, not 0.0. + assert evaluation_result.overall_score == pytest.approx(1.0) + assert evaluation_result.overall_eval_status == EvalStatus.PASSED + def test_evaluate_invocations_coherence_metric_passed(self, mocker): """Test evaluate_invocations function for Coherence metric.""" mock_perform_eval = mocker.patch(