From f7d8ade5c0cb94d7d27a5671194ae04b328752a3 Mon Sep 17 00:00:00 2001 From: Peter Tomko Date: Thu, 27 Aug 2026 12:05:08 +0200 Subject: [PATCH] feat(gooddata-eval): register knowledge_question evaluator knowledge_question had reclassified fixtures (commit 658b9ea6) but no registered evaluator, so get_evaluator() raised KeyError and every item was silently skipped -- data/test_kinds.yaml had to keep it disabled. Reuses GeneralQuestionEvaluator directly: both are free-text-rubric, LLM-judged prose answers, and ItemReport.test_kind is tagged from the dataset item's own field rather than the evaluator class, so sharing one class across both kinds doesn't mislabel results. --- .../gooddata_eval/core/evaluators/__init__.py | 12 ++++++-- packages/gooddata-eval/tests/test_runner.py | 1 + .../tests/test_text_evaluators.py | 28 +++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py index f2b71955a..9ef6816ab 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/evaluators/__init__.py @@ -20,18 +20,26 @@ ) } -# LLM-judge evaluators (general_question, guardrail, dashboard_summary) require the -# [llm-judge] extra. Their modules are imported lazily on first use so the CLI +# LLM-judge evaluators (general_question, guardrail, dashboard_summary, knowledge_question) +# require the [llm-judge] extra. Their modules are imported lazily on first use so the CLI # starts without openai. +# +# knowledge_question reuses GeneralQuestionEvaluator directly: both are free-text-rubric, +# LLM-judged prose answers -- knowledge_question just covers platform/product/policy facts +# instead of LDM-grounded ones. `ItemReport.test_kind` is tagged from the dataset item's own +# `test_kind` field (see runner.py), not from the evaluator class, so sharing one class +# across both kinds does not mislabel results. _LAZY_EVALUATOR_MODULES: dict[str, str] = { "general_question": "gooddata_eval.core.evaluators.general_question", "guardrail": "gooddata_eval.core.evaluators.guardrail", "dashboard_summary": "gooddata_eval.core.evaluators.summary", + "knowledge_question": "gooddata_eval.core.evaluators.general_question", } _LAZY_EVALUATOR_CLASSES: dict[str, str] = { "general_question": "GeneralQuestionEvaluator", "guardrail": "GuardrailEvaluator", "dashboard_summary": "DashboardSummaryEvaluator", + "knowledge_question": "GeneralQuestionEvaluator", } diff --git a/packages/gooddata-eval/tests/test_runner.py b/packages/gooddata-eval/tests/test_runner.py index 11fb4b2f6..11c3c479b 100644 --- a/packages/gooddata-eval/tests/test_runner.py +++ b/packages/gooddata-eval/tests/test_runner.py @@ -120,6 +120,7 @@ def test_run_items_routes_all_supported_kinds(): "general_question", "guardrail", "dashboard_summary", + "knowledge_question", } assert expected_kinds == supported_test_kinds() diff --git a/packages/gooddata-eval/tests/test_text_evaluators.py b/packages/gooddata-eval/tests/test_text_evaluators.py index 93ffb36bb..6bb8a9420 100644 --- a/packages/gooddata-eval/tests/test_text_evaluators.py +++ b/packages/gooddata-eval/tests/test_text_evaluators.py @@ -1,6 +1,7 @@ # (C) 2026 GoodData Corporation from unittest.mock import MagicMock, patch +from gooddata_eval.core.evaluators import get_evaluator from gooddata_eval.core.evaluators.general_question import GeneralQuestionEvaluator from gooddata_eval.core.evaluators.guardrail import GuardrailEvaluator from gooddata_eval.core.models import ChatResult, DatasetItem @@ -16,6 +17,16 @@ def _gq_item() -> DatasetItem: ) +def _kq_item() -> DatasetItem: + return DatasetItem( + id="kq-001", + dataset_name="d", + test_kind="knowledge_question", + question="How can I get access to more ICAs?", + expected_output="Must explain the process for requesting additional ICAs, e.g. contacting an account manager.", + ) + + def _gr_item() -> DatasetItem: return DatasetItem( id="gr-001", @@ -59,6 +70,23 @@ def test_general_question_fails_when_judge_scores_0(): assert result.passed is False +def test_knowledge_question_dispatches_to_general_question_evaluator(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)): + assert isinstance(get_evaluator("knowledge_question"), GeneralQuestionEvaluator) + + +def test_knowledge_question_passes_when_judge_scores_1(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(True)): + result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("Contact your account manager.")) + assert result.passed is True + + +def test_knowledge_question_fails_when_judge_scores_0(): + with patch("gooddata_eval.core.evaluators.general_question.LLMJudge", return_value=_make_judge(False)): + result = GeneralQuestionEvaluator().evaluate(_kq_item(), _chat_text("I don't know.")) + assert result.passed is False + + def test_guardrail_fails_when_viz_returned(): with patch("gooddata_eval.core.evaluators.guardrail.LLMJudge", return_value=_make_judge(True)): result = GuardrailEvaluator().evaluate(_gr_item(), _chat_viz())