From 0671e454fa8ebe584377b00aba3daf1e0c092fc8 Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Mon, 21 Sep 2026 19:53:59 +0100 Subject: [PATCH 1/2] minor stats fixes to printing and NER only measurement --- medcat/medcat/stats/stats.py | 82 ++++++++++++++++++++------------ medcat/tests/stats/test_stats.py | 18 +++---- 2 files changed, 60 insertions(+), 40 deletions(-) diff --git a/medcat/medcat/stats/stats.py b/medcat/medcat/stats/stats.py index 1c27f0c3e..98cd16038 100644 --- a/medcat/medcat/stats/stats.py +++ b/medcat/medcat/stats/stats.py @@ -311,7 +311,7 @@ def _count_gold_annotations( continue state = mode_stats.stats if mode == MetricMode.NER: - key = "__NER__" + key = "DETECTED_ENTITY" state.cui_gold_counts[key] = ( state.cui_gold_counts.get(key, 0) + len(gold_anns) @@ -479,7 +479,7 @@ def _to_ner_views(self, gold_anns: list[GoldAnnotation], pred_anns: list[PredictedAnnotation] ) -> tuple[list[GoldAnnotation], list[PredictedAnnotation]]: - ner_cui = '__NER__' + ner_cui = 'DETECTED_ENTITY' eval_pred_anns: list[PredictedAnnotation] = [ {**pred, "cui": ner_cui} for pred in pred_anns @@ -980,8 +980,9 @@ def print_stats(self, epoch: int, mode_stats: ModeStats, n_samples: int = 10, - stream: TextIO | None = None) -> None: - """Finalise the report / metrics. + stream: TextIO | None = None, + legacy: bool = False) -> None: + """Print / report key metrics. This prints out the overall metrics and calculates per CUI metrics. @@ -993,17 +994,31 @@ def print_stats(self, to instead of stdout. """ if mode_stats.metrics is None: - raise ValueError( - "Metrics have not been computed yet. " - "Call compute_metrics() first." + raise ValueError( + "Metrics have not been computed yet. " + "Call compute_metrics() first." + ) + + if legacy: + print("Epoch: {}, Prec: {}, Rec: {}, F1: {}".format( + epoch, + mode_stats.metrics.overall.precision, + mode_stats.metrics.overall.recall, + mode_stats.metrics.overall.f1 + ), file=stream + ) + else: + print("Epoch: {}, \nPrec: {}, \nRec: {}, \nF1: {}, \nChar IOU: {}, " \ + "\nChar GIOU: {}, \nChar Cohen's K: {}\n".format( + epoch, + mode_stats.metrics.overall.precision, + mode_stats.metrics.overall.recall, + mode_stats.metrics.overall.f1, + mode_stats.metrics.overall.char_iou, + mode_stats.metrics.overall.char_giou, + mode_stats.metrics.overall.char_cohen_k + ), file=stream ) - print("Epoch: {}, Prec: {}, Rec: {}, F1: {}\n".format( - epoch, - mode_stats.metrics.overall.precision, - mode_stats.metrics.overall.recall, - mode_stats.metrics.overall.f1 - ), file=stream - ) # Sort fns & prec fps = {k: v for k, v in sorted(mode_stats.metrics.per_cui.items(), @@ -1013,7 +1028,7 @@ def print_stats(self, tps = {k: v for k, v in sorted(mode_stats.metrics.per_cui.items(), key=lambda item: item[1].tp, reverse=True)} - # Get top 5 + # Get top n pr_fps = [(self._get_pref_name(cui), cui, fps[cui]) for cui in list(fps.keys())[0:n_samples]] pr_fns = [(self._get_pref_name(cui), @@ -1021,7 +1036,7 @@ def print_stats(self, pr_tps = [(self._get_pref_name(cui), cui, tps[cui]) for cui in list(tps.keys())[0:n_samples]] - print("\n\nFalse Positives\n", file=stream) + print("\nFalse Positives\n", file=stream) for one in pr_fps: print("{:70} - {:20} - {:10}".format( str(one[0])[0:69], @@ -1029,7 +1044,7 @@ def print_stats(self, one[2].fp), file=stream ) - print("\n\nFalse Negatives\n", file=stream) + print("\nFalse Negatives\n", file=stream) for one in pr_fns: print("{:70} - {:20} - {:10}".format( str(one[0])[0:69], @@ -1037,7 +1052,7 @@ def print_stats(self, one[2].fn), file=stream ) - print("\n\nTrue Positives\n", file=stream) + print("\nTrue Positives\n", file=stream) for one in pr_tps: print("{:70} - {:20} - {:10}".format( str(one[0])[0:69], @@ -1074,7 +1089,7 @@ def get_stats_calculator(cat: CAT, linking_performance: bool = False, extra_cui_filter: Optional[set[str]] = None, do_print: bool = True,) -> StatsCalculator: - """Return a stats calculator for the given data and parameters. + """Return the stats calculator. This doesn't just return the per project stats for the full pipeline, but all stats for each project and the aggregate, for all modes @@ -1098,12 +1113,12 @@ def get_stats_calculator(cat: CAT, StatsCalculator: An instance of StatsCalculator with computed statistics. """ calculator = StatsCalculator( - filters=cat.config.components.linking.filters, - cui2info=cat.cdb.cui2info, - num_projects=len(data['projects']), - ner_performance=ner_performance, - linking_performance=linking_performance - ) + filters=cat.config.components.linking.filters, + cui2info=cat.cdb.cui2info, + num_projects=len(data['projects']), + ner_performance=ner_performance, + linking_performance=linking_performance + ) # Always compute full pipeline metrics. # If ner is of interest then also compute NER metrics from the same pass. calculator.process_export( @@ -1126,12 +1141,11 @@ def get_stats_calculator(cat: CAT, ) calculator.compute_all_metrics(ner_performance, linking_performance) - if do_print: - to_print = calculator.stats.all_projects.get_mode(MetricMode.FULL) - if to_print is None: + full_stats = calculator.stats.all_projects.get_mode(MetricMode.FULL) + if full_stats is None: raise ValueError("No statistics available for the full pipeline mode.") - calculator.print_stats(epoch, to_print) + calculator.print_stats(epoch, full_stats) return calculator def get_stats(cat: CAT, @@ -1173,10 +1187,16 @@ def get_stats(cat: CAT, calculator = get_stats_calculator( cat=cat, data=data, - epoch=epoch, use_project_filters=use_project_filters, use_overlaps=use_overlaps, - extra_cui_filter=extra_cui_filter + ner_performance=False, + linking_performance=False, + extra_cui_filter=extra_cui_filter, + do_print=False ) full_stats = calculator.stats.all_projects.full_pipeline + if do_print: + if full_stats is None: + raise ValueError("No statistics available for the full pipeline mode.") + calculator.print_stats(epoch, full_stats, legacy=True) return calculator.legacy_stats(full_stats) \ No newline at end of file diff --git a/medcat/tests/stats/test_stats.py b/medcat/tests/stats/test_stats.py index c49f25ccc..af5e88b5d 100644 --- a/medcat/tests/stats/test_stats.py +++ b/medcat/tests/stats/test_stats.py @@ -265,7 +265,7 @@ def test_basic_counts(self) -> None: ner_stats = self.result.stats.all_projects.get_mode(MetricMode.NER).stats # Raw counts of the NER only mode - self.assertEqual(ner_stats.cui_gold_counts["__NER__"], 4) + self.assertEqual(ner_stats.cui_gold_counts["DETECTED_ENTITY"], 4) def test_raw_counts_full_pipe(self) -> None: stats = self.result.stats.all_projects.get_mode(MetricMode.FULL).stats @@ -326,16 +326,16 @@ def test_per_cui_precision_recall_f1(self) -> None: self.assertAlmostEqual(full_pipe[cui].f1, 0.0) ner_pipe = self.result.stats.all_projects.get_mode(MetricMode.NER).metrics.per_cui - self.assertAlmostEqual(ner_pipe["__NER__"].precision, 1.0) - self.assertAlmostEqual(ner_pipe["__NER__"].recall, 0.75) - self.assertAlmostEqual(ner_pipe["__NER__"].f1, 0.85, places=1) + self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].precision, 1.0) + self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].recall, 0.75) + self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].f1, 0.85, places=1) def test_cuis_exist(self) -> None: cui_metrics = self.result.stats.all_projects.get_mode(MetricMode.FULL).metrics.per_cui ner_cui_metrics = self.result.stats.all_projects.get_mode(MetricMode.NER).metrics.per_cui linker_cui_metrics = self.linker_result.stats.all_projects.get_mode(MetricMode.LINKING).metrics.per_cui - self.assertIn("__NER__", ner_cui_metrics) - self.assertNotIn("__NER__", cui_metrics) + self.assertIn("DETECTED_ENTITY", ner_cui_metrics) + self.assertNotIn("DETECTED_ENTITY", cui_metrics) for cui in ["195967001", "387458008", "25609006", "116154003", "387517004"]: self.assertNotIn(cui, ner_cui_metrics) self.assertIn(cui, cui_metrics) @@ -378,9 +378,9 @@ def test_per_cui_character_statistics(self) -> None: # same as previous! intersection = len("asthma") + len("aspirin") + len("patient") union = len("asthma") + len("aspirin") + len("patient") + len("paracetamol") - self.assertAlmostEqual(ner_metrics["__NER__"].char_iou, intersection/union) - self.assertAlmostEqual(ner_metrics["__NER__"].char_giou, intersection/union) - self.assertAlmostEqual(ner_metrics["__NER__"].char_cohen_k, 0.63, places=2) + self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_iou, intersection/union) + self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_giou, intersection/union) + self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_cohen_k, 0.63, places=2) linker_metrics = self.linker_result.stats.all_projects.get_mode(MetricMode.LINKING).metrics.per_cui self.assertAlmostEqual(linker_metrics["195967001"].char_iou, 1.0) From 14dec08c438215aef0965bc80bf05648e57bf72e Mon Sep 17 00:00:00 2001 From: Adam Sutton Date: Mon, 21 Sep 2026 20:00:06 +0100 Subject: [PATCH 2/2] over indented...... --- medcat/medcat/stats/stats.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/medcat/medcat/stats/stats.py b/medcat/medcat/stats/stats.py index 98cd16038..deb0e9796 100644 --- a/medcat/medcat/stats/stats.py +++ b/medcat/medcat/stats/stats.py @@ -994,10 +994,10 @@ def print_stats(self, to instead of stdout. """ if mode_stats.metrics is None: - raise ValueError( - "Metrics have not been computed yet. " - "Call compute_metrics() first." - ) + raise ValueError( + "Metrics have not been computed yet. " + "Call compute_metrics() first." + ) if legacy: print("Epoch: {}, Prec: {}, Rec: {}, F1: {}".format(