Skip to content

minor stats fixes to printing and NER only measurement #620

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mart-r merged 2 commits into from
Sep 29, 2026
Merged

minor stats fixes to printing and NER only measurement #620

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0671e45
minor stats fixes to printing and NER only measurement
adam-sutton-1992 Sep 21, 2026
14dec08
over indented......
adam-sutton-1992 Sep 21, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 48 additions & 28 deletions medcat/medcat/stats/stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,7 @@ def _count_gold_annotations(
continue
state = mode_stats.stats
if mode == MetricMode.NER:
key = "__NER__"
key = "DETECTED_ENTITY"
state.cui_gold_counts[key] = (
state.cui_gold_counts.get(key, 0)
+ len(gold_anns)
Expand Down Expand Up @@ -479,7 +479,7 @@ def _to_ner_views(self,
gold_anns: list[GoldAnnotation],
pred_anns: list[PredictedAnnotation]
) -> tuple[list[GoldAnnotation], list[PredictedAnnotation]]:
ner_cui = '__NER__'
ner_cui = 'DETECTED_ENTITY'
eval_pred_anns: list[PredictedAnnotation] = [
{**pred, "cui": ner_cui}
for pred in pred_anns
Expand Down Expand Up @@ -980,8 +980,9 @@ def print_stats(self,
epoch: int,
mode_stats: ModeStats,
n_samples: int = 10,
stream: TextIO | None = None) -> None:
"""Finalise the report / metrics.
stream: TextIO | None = None,
legacy: bool = False) -> None:
"""Print / report key metrics.

This prints out the overall metrics and calculates per CUI metrics.

Expand All @@ -997,13 +998,27 @@ def print_stats(self,
"Metrics have not been computed yet. "
"Call compute_metrics() first."
)
print("Epoch: {}, Prec: {}, Rec: {}, F1: {}\n".format(
epoch,
mode_stats.metrics.overall.precision,
mode_stats.metrics.overall.recall,
mode_stats.metrics.overall.f1
), file=stream
)

if legacy:
print("Epoch: {}, Prec: {}, Rec: {}, F1: {}".format(
epoch,
mode_stats.metrics.overall.precision,
mode_stats.metrics.overall.recall,
mode_stats.metrics.overall.f1
), file=stream
)
else:
print("Epoch: {}, \nPrec: {}, \nRec: {}, \nF1: {}, \nChar IOU: {}, " \
"\nChar GIOU: {}, \nChar Cohen's K: {}\n".format(
epoch,
mode_stats.metrics.overall.precision,
mode_stats.metrics.overall.recall,
mode_stats.metrics.overall.f1,
mode_stats.metrics.overall.char_iou,
mode_stats.metrics.overall.char_giou,
mode_stats.metrics.overall.char_cohen_k
), file=stream
)

# Sort fns & prec
fps = {k: v for k, v in sorted(mode_stats.metrics.per_cui.items(),
Expand All @@ -1013,31 +1028,31 @@ def print_stats(self,
tps = {k: v for k, v in sorted(mode_stats.metrics.per_cui.items(),
key=lambda item: item[1].tp, reverse=True)}

# Get top 5
# Get top n
pr_fps = [(self._get_pref_name(cui),
cui, fps[cui]) for cui in list(fps.keys())[0:n_samples]]
pr_fns = [(self._get_pref_name(cui),
cui, fns[cui]) for cui in list(fns.keys())[0:n_samples]]
pr_tps = [(self._get_pref_name(cui),
cui, tps[cui]) for cui in list(tps.keys())[0:n_samples]]

print("\n\nFalse Positives\n", file=stream)
print("\nFalse Positives\n", file=stream)
for one in pr_fps:
print("{:70} - {:20} - {:10}".format(
str(one[0])[0:69],
str(one[1])[0:19],
one[2].fp),
file=stream
)
print("\n\nFalse Negatives\n", file=stream)
print("\nFalse Negatives\n", file=stream)
for one in pr_fns:
print("{:70} - {:20} - {:10}".format(
str(one[0])[0:69],
str(one[1])[0:19],
one[2].fn),
file=stream
)
print("\n\nTrue Positives\n", file=stream)
print("\nTrue Positives\n", file=stream)
for one in pr_tps:
print("{:70} - {:20} - {:10}".format(
str(one[0])[0:69],
Expand Down Expand Up @@ -1074,7 +1089,7 @@ def get_stats_calculator(cat: CAT,
linking_performance: bool = False,
extra_cui_filter: Optional[set[str]] = None,
do_print: bool = True,) -> StatsCalculator:
"""Return a stats calculator for the given data and parameters.
"""Return the stats calculator.

This doesn't just return the per project stats for the full pipeline,
but all stats for each project and the aggregate, for all modes
Expand All @@ -1098,12 +1113,12 @@ def get_stats_calculator(cat: CAT,
StatsCalculator: An instance of StatsCalculator with computed statistics.
"""
calculator = StatsCalculator(
filters=cat.config.components.linking.filters,
cui2info=cat.cdb.cui2info,
num_projects=len(data['projects']),
ner_performance=ner_performance,
linking_performance=linking_performance
)
filters=cat.config.components.linking.filters,
cui2info=cat.cdb.cui2info,
num_projects=len(data['projects']),
ner_performance=ner_performance,
linking_performance=linking_performance
)
# Always compute full pipeline metrics.
# If ner is of interest then also compute NER metrics from the same pass.
calculator.process_export(
Expand All @@ -1126,12 +1141,11 @@ def get_stats_calculator(cat: CAT,
)

calculator.compute_all_metrics(ner_performance, linking_performance)

if do_print:
to_print = calculator.stats.all_projects.get_mode(MetricMode.FULL)
if to_print is None:
full_stats = calculator.stats.all_projects.get_mode(MetricMode.FULL)
if full_stats is None:
raise ValueError("No statistics available for the full pipeline mode.")
calculator.print_stats(epoch, to_print)
calculator.print_stats(epoch, full_stats)
return calculator

def get_stats(cat: CAT,
Expand Down Expand Up @@ -1173,10 +1187,16 @@ def get_stats(cat: CAT,
calculator = get_stats_calculator(
cat=cat,
data=data,
epoch=epoch,
use_project_filters=use_project_filters,
use_overlaps=use_overlaps,
extra_cui_filter=extra_cui_filter
ner_performance=False,
linking_performance=False,
extra_cui_filter=extra_cui_filter,
do_print=False
)
full_stats = calculator.stats.all_projects.full_pipeline
if do_print:
if full_stats is None:
raise ValueError("No statistics available for the full pipeline mode.")
calculator.print_stats(epoch, full_stats, legacy=True)
return calculator.legacy_stats(full_stats)
18 changes: 9 additions & 9 deletions medcat/tests/stats/test_stats.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ def test_basic_counts(self) -> None:

ner_stats = self.result.stats.all_projects.get_mode(MetricMode.NER).stats
# Raw counts of the NER only mode
self.assertEqual(ner_stats.cui_gold_counts["__NER__"], 4)
self.assertEqual(ner_stats.cui_gold_counts["DETECTED_ENTITY"], 4)

def test_raw_counts_full_pipe(self) -> None:
stats = self.result.stats.all_projects.get_mode(MetricMode.FULL).stats
Expand Down Expand Up @@ -326,16 +326,16 @@ def test_per_cui_precision_recall_f1(self) -> None:
self.assertAlmostEqual(full_pipe[cui].f1, 0.0)

ner_pipe = self.result.stats.all_projects.get_mode(MetricMode.NER).metrics.per_cui
self.assertAlmostEqual(ner_pipe["__NER__"].precision, 1.0)
self.assertAlmostEqual(ner_pipe["__NER__"].recall, 0.75)
self.assertAlmostEqual(ner_pipe["__NER__"].f1, 0.85, places=1)
self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].precision, 1.0)
self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].recall, 0.75)
self.assertAlmostEqual(ner_pipe["DETECTED_ENTITY"].f1, 0.85, places=1)

def test_cuis_exist(self) -> None:
cui_metrics = self.result.stats.all_projects.get_mode(MetricMode.FULL).metrics.per_cui
ner_cui_metrics = self.result.stats.all_projects.get_mode(MetricMode.NER).metrics.per_cui
linker_cui_metrics = self.linker_result.stats.all_projects.get_mode(MetricMode.LINKING).metrics.per_cui
self.assertIn("__NER__", ner_cui_metrics)
self.assertNotIn("__NER__", cui_metrics)
self.assertIn("DETECTED_ENTITY", ner_cui_metrics)
self.assertNotIn("DETECTED_ENTITY", cui_metrics)
for cui in ["195967001", "387458008", "25609006", "116154003", "387517004"]:
self.assertNotIn(cui, ner_cui_metrics)
self.assertIn(cui, cui_metrics)
Expand Down Expand Up @@ -378,9 +378,9 @@ def test_per_cui_character_statistics(self) -> None:
# same as previous!
intersection = len("asthma") + len("aspirin") + len("patient")
union = len("asthma") + len("aspirin") + len("patient") + len("paracetamol")
self.assertAlmostEqual(ner_metrics["__NER__"].char_iou, intersection/union)
self.assertAlmostEqual(ner_metrics["__NER__"].char_giou, intersection/union)
self.assertAlmostEqual(ner_metrics["__NER__"].char_cohen_k, 0.63, places=2)
self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_iou, intersection/union)
self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_giou, intersection/union)
self.assertAlmostEqual(ner_metrics["DETECTED_ENTITY"].char_cohen_k, 0.63, places=2)

linker_metrics = self.linker_result.stats.all_projects.get_mode(MetricMode.LINKING).metrics.per_cui
self.assertAlmostEqual(linker_metrics["195967001"].char_iou, 1.0)
Expand Down
Loading