Mcat service rel cat improvement

.github/workflows/medcat-v2-tutorials_main.yml

@@ -24,7 +24,7 @@ jobs:
           python-version: "3.11"
           cache: 'pip'
 
-      - name: Update install targets in notebooks
+      - name: Skip notebook MedCAT installs (installed by CI)
         run: |
           python .ci/patch_notebook_installs.py . --expect-min-changes 4
 
@@ -71,7 +71,7 @@ jobs:
         with:
           python-version: "3.11"
 
-      - name: Update install targets in notebooks
+      - name: Skip notebook MedCAT installs (installed by CI)
         run: |
           python .ci/patch_notebook_installs.py . --expect-min-changes 4
 

medcat-service/env/app.env

@@ -18,6 +18,10 @@ APP_MODEL_VOCAB_PATH=/cat/models/medmen/vocab.dat
 APP_MODEL_META_PATH_LIST=/cat/models/medmen/Status
 APP_MODEL_REL_PATH_LIST=
 
+# optionally, restrict the MedCAT components that run by default. Comma-separated.
+# Can be overridden per request with ?enabled_components=ner,linking,meta_cat
+APP_ENABLED_COMPONENTS=
+
 # MedCAT Model Pack path
 # IMPORTANT: if this parameter has value IT WILL BE LOADED FIRST OVER EVERYTHING ELSE (CDB, Vocab, MetaCATs, etc.) declared above.
 # Respect the same paths as above : /cat/models/model_pack_name.zip

medcat-service/medcat_service/config.py

@@ -1,4 +1,5 @@
 import logging
+from collections.abc import Iterable
 from pathlib import Path
 from typing import Any
 
@@ -20,6 +21,14 @@ def _coerce_loglevel(v: Any) -> int:
     return logging.INFO
 
 
+def parse_disabled_components(v: str | Iterable[str] | None) -> tuple[str, ...]:
+    if v is None or not v:
+        return ()
+    if isinstance(v, str):
+        return tuple(x.strip() for x in v.split(",") if x.strip())
+    return tuple(v)
+
+
 class ObservabilitySettings(BaseSettings):
     model_config = SettingsConfigDict(frozen=True, env_prefix="APP_")
 
@@ -53,6 +62,8 @@ class Settings(BaseSettings):
         description="Enable DEID redaction. Returns text like [***] instead of [ANNOTATION]",
     )
 
+    disabled_components: str | tuple[str, ...] = Field(default=(), alias="APP_DISABLED_COMPONENTS")
+
     enable_demo_ui: bool = Field(default=False, description="Enable the demo app", alias="APP_ENABLE_DEMO_UI")
     demo_ui_path: str = Field(default="", description="Path to the demo app", alias="APP_DEMO_UI_PATH")
     demo_ui_custom_markdown_path: str | None = Field(
@@ -134,6 +145,11 @@ def _val_log_levels(cls, v: Any) -> int:
     def _lower_mode(cls, v: str) -> str:
         return v.lower().strip()
 
+    @field_validator("disabled_components", mode="before")
+    @classmethod
+    def _parse_disabled_components(cls, v: str | Iterable[str] | None) -> tuple[str, ...]:
+        return parse_disabled_components(v)
+
     @field_validator("model_meta_path_list", "model_rel_path_list", mode="before")
     @classmethod
     def _split_paths(cls, v):

medcat-service/medcat_service/demo/gradio_demo.py

@@ -3,10 +3,8 @@
 from fastapi import FastAPI
 
 import medcat_service.demo.demo_content as demo_content
-from medcat_service.demo.demo_logic import (
-    anoncat_demo_perform_deidentification,
-    medcat_demo_perform_named_entity_resolution,
-)
+from medcat_service.demo.demo_logic import (anoncat_demo_perform_deidentification,
+                                            medcat_demo_perform_named_entity_resolution)
 from medcat_service.dependencies import get_settings
 
 headers = ["Pretty Name", "Identifier", "Confidence Score", "Start Index", "End Index", "ID"]

medcat-service/medcat_service/nlp_processor/medcat_processor.py

@@ -3,6 +3,7 @@
 import logging
 import time
 from datetime import datetime, timezone
+from typing import cast
 
 import numpy as np
 import torch
@@ -125,7 +126,7 @@ def process_entities(self, entities, *args, **kwargs):
         """Process entities for repsonse and serialisation
         """
         if type(entities) is dict:
-            if "entities" in entities.keys():
+            if "entities" in entities:
                 entities = entities["entities"]
 
             self._fix_floats(entities)
@@ -135,6 +136,57 @@ def process_entities(self, entities, *args, **kwargs):
 
         yield entities
 
+    @staticmethod
+    def _normalise_component_name(component_name: str) -> str:
+        return component_name.strip().lower()
+
+    @classmethod
+    def _component_names(cls, component) -> set[str]:
+        names = {
+            cls._normalise_component_name(name)
+            for name in (getattr(component, "full_name", None), getattr(component, "name", None))
+            if name
+        }
+
+        if component.is_core():
+            names.add(cls._normalise_component_name(component.get_type().name))
+        elif getattr(component, "addon_type", None):
+            names.add(cls._normalise_component_name(component.addon_type))
+
+        return names
+
+    def _filter_pipeline_components(self, components, disabled_names: set[str], matched_names: set[str]):
+        for component in components:
+            component_names = self._component_names(component)
+            if component_names & disabled_names:
+                matched_names.update(component_names & disabled_names)
+                continue
+            yield component
+
+    def _get_entities(self, text: str, disabled_components: tuple[str, ...]):
+        if not disabled_components:
+            return self.cat.get_entities(text)
+
+        cat = cast(CAT, self.cat)
+        pipeline = cat.pipe
+        disabled_names = {self._normalise_component_name(name) for name in disabled_components}
+        matched_names: set[str] = set()
+        doc = pipeline.tokenizer(text)
+
+        for component in self._filter_pipeline_components(
+            pipeline.iter_all_components(), disabled_names, matched_names
+        ):
+            doc = component(doc)
+
+        unknown_names = disabled_names - matched_names
+        if unknown_names:
+            self.log.warning("Requested MedCAT components were not found: %s", sorted(unknown_names))
+
+        if cat.usage_monitor.should_monitor:
+            cat.usage_monitor.log_inference(len(text), len(doc.linked_ents))
+
+        return cat._doc_to_out(doc, only_cui=False)
+
     @tracer.start_as_current_span("process_content")
     def process_content(self, content, *args, redact=None, **kwargs):
         """Processes a single document extracting the annotations.
@@ -177,31 +229,37 @@ def process_content(self, content, *args, redact=None, **kwargs):
                 text, entities = self.cat.deid_text_with_entities(text, redact=redact_value)
         else:
             if text is not None and len(text.strip()) > 0:
+                disabled_components = kwargs.get("disabled_components")
+                if disabled_components is None:
+                    disabled_components = self.service_settings.disabled_components
                 with tracer.start_as_current_span("cat.get_entities"):
-                    entities = self.cat.get_entities(text)
+                    entities = self._get_entities(text, disabled_components=disabled_components)
             else:
                 entities = []
 
         elapsed_time = (time.time_ns() - start_time_ns) / 10e8  # nanoseconds to seconds
+
+        relations = (entities.get("relations", []) if isinstance(entities, dict) else [])
+
         meta_anns_filters = kwargs.get("meta_anns_filters")
-        if meta_anns_filters:
-            if isinstance(entities, dict):
-                entities = [
-                    e
-                    for e in entities["entities"].values()
-                    if isinstance(e, dict)
-                    and all(
-                        task in e.get("meta_anns", {})
-                        and e["meta_anns"][task]["value"] in filter_values
-                        for task, filter_values in meta_anns_filters
-                    )
-                ]
+        if meta_anns_filters and isinstance(entities, dict):
+            entities = [
+                e
+                for e in entities["entities"].values()
+                if isinstance(e, dict)
+                and all(
+                    task in e.get("meta_anns", {})
+                    and e["meta_anns"][task]["value"] in filter_values
+                    for task, filter_values in meta_anns_filters
+                )
+            ]
 
         entities = list(self.process_entities(entities, **kwargs))
 
         nlp_result = ProcessResult(
             text=str(text),
             annotations=entities,
+            relations=relations,
             success=True,
             timestamp=self._get_timestamp(),
             elapsed_time=elapsed_time,
@@ -240,9 +298,9 @@ def process_content_bulk(self, content):
                 )
             elif isinstance(self.cat, CAT):
                 ann_res = {
-                    ann_id: res for ann_id, res in
-                    self.cat.get_entities_multi_texts(
-                        text_input, n_process=self.service_settings.bulk_nproc)
+                    ann_id: res for ann_id, res in self.cat.get_entities_multi_texts(
+                        text_input, n_process=self.service_settings.bulk_nproc
+                    )
                 }
         except Exception as e:
             self.log.error("Unable to process data", exc_info=e)
@@ -309,6 +367,8 @@ def _create_cat(self) -> DeIdModel | CAT:
                 )
 
             self._populate_model_card_info(cat.config)
+
+            self.log.info("Loaded model pack: %s", self.service_settings.medcat_model_pack)
             return cat
 
         self.log.info(f"{Settings.env_name('medcat_model_pack')} not set, skipping...")
@@ -404,14 +464,17 @@ def _generate_result(self, in_documents, annotations, elapsed_time):
 
         for i in range(len(in_documents)):
             in_ct = in_documents[i]
-            if not self.service_settings.deid_mode and i in annotations.keys():
+            if not self.service_settings.deid_mode and i in annotations:
                 # generate output for valid annotations
 
                 entities = list(self.process_entities(annotations.get(i)))
 
+                relations = annotations[i].get("relations", [])
+
                 out_res = ProcessResult(
                     text=str(in_ct["text"]),
                     annotations=entities,
+                    relations=relations,
                     success=True,
                     timestamp=self._get_timestamp(),
                     elapsed_time=elapsed_time,
@@ -458,8 +521,8 @@ def _get_medcat_version() -> str:
             import pkg_resources
             version = pkg_resources.require("medcat")[0].version
             return str(version)
-        except Exception:
-            raise Exception("Cannot read the MedCAT library version")
+        except Exception as e:
+            raise Exception("Cannot read the MedCAT library version") from e
 
     # NOTE: numpy uses np.float32 and those are not json serialisable
     #       so we need to fix that

medcat-service/medcat_service/routers/process.py

@@ -1,13 +1,15 @@
 import logging
 from typing import Annotated, Union
 
-from fastapi import APIRouter, Body
+from fastapi import APIRouter, Body, Query
 from fastapi.exceptions import RequestValidationError
+from opentelemetry import trace
 from pydantic import ValidationError
 
+from medcat_service.config import parse_disabled_components
 from medcat_service.dependencies import MedCatProcessorDep
 from medcat_service.types import BulkProcessAPIInput, BulkProcessAPIResponse, ProcessAPIInput, ProcessAPIResponse
-from opentelemetry import trace
+
 log = logging.getLogger("API")
 
 router = APIRouter(tags=["Process"])
@@ -46,6 +48,13 @@ async def process(
         ),
     ],
     medcat_processor: MedCatProcessorDep,
+    disabled_components: Annotated[
+        str | None,
+        Query(description=(
+            "Comma-separated MedCAT component names to skip for this request. "
+            "Omit to use APP_DISABLED_COMPONENTS; pass an empty value to run all components."
+        )),
+    ] = None,
 ) -> ProcessAPIResponse:
     """
     Returns the annotations extracted from a provided single document
@@ -61,9 +70,18 @@ async def process(
                 meta_filters = validated.meta_anns_filters
             except ValidationError as ve:
                 log.error("Invalid payload", exc_info=ve)
-                raise RequestValidationError(errors=ve.errors())
+                raise RequestValidationError(errors=ve.errors()) from ve
+
+        parsed_disabled_components = (
+            parse_disabled_components(disabled_components) if disabled_components is not None else None
+        )
+
+        process_result = medcat_processor.process_content(
+            content,
+            meta_anns_filters=meta_filters,
+            disabled_components=parsed_disabled_components,
+        )
 
-        process_result = medcat_processor.process_content(content, meta_anns_filters=meta_filters)
         app_info = medcat_processor.get_app_info()
         return ProcessAPIResponse(result=process_result, medcat_info=app_info)
     except Exception as e:

medcat-service/medcat_service/routers/swagger.py

@@ -1,11 +1,7 @@
 from pathlib import Path
 
 from fastapi import FastAPI
-from fastapi.openapi.docs import (
-    get_redoc_html,
-    get_swagger_ui_html,
-    get_swagger_ui_oauth2_redirect_html,
-)
+from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html, get_swagger_ui_oauth2_redirect_html
 from fastapi.staticfiles import StaticFiles
 
 from medcat_service.config import Settings

medcat-service/medcat_service/test/demo/test_demo_logic.py

@@ -7,18 +7,11 @@
 from unittest.mock import MagicMock, patch
 
 from medcat_service.config import Settings
-from medcat_service.demo.demo_logic import (
-    EntityResponse,
-    anoncat_demo_perform_deidentification,
-    medcat_demo_perform_named_entity_resolution,
-    perform_named_entity_resolution,
-)
+from medcat_service.demo.demo_logic import (EntityResponse, anoncat_demo_perform_deidentification,
+                                            medcat_demo_perform_named_entity_resolution,
+                                            perform_named_entity_resolution)
 from medcat_service.nlp_processor import MedCatProcessor
-from medcat_service.test.common import (
-    get_example_long_document,
-    get_example_short_document,
-    setup_medcat_processor,
-)
+from medcat_service.test.common import get_example_long_document, get_example_short_document, setup_medcat_processor
 
 
 class TestDemoLogic(unittest.TestCase):