Merge pull request #116 from CogStack/add-concept-search-fix

CU-862j6fpd2: Fix edge case of adding new concept without typeID

Author: tomolopolis

webapp/api/api/migrations/0068_auto_20230203_1715.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.28 on 2023-02-03 17:15
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('api', '0067_delete_concept'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='projectannotateentities',
+            name='restrict_concept_lookup',
+            field=models.BooleanField(default=False, help_text='Users can only search for concept terms from the list configured for the project, i.e. either from the cuis or cuis_file lists. Checking this when bothcuis and cuis_file are empty does nothing. If "add new entities" is available & added, and cuis or cuis_fileis non-empty the new CUI will be added.'),
+        ),
+    ]

webapp/api/api/models.py

@@ -248,9 +248,12 @@ class ProjectAnnotateEntities(Project):
     add_new_entities = models.BooleanField(default=False,
                                            help_text='Allow the creation of new terms to be added to the CDB')
     restrict_concept_lookup = models.BooleanField(default=False,
-                                                  help_text='Users can only search for concept terms from the list '
-                                                            'configured for the project, i.e. either from the cuis '
-                                                            'or cuis_file lists.')
+                                                  help_text='Users can only search for concept terms from the '
+                                                            'list configured for the project, i.e. either from '
+                                                            'the cuis or cuis_file lists. Checking this when both'
+                                                            'cuis and cuis_file are empty does nothing. If "add new '
+                                                            'entities" is available & added, and cuis or cuis_file'
+                                                            'is non-empty the new CUI will be added.')
     terminate_available = models.BooleanField(default=True,
                                               help_text='Enable the option to terminate concepts.')
     irrelevant_available = models.BooleanField(default=False,

webapp/api/api/solr_utils.py

@@ -79,7 +79,7 @@ def search_collection(cdbs: List[int], raw_query: str):
                         parsed_doc = {
                             'cui': str(d['cui'][0]),
                             'pretty_name': d['pretty_name'][0],
-                            'type_ids': d['type_ids'],
+                            'type_ids': d.get('type_ids', []),
                             'synonyms': d['synonyms']
                         }
                         if d.get('icd10'):

webapp/api/api/utils.py

@@ -1,22 +1,20 @@
 import json
-import os
 import logging
+import os
 from typing import Union, Dict, List, Type
 
 import pkg_resources
+from django.contrib.auth.models import User
 from django.db.models.signals import post_save
 from django.dispatch import receiver
-
-from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, ProjectCuiCounter, \
-    ConceptDB
-
-from medcat.cdb import CDB
-from medcat.vocab import Vocab
 from medcat.cat import CAT
+from medcat.cdb import CDB
 from medcat.utils.filters import check_filters
 from medcat.utils.helpers import tkns_from_doc
+from medcat.vocab import Vocab
 
-from .solr_utils import ensure_concept_searchable
+from .models import Entity, AnnotatedEntity, ICDCode, OPCSCode, ProjectAnnotateEntities, ProjectCuiCounter, \
+    ConceptDB
 
 log = logging.getLogger('trainer')
 
@@ -148,7 +146,8 @@ def _remove_overlap(project, document, start, end):
             ann.delete()
 
 
-def create_annotation(source_val, selection_occurrence_index, cui, user, project, document, cat, icd_code=None,
+def create_annotation(source_val: str, selection_occurrence_index: int, cui: str, user: User,
+                      project: ProjectAnnotateEntities, document, cat: CAT, icd_code=None,
                       opcs_code=None):
     text = document.text
     id = None
@@ -198,9 +197,6 @@ def create_annotation(source_val, selection_occurrence_index, cui, user, project
         ann_ent.save()
         id = ann_ent.id
 
-    # Add concept detail to SOLR search service
-    ensure_concept_searchable(cui, cat.cdb, project.concept_db)
-
     # upload icd / opcs codes if available
     # also expects icd / opcs addl info dicts to include:
     # {code: <the code>: name: <human readable desc>}

webapp/api/api/views.py

@@ -9,13 +9,13 @@
 from rest_framework import viewsets
 from rest_framework.decorators import api_view
 from rest_framework.response import Response
-from rest_framework.viewsets import ViewSet
+
 
 from .admin import download_projects_with_text, download_projects_without_text, \
     import_concepts_from_cdb, upload_projects_export
 from .permissions import *
 from .serializers import *
-from .solr_utils import collections_available, search_collection
+from .solr_utils import collections_available, search_collection, ensure_concept_searchable
 from .utils import get_cached_medcat, \
     clear_cached_medcat
 from .utils import get_medcat, add_annotations, remove_annotations, train_medcat, create_annotation
@@ -356,6 +356,14 @@ def add_concept(request):
                            document=document,
                            cat=cat)
 
+    # ensure new concept detail is available in SOLR search service
+    ensure_concept_searchable(cui, cat.cdb, project.concept_db)
+
+    # add to project cuis if required.
+    if (project.cuis or project.cuis_file) and project.restrict_concept_lookup:
+        project.cuis = ','.join(project.cuis.split(',') + [cui])
+        project.save()
+
     return Response({'message': 'Concept and Annotation added successfully', 'id': id})