From bb17300f3cf227fb5c9f8095e5ebc596825a7416 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Tue, 20 Jan 2026 12:27:11 +0000 Subject: [PATCH 01/22] feat(medcat-trainer): first pass at project admin initial page not using the django admin... --- medcat-trainer/webapp/api/api/permissions.py | 25 + medcat-trainer/webapp/api/api/views.py | 139 +++- medcat-trainer/webapp/api/core/urls.py | 4 + medcat-trainer/webapp/frontend/src/App.vue | 1 + .../webapp/frontend/src/router/index.ts | 6 + .../webapp/frontend/src/views/Home.vue | 26 +- .../frontend/src/views/ProjectAdmin.vue | 673 ++++++++++++++++++ 7 files changed, 868 insertions(+), 6 deletions(-) create mode 100644 medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue diff --git a/medcat-trainer/webapp/api/api/permissions.py b/medcat-trainer/webapp/api/api/permissions.py index e8995c01f..2a2f04ff3 100644 --- a/medcat-trainer/webapp/api/api/permissions.py +++ b/medcat-trainer/webapp/api/api/permissions.py @@ -1,4 +1,7 @@ from rest_framework import permissions +from rest_framework.exceptions import PermissionDenied +from .models import ProjectAnnotateEntities, ProjectGroup + class IsReadOnly(permissions.BasePermission): """ @@ -9,3 +12,25 @@ def has_permission(self, request, view): # Read permissions are allowed to any request, # so we'll always allow GET, HEAD or OPTIONS requests. return request.method in permissions.SAFE_METHODS + + +def is_project_admin(user, project): + """ + Check if a user is an admin of a project. + A user is a project admin if: + 1. They are a member of the project, OR + 2. They are an administrator of the project's group (if the project has a group) + 3. They are a superuser/staff + """ + if user.is_superuser or user.is_staff: + return True + + # Check if user is a member of the project + if project.members.filter(id=user.id).exists(): + return True + + # Check if user is an administrator of the project's group + if project.group and project.group.administrators.filter(id=user.id).exists(): + return True + + return False diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py index ea8b75d69..08955334c 100644 --- a/medcat-trainer/webapp/api/api/views.py +++ b/medcat-trainer/webapp/api/api/views.py @@ -15,7 +15,7 @@ from django_filters import rest_framework as drf from rest_framework import viewsets -from rest_framework.decorators import api_view +from rest_framework.decorators import api_view, permission_classes from rest_framework.response import Response from medcat.components.ner.trf.deid import DeIdModel from medcat.utils.cdb_utils import ch2pt_from_pt2ch, get_all_ch, snomed_ct_concept_path @@ -1003,3 +1003,140 @@ def project_progress(request): out[p] = {'validated_count': val_docs, 'dataset_count': ds_doc_count} return Response(out) + + +@api_view(http_method_names=['GET']) +@permission_classes([permissions.IsAuthenticated]) +def project_admin_projects(request): + """ + Get all projects where the user is a project admin. + """ + user = request.user + projects = ProjectAnnotateEntities.objects.filter(members=user.id) + + # Also include projects where user is admin of the project's group + group_admin_projects = ProjectAnnotateEntities.objects.filter( + group__administrators=user.id + ) + projects = (projects | group_admin_projects).distinct() + + serializer = ProjectAnnotateEntitiesSerializer(projects, many=True) + return Response(serializer.data) + + +@api_view(http_method_names=['GET', 'PUT', 'DELETE']) +@permission_classes([permissions.IsAuthenticated]) +def project_admin_detail(request, project_id): + """ + Get, update, or delete a project (only if user is project admin). + """ + try: + project = ProjectAnnotateEntities.objects.get(id=project_id) + except ProjectAnnotateEntities.DoesNotExist: + return Response({'error': 'Project not found'}, status=404) + + # Check if user is project admin + from .permissions import is_project_admin + if not is_project_admin(request.user, project): + return Response({'error': 'You do not have permission to access this project'}, status=403) + + if request.method == 'GET': + serializer = ProjectAnnotateEntitiesSerializer(project) + return Response(serializer.data) + + elif request.method == 'PUT': + # Handle both JSON and FormData + data = request.data.copy() if hasattr(request.data, 'copy') else dict(request.data) + + # Convert many-to-many fields from lists to proper format + if 'cdb_search_filter' in data and isinstance(data['cdb_search_filter'], list): + # Already a list, keep it + pass + elif 'cdb_search_filter' in request.data: + # FormData sends as multiple values with same key + data['cdb_search_filter'] = request.data.getlist('cdb_search_filter') + + if 'members' in request.data: + if isinstance(request.data.get('members'), list): + data['members'] = request.data['members'] + else: + data['members'] = request.data.getlist('members') + + serializer = ProjectAnnotateEntitiesSerializer(project, data=data, partial=True) + if serializer.is_valid(): + project = serializer.save() + # Handle many-to-many fields manually if needed + if 'cdb_search_filter' in data: + project.cdb_search_filter.set(data['cdb_search_filter']) + if 'members' in data: + project.members.set(data['members']) + return Response(ProjectAnnotateEntitiesSerializer(project).data) + return Response(serializer.errors, status=400) + + elif request.method == 'DELETE': + project.delete() + return Response({'message': 'Project deleted successfully'}, status=200) + + +@api_view(http_method_names=['POST']) +@permission_classes([permissions.IsAuthenticated]) +def project_admin_create(request): + """ + Create a new project (user must be authenticated). + """ + # Handle both JSON and FormData + data = request.data.copy() if hasattr(request.data, 'copy') else dict(request.data) + + # Convert many-to-many fields from FormData format + if 'cdb_search_filter' in request.data: + if isinstance(request.data.get('cdb_search_filter'), list): + data['cdb_search_filter'] = request.data['cdb_search_filter'] + else: + data['cdb_search_filter'] = request.data.getlist('cdb_search_filter') + + if 'members' in request.data: + if isinstance(request.data.get('members'), list): + data['members'] = request.data['members'] + else: + data['members'] = request.data.getlist('members') + + serializer = ProjectAnnotateEntitiesSerializer(data=data) + if serializer.is_valid(): + project = serializer.save() + # Handle many-to-many fields manually + if 'cdb_search_filter' in data: + project.cdb_search_filter.set(data['cdb_search_filter']) + if 'members' in data: + project.members.set(data['members']) + # Add the creator as a member if not already included + if request.user not in project.members.all(): + project.members.add(request.user) + return Response(ProjectAnnotateEntitiesSerializer(project).data, status=201) + return Response(serializer.errors, status=400) + + +@api_view(http_method_names=['POST']) +@permission_classes([permissions.IsAuthenticated]) +def project_admin_reset(request, project_id): + """ + Reset a project (clear all annotations) - only if user is project admin. + This is equivalent to the reset_project admin action. + """ + try: + project = ProjectAnnotateEntities.objects.get(id=project_id) + except ProjectAnnotateEntities.DoesNotExist: + return Response({'error': 'Project not found'}, status=404) + + # Check if user is project admin + from .permissions import is_project_admin + if not is_project_admin(request.user, project): + return Response({'error': 'You do not have permission to reset this project'}, status=403) + + # Remove all annotations and cascade to meta anns + AnnotatedEntity.objects.filter(project=project).delete() + + # Clear validated_documents and prepared_documents + project.validated_documents.clear() + project.prepared_documents.clear() + + return Response({'message': 'Project reset successfully'}, status=200) diff --git a/medcat-trainer/webapp/api/core/urls.py b/medcat-trainer/webapp/api/core/urls.py index e4165a52f..6614244d4 100644 --- a/medcat-trainer/webapp/api/core/urls.py +++ b/medcat-trainer/webapp/api/core/urls.py @@ -60,6 +60,10 @@ path('api/generate-concept-filter-json/', api.views.generate_concept_filter_flat_json), path('api/generate-concept-filter/', api.views.generate_concept_filter), path('api/cuis-to-concepts/', api.views.cuis_to_concepts), + path('api/project-admin/projects/', api.views.project_admin_projects), + path('api/project-admin/projects//', api.views.project_admin_detail), + path('api/project-admin/projects//reset/', api.views.project_admin_reset), + path('api/project-admin/projects/create/', api.views.project_admin_create), path('reset_password/', api.views.ResetPasswordView.as_view(), name='reset_password'), path('reset_password_sent/', pw_views.PasswordResetDoneView.as_view(), name='password_reset_done'), path('reset//', pw_views.PasswordResetConfirmView.as_view(), name='password_reset_confirm'), diff --git a/medcat-trainer/webapp/frontend/src/App.vue b/medcat-trainer/webapp/frontend/src/App.vue index f58626418..f6a9bc177 100644 --- a/medcat-trainer/webapp/frontend/src/App.vue +++ b/medcat-trainer/webapp/frontend/src/App.vue @@ -17,6 +17,7 @@
Projects + Project Admin Metrics Concepts Try Model diff --git a/medcat-trainer/webapp/frontend/src/router/index.ts b/medcat-trainer/webapp/frontend/src/router/index.ts index 15eb3f23b..69e88b3f3 100644 --- a/medcat-trainer/webapp/frontend/src/router/index.ts +++ b/medcat-trainer/webapp/frontend/src/router/index.ts @@ -5,6 +5,7 @@ import Demo from '../views/Demo.vue' import Metrics from '../views/Metrics.vue' import MetricsHome from '../views/MetricsHome.vue' import ConceptDatabase from '../views/ConceptDatabase.vue' +import ProjectAdmin from '../views/ProjectAdmin.vue' const router = createRouter({ @@ -38,6 +39,11 @@ const router = createRouter({ name: 'model-explore', component: ConceptDatabase }, + { + path: '/project-admin', + name: 'project-admin', + component: ProjectAdmin + }, { path: '/:pathMatch(.*)', name: 'home', diff --git a/medcat-trainer/webapp/frontend/src/views/Home.vue b/medcat-trainer/webapp/frontend/src/views/Home.vue index e187acacd..d707f51bb 100644 --- a/medcat-trainer/webapp/frontend/src/views/Home.vue +++ b/medcat-trainer/webapp/frontend/src/views/Home.vue @@ -2,11 +2,19 @@
-
- +
+
+ +
+
+ + + Manage Projects + +
+
+
+

Project Administration

+

Manage your annotation projects

+
+ +
+ + Loading projects... +
+ +
+ +
+
+

Your Projects

+ +
+ + + + + + + +
+

You don't have any projects yet. Create one to get started!

+
+
+ + + + + + + + + + + + + + + + + + +
+
+ + + + + From e608d21b72d40a21d027adde0e513de0e9f2182b Mon Sep 17 00:00:00 2001 From: Mart Ratas Date: Tue, 20 Jan 2026 11:40:03 +0200 Subject: [PATCH 02/22] bug(medcat-den):CU-869bu4tum Fix wrapper save (attempt 2) (#295) * CU-869bu4tum: Add test to make sure that saved model is NOT a wrapper * CU-869bu4tum: Move wrapper to a fully delegating one * CU-869bu4tum: Add a comment regarding setattr * CU-869bu4tum: Add a few more tests for wrapper (i.e getting attributes, properties, methods) * CU-869bu4tum: Fix injection tests * CU-869bu4tum: Add some comments to injection tests * CU-869bu4tum: Fix test time wrapping * CU-869bu4tum: Add a few more comprehensive tests * CU-869bu4tum: Add a few more comprehensive tests for property / method / attribtue getting * CU-869bu4tum: Fix trainer property for wrapper * CU-869bu4tum: Simplify trainer property --- medcat-den/src/medcat_den/wrappers.py | 19 ++++-- .../tests/injection/test_medcat_injection.py | 5 +- medcat-den/tests/test_remote_den_disallows.py | 3 +- medcat-den/tests/test_wrappers.py | 65 +++++++++++++++++-- 4 files changed, 75 insertions(+), 17 deletions(-) diff --git a/medcat-den/src/medcat_den/wrappers.py b/medcat-den/src/medcat_den/wrappers.py index a0de3f10c..cfb2d65f2 100644 --- a/medcat-den/src/medcat_den/wrappers.py +++ b/medcat-den/src/medcat_den/wrappers.py @@ -1,4 +1,4 @@ -from typing import Union, Optional, cast +from typing import Union, Optional from medcat.cat import CAT from medcat.utils.defaults import DEFAULT_PACK_NAME @@ -22,6 +22,14 @@ class CATWrapper(CAT): `force_save_local=True`. But that is generally not advised. """ + def __init__(self, cat: CAT) -> None: + self._delegate = cat + + def __getattr__(self, attr: str) -> None: + return getattr(self._delegate, attr) + + # NOTE: __setattr__ should never be used in normal opration + _model_info: ModelInfo _den_cnf: DenConfig @@ -69,14 +77,13 @@ def save_model_pack( "See `Den.finetune_model` for details or set the config " "option of `allow_push_fine_tuned` to True" ) - return CAT.save_model_pack( - self, + return self._delegate.save_model_pack( target_folder, pack_name, serialiser_type, make_archive, only_archive, add_hash_to_pack_name, change_description) @property def trainer(self) -> Trainer: - tr = super().trainer + tr = self._delegate.trainer return WrappedTrainer(self._den_cnf, tr) @classmethod @@ -115,9 +122,7 @@ def load_model_pack(cls, model_pack_path: str, """ _cat = super().load_model_pack( model_pack_path, config_dict, addon_config_dict) - cat = cast(CATWrapper, _cat) - if not isinstance(cat, CATWrapper): - cat.__class__ = CATWrapper + cat = cls(_cat) if model_info is None: raise CannotWrapModel("Model info must be provided") if den_cnf is None: diff --git a/medcat-den/tests/injection/test_medcat_injection.py b/medcat-den/tests/injection/test_medcat_injection.py index 5ff7a844c..8806f7a94 100644 --- a/medcat-den/tests/injection/test_medcat_injection.py +++ b/medcat-den/tests/injection/test_medcat_injection.py @@ -59,7 +59,10 @@ def den_with_model(tmp_path: str, cat: CAT) -> Den: def den_with_nonempty_model(den_with_model: Den): model = den_with_model.fetch_model( den_with_model.list_available_base_models()[0]) - model.vocab = Vocab() + # NOTE: normally, the CATWrapper setattr isn't used so it's not + # delegating to the original + # but once set, the gettattr will use the correct delegation + model._delegate.vocab = Vocab() model.vocab.add_word("acute", 4, np.array([1, 0, 0])) model.vocab.add_word("chronic", 3, np.array([-1, 0, 0])) model.vocab.add_word("does", 10, np.array([0, 1, 0])) diff --git a/medcat-den/tests/test_remote_den_disallows.py b/medcat-den/tests/test_remote_den_disallows.py index b484963fa..8eb8b320d 100644 --- a/medcat-den/tests/test_remote_den_disallows.py +++ b/medcat-den/tests/test_remote_den_disallows.py @@ -18,8 +18,7 @@ def get_wrapped_model_pack( in_model_pack: CAT, cnf: RemoteDenConfig) -> CATWrapper: # noqa # make it a wrapper - in_model_pack.__class__ = CATWrapper - model_pack = cast(CATWrapper, in_model_pack) + model_pack = CATWrapper(in_model_pack) # set required stuff, mostly the config model_pack._den_cnf = cnf # set model info diff --git a/medcat-den/tests/test_wrappers.py b/medcat-den/tests/test_wrappers.py index f8838dc12..2cdb3da1a 100644 --- a/medcat-den/tests/test_wrappers.py +++ b/medcat-den/tests/test_wrappers.py @@ -1,17 +1,68 @@ +import os + from medcat.cat import CAT +from medcat.storage.schema import load_schema from medcat_den import wrappers -from .test_file_system_den import def_model_pack +from .test_file_system_den import def_model_pack, _load_model, MODEL_PATH + +import pytest def wrap(cat: CAT) -> wrappers.CATWrapper: - # creates a new pipe! - return wrappers.CATWrapper(cat.cdb, cat.vocab, cat.config) + wrapped = wrappers.CATWrapper(cat) + wrapped._model_info = "ABC" + wrapped._den_cnf = 'BBC' + return wrapped -def test_wrapper_saves_as_CAT(tmpdir, def_model_pack): - cat = wrap(def_model_pack) - mpp = cat.save_model_pack(tmpdir, force_save_local=True) - loaded = CAT.load_model_pack(mpp) +@pytest.fixture(scope='module') +def saved_model_pack_path(tmpdir_factory) -> str: + tmpdir = tmpdir_factory.mktemp('model_path') + cat = wrap(_load_model(MODEL_PATH)) + assert isinstance(cat, wrappers.CATWrapper) + return cat.save_model_pack(tmpdir, force_save_local=True) + + +def test_wrapper_schema_CAT(saved_model_pack_path): + schema = os.path.join(saved_model_pack_path.removesuffix(".zip"), '.schema.json') + assert load_schema(schema)[0] == 'medcat.cat.CAT' + + +def test_wrapper_saves_as_CAT(saved_model_pack_path): + loaded = CAT.load_model_pack(saved_model_pack_path) assert isinstance(loaded, CAT) + assert not isinstance(loaded, wrappers.CATWrapper) + + +def test_wrapper_gets_attributes(def_model_pack: CAT): + cat = wrap(def_model_pack) + assert cat.cdb is def_model_pack.cdb + + +def test_wrapper_gets_properties(def_model_pack: CAT): + cat = wrap(def_model_pack) + assert cat.pipe is def_model_pack.pipe + + +def test_wrapper_gets_methods(def_model_pack: CAT): + cat = wrap(def_model_pack) + text = "Kidney disease causes autism and fever in diabetes patients" + assert cat.get_entities(text) == def_model_pack.get_entities(text) + + +def test_wrapper_gets_wrapper_defined_attribute(def_model_pack: CAT): + cat = wrap(def_model_pack) + assert cat._model_info + + +def test_wrapper_gets_wrapped_property(def_model_pack: CAT): + cat = wrap(def_model_pack) + assert isinstance(cat.trainer, wrappers.WrappedTrainer) + + +def test_wrapper_calls_wrapped_method(def_model_pack: CAT): + cat = wrap(def_model_pack) + with pytest.raises(wrappers.CannotSaveOnDiskException): + cat.save_model_pack("SOME-PATH-NOT_EXIST") From 00cee9a04ac39d44ba57c65e07c7548672a48d23 Mon Sep 17 00:00:00 2001 From: Vlad Dinu <62345326+vladd-bit@users.noreply.github.com> Date: Tue, 20 Jan 2026 10:42:09 +0000 Subject: [PATCH 03/22] =?UTF-8?q?MedCAT=20v1:=20fixed=20relation=20infer?= =?UTF-8?q?=20output=20(entities=20not=20outputting=20start=E2=80=A6=20(#2?= =?UTF-8?q?97)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * MedCAT v1: fixed relation infer output (entities not outputting start/end char positions. * Lints. --- v1/medcat/medcat/rel_cat.py | 13 +++++- .../utils/relation_extraction/rel_dataset.py | 45 ++++++++++--------- 2 files changed, 35 insertions(+), 23 deletions(-) diff --git a/v1/medcat/medcat/rel_cat.py b/v1/medcat/medcat/rel_cat.py index b8d72f663..021a16324 100644 --- a/v1/medcat/medcat/rel_cat.py +++ b/v1/medcat/medcat/rel_cat.py @@ -592,6 +592,8 @@ def pipe(self, stream: Iterable[Doc], *args, **kwargs) -> Iterator[Doc]: pred_rel_logits, dim=0).max(0) predicted_label_id = int(confidence[1].item()) + + doc._.relations.append({"relation": self.component.relcat_config.general.idx2labels[predicted_label_id], "label_id": predicted_label_id, "ent1_text": predict_rel_dataset.dataset["output_relations"][rel_idx][ @@ -599,8 +601,10 @@ def pipe(self, stream: Iterable[Doc], *args, **kwargs) -> Iterator[Doc]: "ent2_text": predict_rel_dataset.dataset["output_relations"][rel_idx][ 3], "confidence": float("{:.3f}".format(confidence[0])), - "start_ent_pos": "", - "end_ent_pos": "", + "start_ent1_char_pos": predict_rel_dataset.dataset["output_relations"][rel_idx][18], + "end_ent1_char_pos": predict_rel_dataset.dataset["output_relations"][rel_idx][19], + "start_ent2_char_pos": predict_rel_dataset.dataset["output_relations"][rel_idx][20], + "end_ent2_char_pos": predict_rel_dataset.dataset["output_relations"][rel_idx][21], "start_entity_id": predict_rel_dataset.dataset["output_relations"][rel_idx][8], "end_entity_id": @@ -638,6 +642,8 @@ def predict_text_with_anns(self, text: str, annotations: List[Dict]) -> Doc: Span.set_extension('id', default=0, force=True) Span.set_extension('cui', default=None, force=True) + Span.set_extension('start', default=None, force=True) + Span.set_extension('end', default=None, force=True) Doc.set_extension('ents', default=[], force=True) Doc.set_extension('relations', default=[], force=True) nlp = spacy.blank(self.component.relcat_config.general.language) @@ -647,10 +653,13 @@ def predict_text_with_anns(self, text: str, annotations: List[Dict]) -> Doc: tkn_idx = [] for ind, word in enumerate(doc): end_char = word.idx + len(word.text) + # identify token index by checking against ann char position if end_char <= ann['end'] and end_char > ann['start']: tkn_idx.append(ind) entity = Span(doc, min(tkn_idx), max(tkn_idx) + 1, label=ann["value"]) entity._.cui = ann["cui"] + entity._.start = ann["start"] + entity._.end = ann["end"] doc._.ents.append(entity) doc = self(doc) diff --git a/v1/medcat/medcat/utils/relation_extraction/rel_dataset.py b/v1/medcat/medcat/utils/relation_extraction/rel_dataset.py index c9b134f12..8a43be6c3 100644 --- a/v1/medcat/medcat/utils/relation_extraction/rel_dataset.py +++ b/v1/medcat/medcat/utils/relation_extraction/rel_dataset.py @@ -213,6 +213,9 @@ def _create_relation_validation(self, ent1_token: Union[str, Span] = tmp_doc_text[ent1_start_char_pos: ent1_end_char_pos] ent2_token: Union[str, Span] = tmp_doc_text[ent2_start_char_pos: ent2_end_char_pos] + annotation_token_text = self.tokenizer.hf_tokenizers.convert_ids_to_tokens( + self.config.general.annotation_schema_tag_ids) + if abs(ent2_start_char_pos - ent1_start_char_pos) <= self.config.general.window_size and \ ent1_token != ent2_token: @@ -240,20 +243,13 @@ def _create_relation_validation(self, if is_spacy_doc or is_mct_export: tmp_doc_text = text - _pre_e1 = tmp_doc_text[0: (ent1_start_char_pos)] - _e1_s2 = tmp_doc_text[ent1_end_char_pos: ent2_start_char_pos - 1] - _e2_end = tmp_doc_text[ent2_end_char_pos + 1: text_length] - ent2_token_end_pos = (ent2_token_end_pos + 2) - annotation_token_text = self.tokenizer.hf_tokenizers.convert_ids_to_tokens( - self.config.general.annotation_schema_tag_ids) + s1,e1,s2,e2 = annotation_token_text - tmp_doc_text = str(_pre_e1) + " " + \ - annotation_token_text[0] + " " + \ - str(ent1_token) + " " + \ - annotation_token_text[1] + " " + str(_e1_s2) + " " + \ - annotation_token_text[2] + " " + str(ent2_token) + " " + \ - annotation_token_text[3] + " " + str(_e2_end) + tmp_doc_text = tmp_doc_text[:ent2_end_char_pos] + e2 + tmp_doc_text[ent2_end_char_pos:] + tmp_doc_text = tmp_doc_text[:ent2_start_char_pos] + s2 + tmp_doc_text[ent2_start_char_pos:] + tmp_doc_text = tmp_doc_text[:ent1_end_char_pos] + e1 + tmp_doc_text[ent1_end_char_pos:] + tmp_doc_text = tmp_doc_text[:ent1_start_char_pos] + s1 + tmp_doc_text[ent1_start_char_pos:] ann_tag_token_len = len(annotation_token_text[0]) @@ -262,8 +258,10 @@ def _create_relation_validation(self, else _left_context_start_char_pos _right_context_start_end_pos = right_context_end_char_pos + (ann_tag_token_len * 4) + 8 # 8 for spces - right_context_end_char_pos = len(tmp_doc_text) + 1 if right_context_end_char_pos >= len(tmp_doc_text) or \ - _right_context_start_end_pos >= len(tmp_doc_text) else _right_context_start_end_pos + right_context_end_char_pos = len(tmp_doc_text) if ( + right_context_end_char_pos >= len(tmp_doc_text) + or _right_context_start_end_pos >= len(tmp_doc_text) + ) else _right_context_start_end_pos # reassign the new text with added tags text_length = len(tmp_doc_text) @@ -273,6 +271,9 @@ def _create_relation_validation(self, # is considered invalid window_tokenizer_data = self.tokenizer(tmp_doc_text[left_context_start_char_pos:right_context_end_char_pos], truncation=True) + #if len(window_tokenizer_data["input_ids"]) > self.tokenizer.hf_tokenizers.model_max_length: + # return [] + if self.config.general.annotation_schema_tag_ids: try: ent1_token_start_pos = \ @@ -287,10 +288,7 @@ def _create_relation_validation(self, _ent2_token_end_pos = \ window_tokenizer_data["input_ids"].index( self.config.general.annotation_schema_tag_ids[3]) - assert ent1_token_start_pos - assert ent2_token_start_pos - assert _ent1_token_end_pos - assert _ent2_token_end_pos + except Exception as exception: self.log.error("document id : " + str(doc_id) + " failed to process relation") self.log.info(exception) @@ -303,12 +301,17 @@ def _create_relation_validation(self, ent2_token_start_pos += ent1_token_start_pos ent1_ent2_new_start = (ent1_token_start_pos, ent2_token_start_pos) - en1_start, en1_end = window_tokenizer_data["offset_mapping"][ent1_token_start_pos] - en2_start, en2_end = window_tokenizer_data["offset_mapping"][ent2_token_start_pos] + + s1_start, s1_end = window_tokenizer_data["offset_mapping"][ent1_token_start_pos] + e1_start, e1_end = window_tokenizer_data["offset_mapping"][_ent1_token_end_pos] + + s2_start, s2_end = window_tokenizer_data["offset_mapping"][ent2_token_start_pos] + e2_start, e2_end = window_tokenizer_data["offset_mapping"][_ent2_token_end_pos] return [window_tokenizer_data["input_ids"], ent1_ent2_new_start, ent1_token, ent2_token, "UNK", self.config.model.padding_idx, None, None, None, None, None, None, doc_id, "", - en1_start, en1_end, en2_start, en2_end] + s1_start, e1_end, s2_start, e2_end, + ent1_start_char_pos, ent1_end_char_pos, ent2_start_char_pos, ent2_end_char_pos] return [] def create_base_relations_from_doc(self, doc: Union[Doc, str], doc_id: str, ent1_ent2_tokens_start_pos: Union[List, Tuple] = (-1, -1)) -> Dict: From 9a3758e641ddfc1a8b2f98ea0d9a8679d6292055 Mon Sep 17 00:00:00 2001 From: jocelyneholdbrook Date: Tue, 20 Jan 2026 15:14:05 +0000 Subject: [PATCH 04/22] feat(medcat-trainer): Externalise config for OIDC auth, allow top-level roles and add separate client for backend auth (#298) * refactor(medcat-trainer): Externalise VITE_ env vars, introduce separate client for backend auth for clarity, update API migrations, ensure roles can be read from top-level role group. * doc(medcat-trainer): Provide more comprehensive OIDC auth guide * doc(medcat-trainer): Update OIDC Auth guide and installation file * refactor(medcat-trainer): Address code review feedback * chore(medcat-trainer): Potential fix for code scanning alert no. 151: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> * refactor(medcat-trainer): Address more code review feedback --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- medcat-trainer/OIDC_AUTHENTICATION_GUIDE.md | 309 ++++++++++++++++++ medcat-trainer/docker-compose-dev.yml | 13 + medcat-trainer/docs/installation.md | 42 ++- medcat-trainer/envs/env | 11 - medcat-trainer/webapp/Dockerfile | 4 + medcat-trainer/webapp/api/api/oidc_utils.py | 6 +- medcat-trainer/webapp/api/core/settings.py | 53 +-- medcat-trainer/webapp/frontend/.env | 16 +- .../frontend/public/config.template.json | 7 + medcat-trainer/webapp/frontend/src/App.vue | 7 +- medcat-trainer/webapp/frontend/src/auth.ts | 19 +- .../frontend/src/components/common/Login.vue | 3 +- medcat-trainer/webapp/frontend/src/main.ts | 12 +- .../webapp/frontend/src/runtimeConfig.ts | 71 ++++ .../webapp/frontend/src/views/Home.vue | 3 +- .../webapp/scripts/nginx-entrypoint.sh | 51 +++ medcat-trainer/webapp/scripts/run.sh | 3 + 17 files changed, 555 insertions(+), 75 deletions(-) create mode 100644 medcat-trainer/OIDC_AUTHENTICATION_GUIDE.md create mode 100644 medcat-trainer/webapp/frontend/public/config.template.json create mode 100644 medcat-trainer/webapp/frontend/src/runtimeConfig.ts create mode 100644 medcat-trainer/webapp/scripts/nginx-entrypoint.sh diff --git a/medcat-trainer/OIDC_AUTHENTICATION_GUIDE.md b/medcat-trainer/OIDC_AUTHENTICATION_GUIDE.md new file mode 100644 index 000000000..218a82ce1 --- /dev/null +++ b/medcat-trainer/OIDC_AUTHENTICATION_GUIDE.md @@ -0,0 +1,309 @@ +# OIDC Authentication in MedCAT Trainer + +## Overview + +MedCAT Trainer supports OpenID Connect (OIDC) authentication via Keycloak, allowing users to log in with centralized credentials instead of managing separate accounts. + +### What is OIDC? + +OpenID Connect is an authentication protocol built on top of OAuth 2.0 that allows applications to verify user identity through a trusted identity provider (like Keycloak). + +**Key Concepts:** +- **Identity Provider (IdP)**: Keycloak server that manages user authentication +- **Client**: MedCAT Trainer (the application requesting authentication) +- **Token**: A signed JWT (JSON Web Token) that proves user identity +- **Realm**: A Keycloak namespace containing users, roles, and clients + +--- + +## Architecture + +MedCAT Trainer uses a **two-client architecture** for OIDC: + +### 1. Frontend Public Client (`cogstack-medcattrainer-frontend`) + +**Purpose:** Browser-based authentication flow + +**Type:** Public client (no secret, code runs in browser) + +**Responsibilities:** +- Redirects user to Keycloak login page +- Receives authentication token +- Includes token in API requests + +**Configuration:** +// Frontend config (loaded at runtime) +```json +{ + "USE_OIDC": "1", + "KEYCLOAK_URL": "https://auth.cogstack.example.site", + "KEYCLOAK_REALM": "cogstack", + "KEYCLOAK_CLIENT_ID": "cogstack-medcattrainer-frontend", + "LOGOUT_REDIRECT_URI": "https://launchpad.cogstack.example.site/" +} +``` + +### 2. Backend Confidential Client (`cogstack-medcattrainer-backend`) + +**Purpose:** Server-side token validation + +**Type:** Confidential client (has secret, runs on server) + +**Responsibilities:** +- Validates tokens from frontend +- Extracts user information and roles +- Creates/updates Django users + +**Configuration:** +```python +# Backend settings +OIDC_HOST = "https://auth.cogstack.example.site" +OIDC_REALM = "cogstack" +OIDC_FRONTEND_CLIENT_ID = "cogstack-medcattrainer-frontend" +OIDC_BACKEND_CLIENT_ID = "cogstack-medcattrainer-backend" +OIDC_BACKEND_CLIENT_SECRET = "***secret***" +``` +--- +## Key Files + +### Frontend + +| File | Purpose | +|------|---------| +| `src/runtimeConfig.ts` | Loads and provides runtime config | +| `src/auth.ts` | Keycloak initialization and setup | +| `src/main.ts` | App bootstrap, conditionally loads OIDC | +| `src/App.vue` | Handles login/logout, shows username | +| `public/config.template.json` | Template for runtime config | + +### Backend + +| File | Purpose | +|------|---------| +| `api/core/settings.py` | OIDC configuration and DRF setup | +| `api/api/oidc_utils.py` | User creation/update from token | +| `scripts/nginx-entrypoint.sh` | Runtime config generation | +| `scripts/run.sh` | Startup script (runs nginx-entrypoint.sh) | + +--- + +## User Lifecycle + +### First Login + +1. User logs in via Keycloak +2. Token generated with user info +3. Backend receives token +4. `get_user_by_email()` called +5. Django user created: + ```python + User.objects.get_or_create( + email='john@cogstack.org', + defaults={ + "username": "johndoe", + "first_name": "John", + "last_name": "Doe", + "is_active": True, + "password": secrets.token_urlsafe(32), # Random, unused + "is_superuser": False, # Set based on roles + "is_staff": False, + } + ) + ``` + +### Subsequent Logins + +1. User logs in via Keycloak +2. Token generated +3. Backend finds existing user by email +4. Updates user information: + ```python + user.username = token['preferred_username'] + user.first_name = token['given_name'] + user.last_name = token['family_name'] + user.is_superuser = 'medcattrainer_superuser' in roles + user.is_staff = 'medcattrainer_staff' in roles + user.save() + ``` + +### Role Changes + +If a user's Keycloak roles change: +1. User logs out +2. User logs back in +3. New token includes updated roles +4. Backend updates Django user permissions +5. User immediately has new access level + +--- + +## Troubleshooting + +### User sees old login form instead of Keycloak + +**Symptoms:** +- Old username/password form appears +- No redirect to Keycloak + +**Causes:** +1. Runtime config not loaded +2. `USE_OIDC` not set to `1` +3. Frontend failed to load `/static/config.json` + +**Solutions:** +```bash +# Check config exists +curl http://medcattrainer.../static/config.json + +# Check environment variables +docker exec env | grep VITE_USE_OIDC + +# Check logs +docker logs | grep "RuntimeConfig" +``` + +### API returns 401 Unauthorized + +**Symptoms:** +- Login works but API calls fail +- Browser console shows 401 errors +- Keycloak logs: "Token is not active" + +**Causes:** +1. Backend not accepting frontend client ID in audience +2. Token expired +3. Token signature invalid + +**Solutions:** +```bash +# Check OIDC_FRONTEND_CLIENT_ID is set +docker exec env | grep OIDC_FRONTEND_CLIENT_ID + +# Check backend logs for audience claims +docker logs | grep "Accepted audience claims" + +# Should show: account, backend-client-id, frontend-client-id +``` + +### User has no permissions after login + +**Symptoms:** +- Login successful +- User sees no projects +- Not a superuser/staff + +**Causes:** +1. Keycloak roles not mapped +2. Backend not reading roles from correct location in token +3. User needs role assigned in Keycloak + +**Solutions:** +```bash +# Check user in database +docker exec python manage.py shell -c " +from django.contrib.auth import get_user_model +User = get_user_model() +user = User.objects.get(email='user@example.com') +print(f'is_superuser: {user.is_superuser}') +print(f'is_staff: {user.is_staff}') +" + +# Check token includes roles +# Look in backend logs for printed id_token + +# Assign role in Keycloak: +# Users → Select user → Role mapping → Assign role +``` + +### Token validation fails + +**Symptoms:** +- "Invalid token" errors +- "Signature verification failed" + +**Causes:** +1. Keycloak public key changed +2. Wrong OIDC_HOST configuration + +**Solutions:** +```bash +# Verify OIDC_HOST matches Keycloak +docker exec env | grep OIDC_HOST + +# Check Keycloak is reachable from container +docker exec curl -I /realms/ + +# Restart container to refresh public keys +docker restart +``` + +--- + +## Testing OIDC Locally + +### Prerequisites + +1. Running Keycloak instance +2. Realm configured (`cogstack-realm`) +3. Two clients created: + - `cogstack-medcattrainer-frontend` (public) + - `cogstack-medcattrainer-backend` (confidential with secret) +4. Realm roles: + - `medcattrainer_superuser` + - `medcattrainer_staff` +5. Test user with assigned roles + +### Local Configuration + +```bash +# envs/env +USE_OIDC=1 +OIDC_HOST=http://keycloak:8080 +OIDC_REALM=cogstack-realm +OIDC_FRONTEND_CLIENT_ID=cogstack-medcattrainer-frontend +OIDC_BACKEND_CLIENT_ID=cogstack-medcattrainer-backend +OIDC_BACKEND_CLIENT_SECRET=your-secret-here + +VITE_USE_OIDC=1 +VITE_KEYCLOAK_URL=http://keycloak.cogstack.localhost/ +VITE_KEYCLOAK_REALM=cogstack-realm +VITE_KEYCLOAK_CLIENT_ID=cogstack-medcattrainer-frontend +VITE_LOGOUT_REDIRECT_URI=http://home.cogstack.localhost/ +``` + +### Test Flow + +1. Start services: + ```bash + docker-compose -f docker-compose-dev.yml up + ``` + +2. Visit `http://medcattrainer.cogstack.localhost` + +3. Should redirect to Keycloak login + +4. Login with test user + +5. Should redirect back and show MedCAT Trainer with username in header + +6. Check browser console (F12): + ``` + [RuntimeConfig] OIDC enabled: true + [Bootstrap] OIDC mode enabled + ``` + +7. Check backend logs: + ```bash + docker logs medcat-trainer-medcattrainer-1 | grep OIDC + # Should show: "Using OIDC authentication" + # Should show: "Accepted audience claims: ..." + ``` + +--- + +## References + +- [OpenID Connect Specification](https://openid.net/specs/openid-connect-core-1_0.html) +- [Keycloak Documentation](https://www.keycloak.org/documentation) +- [Django REST Framework Token Authentication](https://www.django-rest-framework.org/api-guide/authentication/) +- [drf-oidc-auth](https://github.com/ByteInternet/drf-oidc-auth) diff --git a/medcat-trainer/docker-compose-dev.yml b/medcat-trainer/docker-compose-dev.yml index 3beaa9c49..26247c249 100644 --- a/medcat-trainer/docker-compose-dev.yml +++ b/medcat-trainer/docker-compose-dev.yml @@ -24,6 +24,19 @@ services: - ./envs/env environment: - MCT_VERSION=latest + # Frontend OIDC Settings (Public Client) + - VITE_USE_OIDC=1 + - VITE_KEYCLOAK_URL=http://keycloak.cogstack.localhost + - VITE_KEYCLOAK_REALM=cogstack-realm + - VITE_KEYCLOAK_CLIENT_ID=cogstack-medcattrainer-frontend + - VITE_LOGOUT_REDIRECT_URI=http://launch.cogstack.localhost/ + # Backend OIDC Settings (Confidential Client) + - USE_OIDC=1 + - OIDC_HOST=http://keycloak:8080 + - OIDC_REALM=cogstack-realm + - OIDC_FRONTEND_CLIENT_ID=cogstack-medcattrainer-frontend + - OIDC_BACKEND_CLIENT_ID=cogstack-medcattrainer-backend + - OIDC_BACKEND_CLIENT_SECRET=*** command: /usr/bin/supervisord -c /etc/supervisord.conf nginx: diff --git a/medcat-trainer/docs/installation.md b/medcat-trainer/docs/installation.md index 12a37b1e3..fc8fece90 100644 --- a/medcat-trainer/docs/installation.md +++ b/medcat-trainer/docs/installation.md @@ -77,25 +77,39 @@ You'll need to `docker stop` the running containers if you have already run the You can enable OIDC (OpenID Connect) authentication for the MedCAT Trainer. To do so, you must configure the following environment variables: -| Variable | Used by | Description | -|-----------------------------------------|-------------------------|----------------------------------------------------------------| -| USE_OIDC | Backend | Enable OIDC login flow (1 (true) / 0 (false)). | -| VITE_USE_OIDC | Frontend | Exposed version of USE_OIDC for Vue. | -| VITE_API_URL | Frontend | Base API URL for frontend calls. | -| VITE_KEYCLOAK_URL | Frontend | Keycloak base URL (e.g. http://keycloak.cogstack.localhost/). | -| VITE_KEYCLOAK_REALM | Frontend | Keycloak realm name. | -| VITE_KEYCLOAK_CLIENT_ID | Frontend | Keycloak client ID for this app. | -| VITE_KEYCLOAK_TOKEN_REFRESH_INTERVAL_MS | Frontend | Token refresh frequency in ms. | -| VITE_KEYCLOAK_TOKEN_MIN_VALIDITY_SECS | Frontend | Minimum token validity before refresh. | -| VITE_LOGOUT_REDIRECT_URI | Frontend | Where to send user after logout. | -| NGINX_HOST | Backend | Host alias used by reverse proxy (Traefik ) | +#### Frontend (Runtime Config) + +| Variable | Example | Description | +|----------|---------|-------------| +| `VITE_USE_OIDC` | `1` | Enable OIDC (1=enabled, 0=traditional auth) | +| `VITE_KEYCLOAK_URL` | `https://cogstack-auth.sites.er.kcl.ac.uk` | Keycloak base URL | +| `VITE_KEYCLOAK_REALM` | `cogstack` | Keycloak realm name | +| `VITE_KEYCLOAK_CLIENT_ID` | `cogstack-medcattrainer-frontend` | Public client ID | +| `VITE_LOGOUT_REDIRECT_URI` | `https://cogstack-launchpad.sites.er.kcl.ac.uk/` | Where to go after logout | + +#### Backend (Django Settings) + +| Variable | Example | Description | +|----------|---------|-------------| +| `USE_OIDC` | `1` | Enable OIDC validation | +| `OIDC_HOST` | `https://cogstack-auth.sites.er.kcl.ac.uk` | Keycloak base URL (for backend) | +| `OIDC_REALM` | `cogstack` | Realm name | +| `OIDC_FRONTEND_CLIENT_ID` | `cogstack-medcattrainer-frontend` | Frontend client ID (for token validation) | +| `OIDC_BACKEND_CLIENT_ID` | `cogstack-medcattrainer-backend` | Backend client ID | +| `OIDC_BACKEND_CLIENT_SECRET` | `***secret***` | Backend client secret | + You can either use the Gateway Auth stack available in cogstack-ops or deploy your own Keycloak instance. If you deploy your own Keycloak instance, make sure to configure the network accordingly. Currently, there are two roles that can be assigned to users: -- medcattrainer_superuser: grants superuser privileges in the application. -- medcattrainer_staff: grants staff-level privileges without full superuser access. + +| Keycloak Role | Django Permission | Capabilities | +|---------------|-------------------|--------------| +| `medcattrainer_superuser` | `is_superuser=True`, `is_staff=True` | Full admin access, Django admin, all projects | +| `medcattrainer_staff` | `is_staff=True` | Staff-level access, can manage assigned projects | +| (no role) | Regular user | Can only access assigned projects, no admin | + ### (Optional) Postgres Database Support MedCAT trainer defaults to a local SQLite database, which is suitable for single-user or small-scale setups. diff --git a/medcat-trainer/envs/env b/medcat-trainer/envs/env index 822a0f3d4..ba7c52d93 100644 --- a/medcat-trainer/envs/env +++ b/medcat-trainer/envs/env @@ -46,14 +46,3 @@ EMAIL_USER=example@cogstack.org EMAIL_PASS="to be changed" EMAIL_HOST=mail.cogstack.org EMAIL_PORT=465 - -# Backend auth -USE_OIDC= - -# Traefik -NGINX_HOST=medcattrainer.cogstack.localhost - -# OIDC settings -OIDC_HOST=http://keycloak:8080 -OIDC_REALM=cogstack-realm -OIDC_FRONTEND_CLIENT_ID=cogstack-medcattrainer-frontend \ No newline at end of file diff --git a/medcat-trainer/webapp/Dockerfile b/medcat-trainer/webapp/Dockerfile index 917344666..a643948e2 100644 --- a/medcat-trainer/webapp/Dockerfile +++ b/medcat-trainer/webapp/Dockerfile @@ -10,6 +10,9 @@ RUN apt-get install -y vim # install supervisor RUN apt-get install -y supervisor +# install gettext for envsubst (used to generate runtime config) +RUN apt-get install -y gettext + # install cron - and remove any default tabs RUN apt-get install -y cron && which cron && rm -rf /etc/cron.*/* @@ -26,6 +29,7 @@ COPY ./ . # copy backup crontab and chmod scripts RUN chmod u+x /home/scripts/entry.sh && chmod u+x /home/scripts/crontab && cp /home/scripts/crontab /etc/crontab +RUN chmod u+x /home/scripts/nginx-entrypoint.sh # Build frontend WORKDIR /home/frontend diff --git a/medcat-trainer/webapp/api/api/oidc_utils.py b/medcat-trainer/webapp/api/api/oidc_utils.py index 93bb6d4eb..d510b7b36 100644 --- a/medcat-trainer/webapp/api/api/oidc_utils.py +++ b/medcat-trainer/webapp/api/api/oidc_utils.py @@ -8,11 +8,7 @@ def get_user_by_email(request, id_token): User = get_user_model() email = id_token.get('email') username = id_token.get('preferred_username') - print(id_token) - roles = [] - if 'realm_access' in id_token: - roles = id_token['realm_access'].get('roles', []) - + roles = id_token.get('roles', []) is_superuser = 'medcattrainer_superuser' in roles is_staff = 'medcattrainer_staff' in roles diff --git a/medcat-trainer/webapp/api/core/settings.py b/medcat-trainer/webapp/api/core/settings.py index 465f72a35..dd1a2c381 100644 --- a/medcat-trainer/webapp/api/core/settings.py +++ b/medcat-trainer/webapp/api/core/settings.py @@ -206,31 +206,36 @@ 'oidc_auth.authentication.BearerTokenAuthentication', ] -OIDC_HOST = os.environ.get("OIDC_HOST", "") -OIDC_REALM = os.environ.get("OIDC_REALM", "") -OIDC_FRONTEND_CLIENT_ID = os.environ.get("OIDC_FRONTEND_CLIENT_ID", "") - -OIDC_AUTH = { - 'OIDC_ENDPOINT': f"{OIDC_HOST}/realms/{OIDC_REALM}", - 'OIDC_CLAIMS_OPTIONS': { - 'aud': { - 'values': [ - 'account', - OIDC_FRONTEND_CLIENT_ID - ], - 'essential': True, - }, - 'iss': { - 'values': [ - f"{OIDC_HOST}/realms/{OIDC_REALM}" - ], - 'essential': True, + OIDC_HOST = os.environ.get('OIDC_HOST', '') + OIDC_REALM = os.environ.get('OIDC_REALM', default='cogstack-realm') + OIDC_BACKEND_CLIENT_ID = os.environ.get('OIDC_BACKEND_CLIENT_ID', default='cogstack-medcattrainer-backend') + OIDC_BACKEND_CLIENT_SECRET = os.environ.get('OIDC_BACKEND_CLIENT_SECRET', default='') + OIDC_FRONTEND_CLIENT_ID = os.environ.get('OIDC_FRONTEND_CLIENT_ID', default='cogstack-medcattrainer-frontend') + OIDC_AUTH = { + 'OIDC_ENDPOINT': f"{OIDC_HOST}/realms/{OIDC_REALM}", + 'OIDC_CLAIMS_OPTIONS': { + 'aud': { + 'values': [ + 'account', + OIDC_BACKEND_CLIENT_ID, + OIDC_FRONTEND_CLIENT_ID + ], + 'essential': True, + }, + 'iss': { + 'values': [ + f"{OIDC_HOST}/realms/{OIDC_REALM}" + ], + 'essential': True, + }, }, - }, - 'USERINFO_ENDPOINT': f"{OIDC_HOST}/realms/{OIDC_REALM}/protocol/openid-connect/userinfo", - 'OIDC_CREATE_USER': True, - 'OIDC_RESOLVE_USER_FUNCTION': 'api.oidc_utils.get_user_by_email', -} + 'USERINFO_ENDPOINT': f"{OIDC_HOST}/realms/{OIDC_REALM}/protocol/openid-connect/userinfo", + 'OIDC_CREATE_USER': True, + 'OIDC_RESOLVE_USER_FUNCTION': 'api.oidc_utils.get_user_by_email', + 'OIDC_CLIENT_ID': OIDC_BACKEND_CLIENT_ID, + 'OIDC_CLIENT_SECRET': OIDC_BACKEND_CLIENT_SECRET, + } + # Internationalization # https://docs.djangoproject.com/en/2.2/topics/i18n/ diff --git a/medcat-trainer/webapp/frontend/.env b/medcat-trainer/webapp/frontend/.env index 52b8dc85a..95450525a 100644 --- a/medcat-trainer/webapp/frontend/.env +++ b/medcat-trainer/webapp/frontend/.env @@ -1,9 +1,17 @@ # Override this with your .env.production # Frontend auth settings -VITE_USE_OIDC=0 +VITE_USE_OIDC=1 VITE_KEYCLOAK_URL=http://keycloak.cogstack.localhost/ VITE_KEYCLOAK_REALM=cogstack-realm VITE_KEYCLOAK_CLIENT_ID=cogstack-medcattrainer-frontend -VITE_KEYCLOAK_TOKEN_REFRESH_INTERVAL_MS=10000 -VITE_KEYCLOAK_TOKEN_MIN_VALIDITY_SECS=30 -VITE_LOGOUT_REDIRECT_URI=http://home.cogstack.localhost/ \ No newline at end of file +VITE_LOGOUT_REDIRECT_URI=http://home.cogstack.localhost/ +VITE_KEYCLOAK_TOKEN_REFRESH_INTERVAL=10000 +VITE_KEYCLOAK_TOKEN_MIN_VALIDITY=60 + +# Backend OIDC Settings (Confidential Client) +USE_OIDC=1 +OIDC_HOST=https://cogstack-auth.sites.er.kcl.ac.uk +OIDC_REALM=cogstack-realm +OIDC_FRONTEND_CLIENT_ID=cogstack-medcattrainer-frontend +OIDC_BACKEND_CLIENT_ID=cogstack-medcattrainer-backend +OIDC_BACKEND_CLIENT_SECRET=*** \ No newline at end of file diff --git a/medcat-trainer/webapp/frontend/public/config.template.json b/medcat-trainer/webapp/frontend/public/config.template.json new file mode 100644 index 000000000..e028730f6 --- /dev/null +++ b/medcat-trainer/webapp/frontend/public/config.template.json @@ -0,0 +1,7 @@ +{ + "USE_OIDC": "${VITE_USE_OIDC}", + "KEYCLOAK_URL": "${VITE_KEYCLOAK_URL}", + "KEYCLOAK_REALM": "${VITE_KEYCLOAK_REALM}", + "KEYCLOAK_CLIENT_ID": "${VITE_KEYCLOAK_CLIENT_ID}", + "LOGOUT_REDIRECT_URI": "${VITE_LOGOUT_REDIRECT_URI}" +} diff --git a/medcat-trainer/webapp/frontend/src/App.vue b/medcat-trainer/webapp/frontend/src/App.vue index f6a9bc177..52e56dd8e 100644 --- a/medcat-trainer/webapp/frontend/src/App.vue +++ b/medcat-trainer/webapp/frontend/src/App.vue @@ -52,8 +52,7 @@ + + + diff --git a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue index 61872d4d1..d036468cb 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/ClinicalText.vue @@ -160,9 +160,10 @@ export default { styleClass = `highlight-task-${btnIndex}` } - if (ent.id === this.currentRelStartEnt.id) { + // Only add relation markers if currentRelStartEnt/EndEnt have valid IDs + if (this.currentRelStartEnt && this.currentRelStartEnt.id && ent.id === this.currentRelStartEnt.id) { styleClass += ' current-rel-start' - } else if (ent.id === this.currentRelEndEnt.id) { + } else if (this.currentRelEndEnt && this.currentRelEndEnt.id && ent.id === this.currentRelEndEnt.id) { styleClass += ' current-rel-end' } diff --git a/medcat-trainer/webapp/frontend/src/components/common/ConceptSummary.vue b/medcat-trainer/webapp/frontend/src/components/common/ConceptSummary.vue index e51d46983..5c0f6bc54 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/ConceptSummary.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/ConceptSummary.vue @@ -188,6 +188,14 @@ export default { diff --git a/medcat-trainer/webapp/frontend/src/components/common/ProjectList.vue b/medcat-trainer/webapp/frontend/src/components/common/ProjectList.vue index c79e8eafd..e6152b7e8 100644 --- a/medcat-trainer/webapp/frontend/src/components/common/ProjectList.vue +++ b/medcat-trainer/webapp/frontend/src/components/common/ProjectList.vue @@ -299,13 +299,13 @@ export default { }, confirmClearLoadedModel (projectId) { this.clearModelModal = false - this.$http.delete(`/api/cache-model/${projectId}/`).then(_ => { + this.$http.delete(`/api/cache-project-model/${projectId}/`).then(_ => { this.fetchModelsLoaded() }) }, loadProjectCDB (projectId) { this.loadingModel = projectId - this.$http.get(`/api/cache-model/${projectId}/`).then(_ => { + this.$http.get(`/api/cache-project-model/${projectId}/`).then(_ => { this.loadingModel = false this.fetchModelsLoaded() }).catch(_ => { @@ -334,10 +334,10 @@ export default { if (this.selectedProjects.length === 0) { return {class: ''} } else { - let disabled = !(this.selectedProjects[0].concept_db === data.item.concept_db && + let disabled = !(this.selectedProjects[0].concept_db === data.item.concept_db && this.selectedProjects[0].vocab === data.item.vocab) || this.selectedProjects[0].model_pack !== data.item.model_pack - return {class: disabled ? ' disabled-row' : ''} + return {class: disabled ? ' disabled-row' : ''} } }, submitMetricsReportReq () { diff --git a/medcat-trainer/webapp/frontend/src/tests/views/ConceptDatabase.spec.ts b/medcat-trainer/webapp/frontend/src/tests/views/ConceptDatabase.spec.ts index 579d8c339..d33616c0b 100644 --- a/medcat-trainer/webapp/frontend/src/tests/views/ConceptDatabase.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/views/ConceptDatabase.spec.ts @@ -11,9 +11,9 @@ const mockCdbsPage1 = { } describe('ConceptDatabase.vue', () => { - let mockGet + let mockGet: ReturnType beforeEach(() => { - mockGet = vi.fn((url) => { + mockGet = vi.fn((url: string) => { if (url === '/api/concept-dbs/') { return Promise.resolve({ data: mockCdbsPage1 }) } diff --git a/medcat-trainer/webapp/frontend/src/tests/views/Demo.spec.ts b/medcat-trainer/webapp/frontend/src/tests/views/Demo.spec.ts index 606c4d5c6..a0ea47f19 100644 --- a/medcat-trainer/webapp/frontend/src/tests/views/Demo.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/views/Demo.spec.ts @@ -1,33 +1,65 @@ -import { describe, it, expect, vi } from 'vitest' -import { mount, flushPromises } from '@vue/test-utils' -import Demo from '@/views/Demo.vue' - -describe('Demo.vue', () => { - it('posts to /api/annotate-text/ with correct payload when annotate is clicked', async () => { - const mockPost = vi.fn().mockResolvedValue({ data: { entities: [], message: 'annotated!' } }) - const mockGet = vi.fn().mockResolvedValue({ data: { count: 1, results: [{ id: 42, name: 'Test Project', cdb_search_filter: [] }], next: null } }) - const wrapper = mount(Demo, { - global: { - mocks: { - $http: { get: mockGet, post: mockPost } - }, - stubs: ['clinical-text', 'concept-summary'] - } - }) - await flushPromises() - // Set up form values - await wrapper.setData({ - selectedProject: { id: 42, name: 'Test Project', cdb_search_filter: [] }, - exampleText: 'Some text to annotate', - cuiFilters: 'C1234,C5678' - }) - // Find and click the annotate button - await wrapper.find('button.btn-primary').trigger('click') - await flushPromises() - expect(mockPost).toHaveBeenCalledWith('/api/annotate-text/', { - project_id: 42, - message: 'Some text to annotate', - cuis: 'C1234,C5678' - }) - }) -}) +import { describe, it, expect, vi } from 'vitest' +import { mount, flushPromises } from '@vue/test-utils' +import Demo from '@/views/Demo.vue' + +describe('Demo.vue', () => { + it('posts to /api/annotate-text/ with correct payload when annotate is clicked', async () => { + vi.useFakeTimers() + const mockPost = vi.fn().mockResolvedValue({ data: { entities: [], message: 'annotated!' } }) + const mockGet = vi.fn((url: string) => { + if (url === '/api/project-annotate-entities/') { + return Promise.resolve({ + data: { + count: 1, + results: [{ id: 42, name: 'Test Project', model_pack: 7, cdb_search_filter: [] }], + next: null + } + }) + } + if (url === '/api/modelpacks/') { + return Promise.resolve({ + data: { + count: 1, + results: [{ id: 7, name: 'Test ModelPack' }], + next: null + } + }) + } + if (url === '/api/cache-modelpack/7/') { + return Promise.resolve({ data: 'success' }) + } + return Promise.resolve({ data: { count: 0, results: [], next: null } }) + }) + const wrapper = mount(Demo, { + global: { + mocks: { + $http: { get: mockGet, post: mockPost } + }, + stubs: { + 'clinical-text': true, + 'concept-summary': true, + 'concept-picker': true, + 'meta-annotations-summary': true + } + } + }) + await flushPromises() + // Paste CUIs (optional box) still drives payload + await wrapper.setData({ cuiFilters: 'C1234,C5678' }) + + // Enter message in the annotate component textarea + const textarea = wrapper.find('textarea[name="message"]') + expect(textarea.exists()).toBe(true) + await textarea.setValue('Some text to annotate') + // Debounced auto-annotate after 1500ms of inactivity + vi.advanceTimersByTime(1500) + await flushPromises() + expect(mockPost).toHaveBeenCalledWith('/api/annotate-text/', expect.objectContaining({ + modelpack_id: 7, + message: 'Some text to annotate', + cuis: 'C1234,C5678', + include_sub_concepts: false + })) + vi.useRealTimers() + }) +}) diff --git a/medcat-trainer/webapp/frontend/src/tests/views/MetricsHome.spec.ts b/medcat-trainer/webapp/frontend/src/tests/views/MetricsHome.spec.ts index ece0ccd6d..15a1358a6 100644 --- a/medcat-trainer/webapp/frontend/src/tests/views/MetricsHome.spec.ts +++ b/medcat-trainer/webapp/frontend/src/tests/views/MetricsHome.spec.ts @@ -20,9 +20,9 @@ const mockProjects = { } describe('MetricsHome.vue', () => { - let mockGet + let mockGet: ReturnType beforeEach(() => { - mockGet = vi.fn((url) => { + mockGet = vi.fn((url: string) => { if (url === '/api/metrics-job/') { return Promise.resolve({ data: { reports: mockReports } }) } @@ -40,7 +40,7 @@ describe('MetricsHome.vue', () => { mount(MetricsHome, { global: { mocks: { $http: { get: mockGet } }, - stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'modal', 'font-awesome-icon', 'router-link'] + stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'v-tooltip', 'v-runtime-template', 'modal', 'font-awesome-icon', 'router-link'] } }) await flushPromises() @@ -51,7 +51,7 @@ describe('MetricsHome.vue', () => { mount(MetricsHome, { global: { mocks: { $http: { get: mockGet } }, - stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'modal', 'font-awesome-icon', 'router-link'] + stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'v-tooltip', 'v-runtime-template', 'modal', 'font-awesome-icon', 'router-link'] } }) await flushPromises() @@ -63,11 +63,11 @@ describe('MetricsHome.vue', () => { const wrapper = mount(MetricsHome, { global: { mocks: { $http: { get: mockGet } }, - stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'modal', 'font-awesome-icon', 'router-link'] + stubs: ['v-data-table', 'v-overlay', 'v-progress-circular', 'v-tooltip', 'v-runtime-template', 'modal', 'font-awesome-icon', 'router-link'] } }) await flushPromises() - const headers = wrapper.vm.reports.headers.map(h => h.title) + const headers = wrapper.vm.reports.headers.map((h: { title: string }) => h.title) expect(headers).toEqual([ 'ID', 'Report Name', diff --git a/medcat-trainer/webapp/frontend/src/views/Demo.vue b/medcat-trainer/webapp/frontend/src/views/Demo.vue index 2a3c28ce6..b7d825c75 100644 --- a/medcat-trainer/webapp/frontend/src/views/Demo.vue +++ b/medcat-trainer/webapp/frontend/src/views/Demo.vue @@ -3,40 +3,91 @@
- - +
-
- - +
+ {{ toast.message }}
- - + +
+ + +
+ +
+
+ Selected ModelPack does not have a Concept DB. +
+ +
+ +
+ + {{ item.cui }} - {{ item.name }} + + +
+
-
- +
- +
+ +
+
diff --git a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue index 257569fe6..087acb455 100644 --- a/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue +++ b/medcat-trainer/webapp/frontend/src/views/TrainAnnotations.vue @@ -492,7 +492,7 @@ export default { this.fetchEntities() } else { this.loadingMsg = "Loading MedCAT model..." - this.$http.get(`/api/cache-model/${this.project.id}/`).then(_ => { + this.$http.get(`/api/cache-project-model/${this.project.id}/`).then(_ => { this.loadingMsg = "Preparing Document..." let payload = { project_id: this.project.id, diff --git a/medcat-trainer/webapp/frontend/tsconfig.vitest.json b/medcat-trainer/webapp/frontend/tsconfig.vitest.json index 571995d11..1f7f6ce5f 100644 --- a/medcat-trainer/webapp/frontend/tsconfig.vitest.json +++ b/medcat-trainer/webapp/frontend/tsconfig.vitest.json @@ -1,5 +1,6 @@ { "extends": "./tsconfig.app.json", + "include": ["env.d.ts", "src/tests/**/*.ts"], "exclude": [], "compilerOptions": { "composite": true, From 89d377826c1e167f3c248c1f99f16c2ec888f8ad Mon Sep 17 00:00:00 2001 From: Alistair Hendrickson Date: Wed, 21 Jan 2026 10:06:54 +0000 Subject: [PATCH 06/22] fix(medcat-trainer): Postgres load_examples fails on unique constraint as models.py calls insert twice (#299) --- medcat-trainer/webapp/api/api/models.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/medcat-trainer/webapp/api/api/models.py b/medcat-trainer/webapp/api/api/models.py index e89459752..08ea6c877 100644 --- a/medcat-trainer/webapp/api/api/models.py +++ b/medcat-trainer/webapp/api/api/models.py @@ -132,7 +132,12 @@ def save(self, *args, skip_load=False, **kwargs): except Exception as exc: raise MedCATLoadException(f'Failure loading MetaCAT models - {unpacked_model_pack_path}') from exc - super().save(*args, **kwargs) + if not is_new: + super().save(*args, **kwargs) + else: + # For new objects, just update the FK fields without full save + # Fixes psycopg.errors.UniqueViolation: duplicate key value violates unique constraint "api_modelpack_pkey" + super().save(update_fields=['concept_db', 'vocab']) def __str__(self): return self.name From 7bb8b6ffe7b95614e306c3fff1b3249efa6f0e43 Mon Sep 17 00:00:00 2001 From: Alistair Hendrickson Date: Wed, 21 Jan 2026 10:33:59 +0000 Subject: [PATCH 07/22] fix(medcat-trainer): Pass kwargs to ModelPack.save to stay consistent (#300) --- medcat-trainer/webapp/api/api/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/medcat-trainer/webapp/api/api/models.py b/medcat-trainer/webapp/api/api/models.py index 08ea6c877..eee883c31 100644 --- a/medcat-trainer/webapp/api/api/models.py +++ b/medcat-trainer/webapp/api/api/models.py @@ -137,7 +137,7 @@ def save(self, *args, skip_load=False, **kwargs): else: # For new objects, just update the FK fields without full save # Fixes psycopg.errors.UniqueViolation: duplicate key value violates unique constraint "api_modelpack_pkey" - super().save(update_fields=['concept_db', 'vocab']) + super().save(*args, update_fields=['concept_db', 'vocab'], **kwargs) def __str__(self): return self.name From 05fdc42c4fc9403f1229b4b6a18eddc01cfc53c6 Mon Sep 17 00:00:00 2001 From: jocelyneholdbrook Date: Wed, 21 Jan 2026 10:51:46 +0000 Subject: [PATCH 08/22] fix(medcat-trainer): Make VITE_ env vars mandatory only when VITE_USE_OIDC is set to true (#301) --- .../webapp/frontend/src/runtimeConfig.ts | 12 ++-- .../webapp/scripts/nginx-entrypoint.sh | 58 +++++++++++-------- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/runtimeConfig.ts b/medcat-trainer/webapp/frontend/src/runtimeConfig.ts index a23b111e6..6acc7d016 100644 --- a/medcat-trainer/webapp/frontend/src/runtimeConfig.ts +++ b/medcat-trainer/webapp/frontend/src/runtimeConfig.ts @@ -28,12 +28,12 @@ declare global { */ const DEFAULT_CONFIG: RuntimeConfig = { USE_OIDC: '0', - KEYCLOAK_URL: 'http://keycloak.cogstack.localhost/', - KEYCLOAK_REALM: 'cogstack-realm', - KEYCLOAK_CLIENT_ID: 'cogstack-medcattrainer-frontend', - LOGOUT_REDIRECT_URI: 'http://launch.cogstack.localhost/', - KEYCLOAK_TOKEN_MIN_VALIDITY: 10000, - KEYCLOAK_TOKEN_REFRESH_INTERVAL: 30 + KEYCLOAK_URL: '', + KEYCLOAK_REALM: '', + KEYCLOAK_CLIENT_ID: '', + LOGOUT_REDIRECT_URI: '', + KEYCLOAK_TOKEN_MIN_VALIDITY: 0, + KEYCLOAK_TOKEN_REFRESH_INTERVAL: 0 }; /** diff --git a/medcat-trainer/webapp/scripts/nginx-entrypoint.sh b/medcat-trainer/webapp/scripts/nginx-entrypoint.sh index 8d44f63cc..fecfb0fe7 100644 --- a/medcat-trainer/webapp/scripts/nginx-entrypoint.sh +++ b/medcat-trainer/webapp/scripts/nginx-entrypoint.sh @@ -3,30 +3,40 @@ set -e echo "Generating runtime config.json from template..." -# Verify required environment variables are set -if [ -z "$VITE_USE_OIDC" ]; then - echo "ERROR: VITE_USE_OIDC environment variable is required" - exit 1 -fi - -if [ -z "$VITE_KEYCLOAK_URL" ]; then - echo "ERROR: VITE_KEYCLOAK_URL environment variable is required" - exit 1 -fi - -if [ -z "$VITE_KEYCLOAK_REALM" ]; then - echo "ERROR: VITE_KEYCLOAK_REALM environment variable is required" - exit 1 -fi - -if [ -z "$VITE_KEYCLOAK_CLIENT_ID" ]; then - echo "ERROR: VITE_KEYCLOAK_CLIENT_ID environment variable is required" - exit 1 -fi - -if [ -z "$VITE_LOGOUT_REDIRECT_URI" ]; then - echo "ERROR: VITE_LOGOUT_REDIRECT_URI environment variable is required" - exit 1 +# Set VITE_USE_OIDC to 0 if not provided (traditional auth mode) +export VITE_USE_OIDC="${VITE_USE_OIDC:-0}" + +# If OIDC is enabled, require all OIDC-related variables +if [ "$VITE_USE_OIDC" = "1" ]; then + echo "OIDC mode enabled - validating OIDC environment variables..." + + if [ -z "$VITE_KEYCLOAK_URL" ]; then + echo "ERROR: VITE_KEYCLOAK_URL environment variable is required when VITE_USE_OIDC=1" + exit 1 + fi + + if [ -z "$VITE_KEYCLOAK_REALM" ]; then + echo "ERROR: VITE_KEYCLOAK_REALM environment variable is required when VITE_USE_OIDC=1" + exit 1 + fi + + if [ -z "$VITE_KEYCLOAK_CLIENT_ID" ]; then + echo "ERROR: VITE_KEYCLOAK_CLIENT_ID environment variable is required when VITE_USE_OIDC=1" + exit 1 + fi + + if [ -z "$VITE_LOGOUT_REDIRECT_URI" ]; then + echo "ERROR: VITE_LOGOUT_REDIRECT_URI environment variable is required when VITE_USE_OIDC=1" + exit 1 + fi + +else + echo "Traditional auth mode enabled (VITE_USE_OIDC=0)" + # Traditional auth mode - set defaults for unused variables + export VITE_KEYCLOAK_URL="${VITE_KEYCLOAK_URL:-http://localhost}" + export VITE_KEYCLOAK_REALM="${VITE_KEYCLOAK_REALM:-default}" + export VITE_KEYCLOAK_CLIENT_ID="${VITE_KEYCLOAK_CLIENT_ID:-medcattrainer}" + export VITE_LOGOUT_REDIRECT_URI="${VITE_LOGOUT_REDIRECT_URI:-/}" fi # Check if template exists From fa3d01ba62f036bbfab2ca4b4070d5403f024c70 Mon Sep 17 00:00:00 2001 From: Alistair Hendrickson Date: Wed, 21 Jan 2026 13:13:55 +0000 Subject: [PATCH 09/22] build(medcat-trainer): Improve dockerfile layer caching (#304) --- medcat-trainer/webapp/Dockerfile | 38 ++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/medcat-trainer/webapp/Dockerfile b/medcat-trainer/webapp/Dockerfile index a643948e2..dc252d70b 100644 --- a/medcat-trainer/webapp/Dockerfile +++ b/medcat-trainer/webapp/Dockerfile @@ -23,26 +23,36 @@ RUN apt install -y nodejs && apt install -y npm RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" -# Copy project -WORKDIR /home -COPY ./ . - -# copy backup crontab and chmod scripts -RUN chmod u+x /home/scripts/entry.sh && chmod u+x /home/scripts/crontab && cp /home/scripts/crontab /etc/crontab -RUN chmod u+x /home/scripts/nginx-entrypoint.sh - -# Build frontend +# Copy dependency files first for better layer caching WORKDIR /home/frontend -RUN npm install && npm run build +COPY frontend/package.json frontend/package-lock.json ./ +RUN npm install -# Install requirements for backend -WORKDIR /home/ +# Install Python dependencies +WORKDIR /home +COPY requirements.txt ./ RUN pip install pip --upgrade RUN pip install --upgrade setuptools RUN pip install --no-cache-dir -r requirements.txt + +# Download spaCy models (only requires spaCy, not application code) ARG SPACY_MODELS="en_core_web_md" RUN for SPACY_MODEL in ${SPACY_MODELS}; do python -m spacy download ${SPACY_MODEL}; done +# Copy rest of project +WORKDIR /home +COPY ./ . + +# Build frontend +WORKDIR /home/frontend +RUN npm run build + +# copy backup crontab and chmod scripts +RUN chmod u+x /home/scripts/entry.sh && \ + chmod u+x /home/scripts/crontab && cp /home/scripts/crontab /etc/crontab && \ + chmod a+x /home/scripts/run.sh && \ + chmod a+x /home/scripts/run-bg-process.sh && \ + chmod u+x /home/scripts/nginx-entrypoint.sh + WORKDIR /home/api/ -RUN chmod a+x /home/scripts/run.sh -RUN chmod a+x /home/scripts/run-bg-process.sh + From 3eb6c3af7ceee88b1f553fd3daa751fd3ba69804 Mon Sep 17 00:00:00 2001 From: Mart Ratas Date: Wed, 21 Jan 2026 15:37:12 +0200 Subject: [PATCH 10/22] fix(medcat): CU-869bvptgq Relcat infer output for v2 (#303) * CU-869bvptgq: Simplify relations output somewhat * CU-869bvptgq: Add relation both entity start and end char positions * CU-869bvptgq: Add entity start / end positions to rel-cat * CU-869bvptgq: Add relevant changes to relation datasets * CU-869bvptgq: Minor linting change * CU-869bvptgq: Fix minor linting issue --- .../addons/relation_extraction/rel_cat.py | 29 +++++----- .../addons/relation_extraction/rel_dataset.py | 54 ++++++++++--------- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/medcat-v2/medcat/components/addons/relation_extraction/rel_cat.py b/medcat-v2/medcat/components/addons/relation_extraction/rel_cat.py index 37d91f253..2bc7dc0fa 100644 --- a/medcat-v2/medcat/components/addons/relation_extraction/rel_cat.py +++ b/medcat-v2/medcat/components/addons/relation_extraction/rel_cat.py @@ -212,6 +212,9 @@ def __init__(self, base_tokenizer: BaseTokenizer, def _init_data_paths(self): doc_cls = self.base_tokenizer.get_doc_class() doc_cls.register_addon_path('relations', def_val=[], force=True) + entity_cls = self.base_tokenizer.get_entity_class() + entity_cls.register_addon_path('start', def_val=None, force=True) + entity_cls.register_addon_path('end', def_val=None, force=True) def save(self, save_path: str = "./") -> None: self.component.save(save_path=save_path) @@ -833,27 +836,23 @@ def pipe(self, stream: Iterable[MutableDocument], *args, **kwargs relations: list = doc.get_addon_data( # type: ignore "relations") + out_rels = predict_rel_dataset.dataset[ + "output_relations"][rel_idx] relations.append( { "relation": rc_cnf.general.idx2labels[ predicted_label_id], "label_id": predicted_label_id, - "ent1_text": predict_rel_dataset.dataset[ - "output_relations"][rel_idx][ - 2], - "ent2_text": predict_rel_dataset.dataset[ - "output_relations"][rel_idx][ - 3], + "ent1_text": out_rels[2], + "ent2_text": out_rels[3], "confidence": float("{:.3f}".format( confidence[0])), - "start_ent_pos": "", - "end_ent_pos": "", - "start_entity_id": - predict_rel_dataset.dataset[ - "output_relations"][rel_idx][8], - "end_entity_id": - predict_rel_dataset.dataset[ - "output_relations"][rel_idx][9] + "start_ent1_char_pos": out_rels[18], + "end_ent1_char_pos": out_rels[19], + "start_ent2_char_pos": out_rels[20], + "end_ent2_char_pos": out_rels[21], + "start_entity_id": out_rels[8], + "end_entity_id": out_rels[9], }) pbar.update(len(token_ids)) pbar.close() @@ -901,6 +900,8 @@ def predict_text_with_anns(self, text: str, annotations: list[dict] entity = base_tokenizer.create_entity( doc, min(tkn_idx), max(tkn_idx) + 1, label=ann["value"]) entity.cui = ann["cui"] + entity.set_addon_data('start', ann['strat']) + entity.set_addon_data('end', ann['end']) doc.ner_ents.append(entity) doc = self(doc) diff --git a/medcat-v2/medcat/components/addons/relation_extraction/rel_dataset.py b/medcat-v2/medcat/components/addons/relation_extraction/rel_dataset.py index 149410538..6130f34f3 100644 --- a/medcat-v2/medcat/components/addons/relation_extraction/rel_dataset.py +++ b/medcat-v2/medcat/components/addons/relation_extraction/rel_dataset.py @@ -246,6 +246,9 @@ def _create_relation_validation(self, ent2_token: Union[str, MutableEntity] = tmp_doc_text[ ent2_start_char_pos: ent2_end_char_pos] + annotation_token_text = self.tokenizer.hf_tokenizers.convert_ids_to_tokens( + self.config.general.annotation_schema_tag_ids) + if (abs(ent2_start_char_pos - ent1_start_char_pos ) <= self.config.general.window_size and ent1_token != ent2_token): @@ -281,24 +284,20 @@ def _create_relation_validation(self, if is_spacy_doc or is_mct_export: tmp_doc_text = text - _pre_e1 = tmp_doc_text[0: (ent1_start_char_pos)] - _e1_s2 = ( - tmp_doc_text[ent1_end_char_pos: ent2_start_char_pos - 1]) - _e2_end = tmp_doc_text[ent2_end_char_pos + 1: text_length] - ent2_token_end_pos = (ent2_token_end_pos + 2) - - annotation_token_text = ( - self.tokenizer.hf_tokenizers.convert_ids_to_tokens( - self.config.general.annotation_schema_tag_ids)) + s1, e1, s2, e2 = annotation_token_text tmp_doc_text = ( - str(_pre_e1) + " " + - annotation_token_text[0] + " " + - str(ent1_token) + " " + - annotation_token_text[1] + " " + str(_e1_s2) + " " + - annotation_token_text[2] + " " + str(ent2_token) + " " + - annotation_token_text[3] + " " + str(_e2_end) - ) + tmp_doc_text[:ent2_end_char_pos] + + e2 + tmp_doc_text[ent2_end_char_pos:]) + tmp_doc_text = ( + tmp_doc_text[:ent2_start_char_pos] + + s2 + tmp_doc_text[ent2_start_char_pos:]) + tmp_doc_text = ( + tmp_doc_text[:ent1_end_char_pos] + + e1 + tmp_doc_text[ent1_end_char_pos:]) + tmp_doc_text = ( + tmp_doc_text[:ent1_start_char_pos] + + s1 + tmp_doc_text[ent1_start_char_pos:]) ann_tag_token_len = len(annotation_token_text[0]) @@ -309,11 +308,10 @@ def _create_relation_validation(self, _right_context_start_end_pos = ( # 8 for spces right_context_end_char_pos + (ann_tag_token_len * 4) + 8) - right_context_end_char_pos = ( - len(tmp_doc_text) + 1 if - right_context_end_char_pos >= len(tmp_doc_text) or - _right_context_start_end_pos >= len(tmp_doc_text) - else _right_context_start_end_pos) + right_context_end_char_pos = len(tmp_doc_text) if ( + right_context_end_char_pos >= len(tmp_doc_text) + or _right_context_start_end_pos >= len(tmp_doc_text) + ) else _right_context_start_end_pos # reassign the new text with added tags text_length = len(tmp_doc_text) @@ -363,16 +361,20 @@ def _create_relation_validation(self, ent2_token_start_pos += ent1_token_start_pos ent1_ent2_new_start = (ent1_token_start_pos, ent2_token_start_pos) - en1_start, en1_end = window_tokenizer_data[ - "offset_mapping"][ent1_token_start_pos] - en2_start, en2_end = window_tokenizer_data[ - "offset_mapping"][ent2_token_start_pos] + os_map = window_tokenizer_data["offset_mapping"] + s1_start, s1_end = os_map[ent1_token_start_pos] + e1_start, e1_end = os_map[_ent1_token_end_pos] + + s2_start, s2_end = os_map[ent2_token_start_pos] + e2_start, e2_end = os_map[_ent2_token_end_pos] return [window_tokenizer_data["input_ids"], ent1_ent2_new_start, ent1_token, ent2_token, "UNK", self.config.model.padding_idx, None, None, None, None, None, None, doc_id, "", - en1_start, en1_end, en2_start, en2_end] + s1_start, e1_end, s2_start, e2_end, + ent1_start_char_pos, ent1_end_char_pos, + ent2_start_char_pos, ent2_end_char_pos] return [] def _get_token_type_and_start_end( From 811b6144edd32d93e15223fcef4c1ef1300f96fd Mon Sep 17 00:00:00 2001 From: Alistair Hendrickson Date: Wed, 21 Jan 2026 13:41:24 +0000 Subject: [PATCH 11/22] build(medcat-trainer): Consolidate github flows into one. Use standardised github actions for docker (#302) * build(medcat-trainer): Use Docker actions for github CI builds * build(medcat-trainer): Create a docker image for every PR * build(medcat-trainer): Consolidate workflows into one file. Updating docker action --- .github/workflows/medcat-trainer_ci.yml | 89 +++++++++- .github/workflows/medcat-trainer_qa.yml | 153 ----------------- .github/workflows/medcat-trainer_release.yml | 163 ------------------- 3 files changed, 80 insertions(+), 325 deletions(-) delete mode 100644 .github/workflows/medcat-trainer_qa.yml delete mode 100644 .github/workflows/medcat-trainer_release.yml diff --git a/.github/workflows/medcat-trainer_ci.yml b/.github/workflows/medcat-trainer_ci.yml index f4037c760..ec2679437 100644 --- a/.github/workflows/medcat-trainer_ci.yml +++ b/.github/workflows/medcat-trainer_ci.yml @@ -1,11 +1,18 @@ name: medcat-trainer ci-build on: + push: + branches: [main] + tags: + - "medcat-trainer/v*.*.*" pull_request: paths: - "medcat-trainer/**" - ".github/workflows/medcat-trainer**" +permissions: + id-token: write + defaults: run: working-directory: ./medcat-trainer @@ -45,6 +52,23 @@ jobs: cd client python -m build + - name: Bump version for TestPyPI + if: github.ref == 'refs/heads/main' + run: sed -i "s/^version = .*/version = \"1.0.0.dev$(date +%s)\"/" client/pyproject.toml + + - name: Publish dev distribution to Test PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + if: github.ref == 'refs/heads/main' + with: + repository_url: https://test.pypi.org/legacy/ + packages_dir: medcat-trainer/client/dist + + - name: Publish production distribution to PyPI + if: startsWith(github.ref, 'refs/tags') && ! github.event.release.prerelease + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages_dir: medcat-trainer/client/dist + test-frontend: runs-on: ubuntu-latest steps: @@ -115,20 +139,67 @@ jobs: - test-client - test-frontend - test-backend + outputs: + image_version: ${{ steps.meta.outputs.version }} steps: - name: Checkout main uses: actions/checkout@v6 with: ref: ${{ github.ref }} - - name: Build - env: - IMAGE_TAG: ${{ env.RELEASE_VERSION }} - run: | - docker build -t cogstacksystems/medcat-trainer:dev-latest webapp/. + - name: Log in to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract metadata (tags, labels) for Docker MedCATtrainer + id: meta + uses: docker/metadata-action@v5 + with: + images: cogstacksystems/medcat-trainer + tags: | + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + # Include all default tags + type=schedule + type=ref,event=branch + type=ref,event=tag + type=ref,event=pr + type=sha + # Create version tag based on tag prefix + type=match,pattern=medcat-trainer/v(\d+\.\d+\.\d+),group=1 + flavor: latest=false + + - name: Build Docker MedCATtrainer image for testing + id: docker_build + uses: docker/build-push-action@v6 + with: + context: ./medcat-trainer/webapp/ + load: true # https://docs.docker.com/build/ci/github-actions/test-before-push/ + tags: cogstacksystems/medcat-trainer:test + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=cogstacksystems/medcat-trainer:buildcache + cache-to: type=registry,ref=cogstacksystems/medcat-trainer:buildcache,mode=max + - name: Run Django Tests - env: - IMAGE_TAG: ${{ env.RELEASE_VERSION }} run: | - # run tests - docker run --rm cogstacksystems/medcat-trainer:dev-latest python manage.py test + # run tests using the built image + docker run --rm cogstacksystems/medcat-trainer:test python manage.py test + + - name: Push Docker MedCATtrainer image + id: docker_build_push + uses: docker/build-push-action@v6 + with: + context: ./medcat-trainer/webapp/ + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=registry,ref=cogstacksystems/medcat-trainer:buildcache + cache-to: type=registry,ref=cogstacksystems/medcat-trainer:buildcache,mode=max + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/workflows/medcat-trainer_qa.yml b/.github/workflows/medcat-trainer_qa.yml deleted file mode 100644 index 55705f211..000000000 --- a/.github/workflows/medcat-trainer_qa.yml +++ /dev/null @@ -1,153 +0,0 @@ -name: medcat-trainer qa-build - -on: - push: - branches: [main] - -permissions: - id-token: write - -defaults: - run: - working-directory: ./medcat-trainer - -jobs: - # Test and build client library - test-client: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install requests pytest build - - - name: Install client package in development mode - run: | - cd client - pip install -e . - - - name: Run client tests - run: | - cd client - python -m pytest tests/ -v - - - name: Bump version for TestPyPI - run: sed -i "s/^version = .*/version = \"1.0.0.dev$(date +%s)\"/" client/pyproject.toml - - - name: Build client package - run: | - cd client - python -m build - - - name: Publish dev distribution to Test PyPI - uses: pypa/gh-action-pypi-publish@release/v1 - with: - repository_url: https://test.pypi.org/legacy/ - packages_dir: medcat-trainer/client/dist - - test-frontend: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "20" - - - name: Install dependencies - run: | - cd webapp/frontend - npm ci - - - name: Run frontend tests - run: | - cd webapp/frontend - npm run test:unit - - test-backend: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - cd webapp - pip install -r requirements.txt - - - name: Download spaCy model - run: | - python -m spacy download en_core_web_md - - - name: Run Django tests - env: - DB_ENGINE: sqlite3 - SECRET_KEY: test-secret-key - DEBUG: 1 - run: | - cd webapp/api - python manage.py test - - # Build and test webapp container - build-and-push: - runs-on: ubuntu-latest - needs: - - test-client - - test-frontend - - test-backend - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Build - run: | - docker build -t cogstacksystems/medcat-trainer:latest webapp/. - - - name: Run Django Tests - env: - IMAGE_TAG: latest - run: | - # run tests - docker run --rm cogstacksystems/medcat-trainer:latest python manage.py test - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Push to DockerHub - run: | - docker push cogstacksystems/medcat-trainer:latest diff --git a/.github/workflows/medcat-trainer_release.yml b/.github/workflows/medcat-trainer_release.yml deleted file mode 100644 index 4812a43ca..000000000 --- a/.github/workflows/medcat-trainer_release.yml +++ /dev/null @@ -1,163 +0,0 @@ -name: medcat-trainer release-build - -on: - push: - tags: - - "medcat-trainer/v*.*.*" - -permissions: - id-token: write - -defaults: - run: - working-directory: ./medcat-trainer - -jobs: - # Test, build and publish client library - test-and-publish-client: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Release Tag - # If GITHUB_REF=refs/tags/medcat-trainer/v0.1.2, this returns v0.1.2. Note it's including the "v" though it probably shouldnt - run: echo "RELEASE_VERSION=${GITHUB_REF##refs/*/}" >> $GITHUB_ENV - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.10" - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install requests pytest build twine - - - name: Install client package in development mode - run: | - cd client - pip install -e . - - - name: Run client tests - run: | - cd client - python -m pytest tests/ -v - - - name: Build client package - run: | - cd client - python -m build - - - name: Publish production distribution to PyPI - if: startsWith(github.ref, 'refs/tags') && ! github.event.release.prerelease - uses: pypa/gh-action-pypi-publish@release/v1 - with: - packages_dir: medcat-trainer/client/dist - - test-frontend: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Set up Node.js - uses: actions/setup-node@v6 - with: - node-version: "20" - - - name: Install dependencies - run: | - cd webapp/frontend - npm ci - - - name: Run frontend tests - run: | - cd webapp/frontend - npm run test:unit - - test-backend: - runs-on: ubuntu-latest - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Set up Python - uses: actions/setup-python@v6 - with: - python-version: "3.11" - - - name: Install system dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential - - - name: Install Rust - run: | - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y - echo "$HOME/.cargo/bin" >> $GITHUB_PATH - - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - cd webapp - pip install -r requirements.txt - - - name: Download spaCy model - run: | - python -m spacy download en_core_web_md - - - name: Run Django tests - env: - DB_ENGINE: sqlite3 - SECRET_KEY: test-secret-key - DEBUG: 1 - run: | - cd webapp/api - python manage.py test - - # Build and test webapp container - build-and-push: - runs-on: ubuntu-latest - needs: - - test-frontend - - test-backend - steps: - - name: Checkout main - uses: actions/checkout@v6 - with: - ref: "main" - - - name: Release Tag - # If GITHUB_REF=refs/tags/medcat-trainer/v0.1.2, this returns v0.1.2. Note it's including the "v" though it probably shouldnt - run: echo "RELEASE_VERSION=${GITHUB_REF##refs/*/}" >> $GITHUB_ENV - - name: Build - env: - IMAGE_TAG: ${{ env.RELEASE_VERSION }} - run: | - docker build -t cogstacksystems/medcat-trainer:$IMAGE_TAG -t cogstacksystems/medcat-trainer:latest webapp/. - - name: Run Django Tests - env: - IMAGE_TAG: ${{ env.RELEASE_VERSION }} - run: | - # run tests - docker run --rm cogstacksystems/medcat-trainer:$IMAGE_TAG python manage.py test - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - - name: Push to DockerHub - env: - IMAGE_TAG: ${{ env.RELEASE_VERSION }} - run: | - docker push cogstacksystems/medcat-trainer:$IMAGE_TAG - docker push cogstacksystems/medcat-trainer:latest From 6f97f6351db40f3a7d9d653c60277de82fda00e9 Mon Sep 17 00:00:00 2001 From: alhendrickson <159636032+alhendrickson@users.noreply.github.com.> Date: Wed, 21 Jan 2026 17:11:26 +0000 Subject: [PATCH 12/22] build(medcat-trainer): Allow client pypi publish to fail for release builds --- .github/workflows/medcat-trainer_ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/medcat-trainer_ci.yml b/.github/workflows/medcat-trainer_ci.yml index ec2679437..adca3e0dc 100644 --- a/.github/workflows/medcat-trainer_ci.yml +++ b/.github/workflows/medcat-trainer_ci.yml @@ -59,6 +59,7 @@ jobs: - name: Publish dev distribution to Test PyPI uses: pypa/gh-action-pypi-publish@release/v1 if: github.ref == 'refs/heads/main' + continue-on-error: true with: repository_url: https://test.pypi.org/legacy/ packages_dir: medcat-trainer/client/dist @@ -66,6 +67,7 @@ jobs: - name: Publish production distribution to PyPI if: startsWith(github.ref, 'refs/tags') && ! github.event.release.prerelease uses: pypa/gh-action-pypi-publish@release/v1 + continue-on-error: true with: packages_dir: medcat-trainer/client/dist From 6f61b8df735f91db7b0e9dc995f12b9056f6a7c3 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Thu, 22 Jan 2026 23:14:59 +0000 Subject: [PATCH 13/22] feat(medcat-trainer): interim commit for project admin page --- .../frontend/src/views/ProjectAdmin.vue | 1095 +++++++++++++---- 1 file changed, 862 insertions(+), 233 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue index 6e7c8b134..59f17f4c7 100644 --- a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue +++ b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue @@ -1,205 +1,256 @@ @@ -247,11 +301,13 @@

Confirm Reset

@@ -281,6 +337,7 @@ export default { projectToDelete: null, projectToReset: null, saving: false, + useBackupOption: false, formData: { name: '', description: '', @@ -387,11 +444,14 @@ export default { cuis_file: null, members: project.members ? project.members.map(m => typeof m === 'object' ? m.id : m) : [] } + // Show backup options if CDB or Vocab are set + this.useBackupOption = !!(project.concept_db || project.vocab) this.showCreateForm = true }, closeForm() { this.showCreateForm = false this.editingProject = null + this.useBackupOption = false this.resetForm() }, resetForm() { @@ -418,6 +478,7 @@ export default { cuis_file: null, members: [] } + this.useBackupOption = false }, handleCuiFileChange(event) { const file = event.target.files[0] @@ -430,6 +491,16 @@ export default { try { const formDataToSend = new FormData() + // If not using backup option, clear CDB and Vocab + if (!this.useBackupOption) { + this.formData.concept_db = null + this.formData.vocab = null + } + + // CDB Search Filter is hidden - will use ModelPack by default + // Clear it so backend uses ModelPack + this.formData.cdb_search_filter = [] + // Add all form fields to FormData Object.keys(this.formData).forEach(key => { if (key === 'cuis_file' && this.formData[key]) { @@ -517,157 +588,715 @@ export default { 'D': 'Discontinued' } return texts[status] || status + }, + getStatusIcon(status) { + const icons = { + 'A': 'spinner', + 'C': 'check-circle', + 'D': 'times-circle' + } + return icons[status] || 'circle' + }, + getDatasetName(datasetId) { + const dataset = this.datasets.find(ds => ds.id === datasetId) + return dataset ? dataset.name : 'N/A' } } } From b1cfe08b3fa0d3361a3d409832fc5b428fcb3e57 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Fri, 23 Jan 2026 00:05:32 +0000 Subject: [PATCH 14/22] feat(medcat-trainer): added same picker from demo page to admin project page, more options on the page --- .../frontend/src/views/ProjectAdmin.vue | 263 +++++++++++++++++- 1 file changed, 253 insertions(+), 10 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue index 59f17f4c7..2432bb4d4 100644 --- a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue +++ b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue @@ -160,7 +160,7 @@
- @@ -168,7 +168,7 @@
@@ -188,6 +188,21 @@
+
+
+ +
+
+
+
+ + + URL of the remote MedCAT service API (e.g., http://medcat-service:8000). Note: interim model training is not supported for remote model service projects. +
+
@@ -238,20 +253,60 @@
-
+

Concept Filtering

-
-
- - +
+ + +
+ +
+
+
+ Please select a Model Pack or enable backup option with Concept DB. +
+
-
+
JSON file containing array of CUI strings
+ +
+ + {{ item.cui }} - {{ item.name }} + + +
+ +
@@ -317,11 +372,13 @@ @@ -1033,6 +1169,12 @@ export default { box-shadow: 0 0 0 2px rgba(0, 114, 206, 0.1); } + &:disabled { + background-color: #f8f9fa; + cursor: not-allowed; + opacity: 0.6; + } + &::placeholder { color: var(--color-text); opacity: 0.5; @@ -1122,6 +1264,96 @@ export default { gap: 8px; } + // Concept Filtering Styles (from Demo.vue) + .cui-filter-controls { + margin: -4px 0 8px 0; + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + } + + .cui-filter-checkbox { + display: flex; + align-items: center; + gap: 6px; + margin: 0; + font-size: 0.9rem; + cursor: pointer; + font-weight: 400; + color: var(--color-text); + } + + .cui-filter-paste-toggle { + padding: 0; + font-size: 0.85rem; + } + + .cui-filter-row { + display: flex; + gap: 16px; + align-items: flex-start; + margin: 6px 0 10px 0; + } + + .cui-filter-picker { + flex: 0 0 50%; + max-width: 50%; + } + + .cui-file-picker { + flex: 0 0 calc(50% - 16px); + max-width: calc(50% - 16px); + + label { + display: block; + margin-bottom: 4px; + font-weight: 500; + color: var(--color-heading); + font-size: 0.9rem; + } + } + + .cui-pill-row { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin: 6px 0 10px 0; + } + + .cui-pill { + display: inline-flex; + align-items: center; + gap: 8px; + border: 1px solid rgba(0, 0, 0, 0.15); + background: rgba(13, 110, 253, 0.08); + color: #0b5ed7; + border-radius: 999px; + padding: 4px 10px; + font-size: 0.75rem; + line-height: 1; + } + + .cui-pill-text { + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace; + } + + .cui-pill-remove { + border: none; + background: transparent; + color: inherit; + padding: 0; + cursor: pointer; + font-size: 16px; + line-height: 1; + opacity: 0.7; + transition: opacity 0.2s ease; + + &:hover { + opacity: 1; + } + } + .form-actions { display: flex; justify-content: flex-end; @@ -1298,5 +1530,16 @@ export default { .checkbox-grid { grid-template-columns: 1fr; } + + .cui-filter-row { + flex-direction: column; + gap: 12px; + + .cui-filter-picker, + .cui-file-picker { + flex: 1 1 100%; + max-width: 100%; + } + } } From 2c29bc5cdfe57766a5898ad86ae69a959fa80139 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Fri, 23 Jan 2026 17:07:27 +0000 Subject: [PATCH 15/22] feat(medcat-trainer): added clone feature to each project, can rename the clone. --- .../frontend/src/views/ProjectAdmin.vue | 160 +++++++++++++++--- 1 file changed, 136 insertions(+), 24 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue index 2432bb4d4..f125a4871 100644 --- a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue +++ b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue @@ -56,10 +56,10 @@ + + + + + +
@@ -393,6 +419,8 @@ export default { editingProject: null, projectToDelete: null, projectToReset: null, + projectToClone: null, + cloneName: '', saving: false, useBackupOption: false, selectedCuiFilterConcepts: [], @@ -478,8 +506,9 @@ export default { const response = await this.$http.get('/api/users/') this.users = response.data.results || response.data }, - selectProject(project) { - this.editProject(project) + selectProject(event, { item }) { + // v-data-table click:row passes (event, { item }) + this.editProject(item) }, editProject(project) { this.editingProject = project @@ -626,12 +655,24 @@ export default { if (key === 'cuis_file' && this.formData[key]) { formDataToSend.append(key, this.formData[key]) } else if (key === 'cdb_search_filter' || key === 'members') { - // Handle arrays - this.formData[key].forEach(val => { - formDataToSend.append(key, val) - }) + // Handle arrays - send empty array as empty (don't append anything for empty arrays) + if (Array.isArray(this.formData[key]) && this.formData[key].length > 0) { + this.formData[key].forEach(val => { + if (val !== null && val !== undefined) { + formDataToSend.append(key, val) + } + }) + } + // For empty arrays, don't send anything (backend will handle as empty) } else if (this.formData[key] !== null && this.formData[key] !== undefined) { - formDataToSend.append(key, this.formData[key]) + // Convert null-like values to empty strings for optional fields + const value = this.formData[key] + // For boolean false, send as string 'false' + if (typeof value === 'boolean') { + formDataToSend.append(key, value.toString()) + } else { + formDataToSend.append(key, value) + } } }) @@ -652,17 +693,72 @@ export default { ) } + // If we get here, the request was successful this.$toast?.success(`Project ${this.editingProject ? 'updated' : 'created'} successfully`) this.closeForm() await this.fetchProjects() } catch (error) { console.error('Error saving project:', error) - const errorMsg = error.response?.data?.error || error.response?.data?.message || 'Failed to save project' + console.error('Error response:', error.response?.data) + let errorMsg = 'Failed to save project' + if (error.response?.data) { + if (typeof error.response.data === 'string') { + errorMsg = error.response.data + } else if (error.response.data.error) { + errorMsg = error.response.data.error + } else if (error.response.data.message) { + errorMsg = error.response.data.message + } else if (typeof error.response.data === 'object') { + // Try to format validation errors + const errors = Object.entries(error.response.data) + .map(([field, messages]) => { + const msg = Array.isArray(messages) ? messages.join(', ') : String(messages) + return `${field}: ${msg}` + }) + .join('; ') + errorMsg = errors || errorMsg + } + } this.$toast?.error(errorMsg) } finally { this.saving = false } }, + cloneProject(project) { + this.projectToClone = project + this.cloneName = `${project.name} (Clone)` + // Focus the input after modal opens + this.$nextTick(() => { + if (this.$refs.cloneNameInput) { + this.$refs.cloneNameInput.focus() + this.$refs.cloneNameInput.select() + } + }) + }, + closeCloneModal() { + this.projectToClone = null + this.cloneName = '' + }, + async performClone() { + if (!this.cloneName || this.cloneName.trim() === '') { + this.$toast?.error('Please enter a project name') + return + } + try { + const response = await this.$http.post( + `/api/project-admin/projects/${this.projectToClone.id}/clone/`, + { name: this.cloneName.trim() }, + { headers: { 'Content-Type': 'application/json' } } + ) + this.$toast?.success(`Project "${this.cloneName}" cloned successfully`) + this.closeCloneModal() + await this.fetchProjects() + } catch (error) { + console.error('Error cloning project:', error) + const errorMsg = error.response?.data?.error || 'Failed to clone project' + this.$toast?.error(errorMsg) + } + }, confirmDelete(project) { this.projectToDelete = project }, @@ -752,7 +848,6 @@ export default { } .project-admin-header { - margin-bottom: 40px; padding-bottom: 20px; border-bottom: 2px solid var(--color-border); @@ -929,12 +1024,12 @@ export default { transform: translateY(-1px); } - &.btn-edit { - color: $primary; - border-color: $primary; + &.btn-clone { + color: $success; + border-color: $success; &:hover { - background-color: $primary; + background-color: $success; color: white; } } @@ -1044,8 +1139,8 @@ export default { box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); display: flex; flex-direction: column; - height: calc(100vh - 200px); - min-height: 600px; + height: calc(100vh - 180px); + max-height: calc(100vh - 180px); .form-header { padding: 12px 20px; @@ -1089,20 +1184,36 @@ export default { .form-content { flex: 1; - overflow-y: auto; + overflow: hidden; padding: 16px 20px; + display: flex; + flex-direction: column; + min-height: 0; } .project-form { padding: 0; max-width: 1400px; margin: 0 auto; + flex: 1; + display: flex; + flex-direction: column; + min-height: 0; + } + + .form-sections-wrapper { + flex: 1; + overflow-y: auto; + overflow-x: hidden; + min-height: 0; + padding-bottom: 20px; } .form-section { margin-bottom: 20px; padding-bottom: 16px; border-bottom: 1px solid var(--color-border); + flex-shrink: 0; &:last-child { border-bottom: none; @@ -1358,9 +1469,10 @@ export default { display: flex; justify-content: flex-end; gap: 10px; - margin-top: 20px; + margin-top: auto; padding-top: 16px; border-top: 1px solid var(--color-border); + flex-shrink: 0; .btn { padding: 8px 20px; @@ -1479,7 +1591,7 @@ export default { .project-form-section { height: calc(100vh - 150px); - min-height: 500px; + max-height: calc(100vh - 150px); .form-header { padding: 10px 16px; From 364338b8d48d45a140ec64436fb69ab830d95415 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Fri, 23 Jan 2026 17:08:05 +0000 Subject: [PATCH 16/22] feat(medcat-trainer): added clone feature to each project, can rename the clone. --- medcat-trainer/webapp/api/api/views.py | 112 ++++++++++++++++++++----- medcat-trainer/webapp/api/core/urls.py | 1 + 2 files changed, 94 insertions(+), 19 deletions(-) diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py index f7e28e699..74beab16f 100644 --- a/medcat-trainer/webapp/api/api/views.py +++ b/medcat-trainer/webapp/api/api/views.py @@ -1121,30 +1121,50 @@ def project_admin_detail(request, project_id): # Handle both JSON and FormData data = request.data.copy() if hasattr(request.data, 'copy') else dict(request.data) - # Convert many-to-many fields from lists to proper format - if 'cdb_search_filter' in data and isinstance(data['cdb_search_filter'], list): - # Already a list, keep it - pass - elif 'cdb_search_filter' in request.data: - # FormData sends as multiple values with same key - data['cdb_search_filter'] = request.data.getlist('cdb_search_filter') + # Extract many-to-many fields before serializer validation + cdb_search_filter_ids = [] + if 'cdb_search_filter' in request.data: + if isinstance(request.data.get('cdb_search_filter'), list): + cdb_search_filter_ids = request.data['cdb_search_filter'] + else: + # FormData sends as multiple values with same key + cdb_search_filter_ids = request.data.getlist('cdb_search_filter') + # Remove from data dict so serializer doesn't try to validate it + data.pop('cdb_search_filter', None) + members_ids = [] if 'members' in request.data: if isinstance(request.data.get('members'), list): - data['members'] = request.data['members'] + members_ids = request.data['members'] else: - data['members'] = request.data.getlist('members') + members_ids = request.data.getlist('members') + # Remove from data dict so serializer doesn't try to validate it + data.pop('members', None) + + # Convert string booleans to actual booleans + boolean_fields = ['project_locked', 'annotation_classification', 'require_entity_validation', + 'train_model_on_submit', 'add_new_entities', 'restrict_concept_lookup', + 'terminate_available', 'irrelevant_available', 'enable_entity_annotation_comments', + 'use_model_service'] + for field in boolean_fields: + if field in data: + if isinstance(data[field], str): + data[field] = data[field].lower() in ('true', '1', 'yes', 'on') serializer = ProjectAnnotateEntitiesSerializer(project, data=data, partial=True) if serializer.is_valid(): - project = serializer.save() - # Handle many-to-many fields manually if needed - if 'cdb_search_filter' in data: - project.cdb_search_filter.set(data['cdb_search_filter']) - if 'members' in data: - project.members.set(data['members']) - return Response(ProjectAnnotateEntitiesSerializer(project).data) - return Response(serializer.errors, status=400) + try: + project = serializer.save() + # Handle many-to-many fields manually after saving + project.cdb_search_filter.set(cdb_search_filter_ids) + project.members.set(members_ids) + return Response(ProjectAnnotateEntitiesSerializer(project).data) + except Exception as e: + logger.error(f"Error saving project {project_id}: {e}", exc_info=e) + return Response({'error': f'Failed to save project: {str(e)}'}, status=400) + else: + logger.warning(f"Validation errors for project {project_id}: {serializer.errors}") + return Response(serializer.errors, status=400) elif request.method == 'DELETE': project.delete() @@ -1165,13 +1185,21 @@ def project_admin_create(request): if isinstance(request.data.get('cdb_search_filter'), list): data['cdb_search_filter'] = request.data['cdb_search_filter'] else: - data['cdb_search_filter'] = request.data.getlist('cdb_search_filter') + cdb_filter_list = request.data.getlist('cdb_search_filter') + # Only include if list has items, otherwise set to empty list + data['cdb_search_filter'] = cdb_filter_list if cdb_filter_list else [] + else: + data['cdb_search_filter'] = [] if 'members' in request.data: if isinstance(request.data.get('members'), list): data['members'] = request.data['members'] else: - data['members'] = request.data.getlist('members') + members_list = request.data.getlist('members') + # Only include if list has items + data['members'] = members_list if members_list else [] + else: + data['members'] = [] serializer = ProjectAnnotateEntitiesSerializer(data=data) if serializer.is_valid(): @@ -1188,6 +1216,52 @@ def project_admin_create(request): return Response(serializer.errors, status=400) +@api_view(http_method_names=['POST']) +@permission_classes([permissions.IsAuthenticated]) +def project_admin_clone(request, project_id): + """ + Clone a project (user must be authenticated and have permission). + """ + import copy + try: + project = ProjectAnnotateEntities.objects.get(id=project_id) + except ProjectAnnotateEntities.DoesNotExist: + return Response({'error': 'Project not found'}, status=404) + + # Check if user is project admin + from .permissions import is_project_admin + if not is_project_admin(request.user, project): + return Response({'error': 'You do not have permission to clone this project'}, status=403) + + try: + # Get custom name from request, or use default + custom_name = request.data.get('name', None) if hasattr(request.data, 'get') else None + if not custom_name: + custom_name = f'{project.name} (Clone)' + + # Create a copy of the project + project_copy = copy.copy(project) + project_copy.id = None + project_copy.pk = None + project_copy.name = custom_name + project_copy.save() + + # Copy many-to-many fields + for m in project.members.all(): + project_copy.members.add(m) + for c in project.cdb_search_filter.all(): + project_copy.cdb_search_filter.add(c) + for t in project.tasks.all(): + project_copy.tasks.add(t) + + project_copy.save() + serializer = ProjectAnnotateEntitiesSerializer(project_copy) + return Response(serializer.data, status=201) + except Exception as e: + logger.error(f"Failed to clone project: {e}", exc_info=e) + return Response({'error': f'Failed to clone project: {str(e)}'}, status=500) + + @api_view(http_method_names=['POST']) @permission_classes([permissions.IsAuthenticated]) def project_admin_reset(request, project_id): diff --git a/medcat-trainer/webapp/api/core/urls.py b/medcat-trainer/webapp/api/core/urls.py index 80f774f9b..2934895d9 100644 --- a/medcat-trainer/webapp/api/core/urls.py +++ b/medcat-trainer/webapp/api/core/urls.py @@ -63,6 +63,7 @@ path('api/cuis-to-concepts/', api.views.cuis_to_concepts), path('api/project-admin/projects/', api.views.project_admin_projects), path('api/project-admin/projects//', api.views.project_admin_detail), + path('api/project-admin/projects//clone/', api.views.project_admin_clone), path('api/project-admin/projects//reset/', api.views.project_admin_reset), path('api/project-admin/projects/create/', api.views.project_admin_create), path('reset_password/', api.views.ResetPasswordView.as_view(), name='reset_password'), From bf018705c9e8e6983b249a24c784bae38f387f2c Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Mon, 26 Jan 2026 17:31:17 +0000 Subject: [PATCH 17/22] feat(medcat-trainer): datasets, users, and model packs available now from the 'setting' cog' --- medcat-trainer/webapp/frontend/src/App.vue | 38 +- .../webapp/frontend/src/router/index.ts | 31 +- .../webapp/frontend/src/views/Home.vue | 4 - .../frontend/src/views/ProjectAdmin.vue | 995 ++++++++++++++++-- 4 files changed, 998 insertions(+), 70 deletions(-) diff --git a/medcat-trainer/webapp/frontend/src/App.vue b/medcat-trainer/webapp/frontend/src/App.vue index 52e56dd8e..2c2de5270 100644 --- a/medcat-trainer/webapp/frontend/src/App.vue +++ b/medcat-trainer/webapp/frontend/src/App.vue @@ -17,7 +17,6 @@
Projects - Project Admin Metrics Concepts Try Model @@ -28,6 +27,9 @@
+ + +
- - - Manage Projects -
diff --git a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue index f125a4871..35240cd4e 100644 --- a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue +++ b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue @@ -6,19 +6,67 @@

Project Administration

Manage your annotation projects

- +
+ + + + +
+ +
+ + + + +
+
- Loading projects... + Loading...
+ +
@@ -159,8 +207,8 @@

Model Configuration

- - @@ -392,6 +440,308 @@
+
+ + + +
+
+
+

Model Packs ({{ modelPacks.length }})

+
+
+ + + + + +
+
+

No Model Packs

+

Add a model pack to get started.

+
+
+ + +
+
+ +

{{ editingModelPack ? 'Edit Model Pack' : 'Add Model Pack' }}

+
+
+
+
+
+
+ + +
+
+ + + Upload a .zip file containing the model pack +
+
+
+
+ + +
+
+ + +
+
+
+
+ + +
+
+
+
+
+ + + +
+
+
+

Datasets ({{ datasets.length }})

+
+
+ + + +
+
+

No Datasets

+

Add a dataset to get started.

+
+
+ + +
+
+ +

{{ editingDataset ? 'Edit Dataset' : 'Add Dataset' }}

+
+
+
+
+
+
+ + +
+
+ + +
+
+
+
+ + + Upload a .csv or .xlsx file. Must contain 'name' and 'text' columns. +
+
+
+
+ + +
+
+
+
+
+ + + +
+
+
+

Users ({{ users.length }})

+
+
+ + + + + +
+
+

No Users

+

Add a user to get started.

+
+
+ + +
+
+ +

{{ editingUser ? 'Edit User' : 'Add User' }}

+
+
+
+
+
+
+ + +
+
+ + +
+
+
+
+ + + Note: Password cannot be set via API. Users should set their password through password reset or Django admin. +
+
+
+
+
+ +
+
+ +
+
+
+
+
+ + +
+
+
+
+
+ + + + + + + + + + + + +
@@ -408,6 +758,7 @@ export default { }, data() { return { + activeTab: 'projects', loading: true, projects: [], datasets: [], @@ -426,6 +777,35 @@ export default { selectedCuiFilterConcepts: [], includeSubConcepts: false, showCuiFilterTextarea: false, + // Model Pack management + showModelPackForm: false, + editingModelPack: null, + modelPackToDelete: null, + modelPackForm: { + name: '', + model_pack: null, + concept_db: null, + vocab: null + }, + // Dataset management + showDatasetForm: false, + editingDataset: null, + datasetToDelete: null, + datasetForm: { + name: '', + description: '', + original_file: null + }, + // User management + showUserForm: false, + editingUser: null, + userForm: { + username: '', + email: '', + password: '', + is_staff: false, + is_superuser: false + }, formData: { name: '', description: '', @@ -457,6 +837,24 @@ export default { { title: 'Status', value: 'status' }, { title: 'Dataset', value: 'dataset' }, { title: 'Actions', value: 'actions', sortable: false } + ], + modelPackHeaders: [ + { title: 'Name', value: 'name' }, + { title: 'Concept DB', value: 'concept_db' }, + { title: 'Vocabulary', value: 'vocab' }, + { title: 'Actions', value: 'actions', sortable: false } + ], + datasetHeaders: [ + { title: 'Name', value: 'name' }, + { title: 'Description', value: 'description' }, + { title: 'Actions', value: 'actions', sortable: false } + ], + userHeaders: [ + { title: 'Username', value: 'username' }, + { title: 'Email', value: 'email' }, + { title: 'Staff', value: 'is_staff' }, + { title: 'Admin', value: 'is_superuser' }, + { title: 'Actions', value: 'actions', sortable: false } ] } }, @@ -556,6 +954,12 @@ export default { this.showCuiFilterTextarea = false this.resetForm() }, + closeAllForms() { + this.closeForm() + this.closeModelPackForm() + this.closeDatasetForm() + this.closeUserForm() + }, resetForm() { this.formData = { name: '', @@ -816,7 +1220,235 @@ export default { getDatasetName(datasetId) { const dataset = this.datasets.find(ds => ds.id === datasetId) return dataset ? dataset.name : 'N/A' - } + }, + getConceptDbName(conceptDbId) { + if (!conceptDbId) return 'N/A' + const cdb = this.conceptDbs.find(c => c.id === (typeof conceptDbId === 'object' ? conceptDbId.id : conceptDbId)) + return cdb ? cdb.name : 'N/A' + }, + getVocabName(vocabId) { + if (!vocabId) return 'N/A' + const vocab = this.vocabs.find(v => v.id === (typeof vocabId === 'object' ? vocabId.id : vocabId)) + return vocab ? vocab.name : 'N/A' + }, + // Model Pack methods + selectModelPack(event, { item }) { + this.editModelPack(item) + }, + editModelPack(modelPack) { + this.editingModelPack = modelPack + this.modelPackForm = { + name: modelPack.name || '', + model_pack: null, + concept_db: modelPack.concept_db || null, + vocab: modelPack.vocab || null + } + this.showModelPackForm = true + }, + closeModelPackForm() { + this.showModelPackForm = false + this.editingModelPack = null + this.modelPackForm = { + name: '', + model_pack: null, + concept_db: null, + vocab: null + } + }, + handleModelPackFileChange(event) { + const file = event.target.files[0] + if (file) { + this.modelPackForm.model_pack = file + } + }, + async saveModelPack() { + this.saving = true + try { + const formData = new FormData() + formData.append('name', this.modelPackForm.name) + if (this.modelPackForm.model_pack) { + formData.append('model_pack', this.modelPackForm.model_pack) + } + if (this.modelPackForm.concept_db) { + formData.append('concept_db', this.modelPackForm.concept_db) + } + if (this.modelPackForm.vocab) { + formData.append('vocab', this.modelPackForm.vocab) + } + + if (this.editingModelPack) { + await this.$http.put( + `/api/modelpacks/${this.editingModelPack.id}/`, + formData, + { headers: { 'Content-Type': 'multipart/form-data' } } + ) + } else { + await this.$http.post( + '/api/modelpacks/', + formData, + { headers: { 'Content-Type': 'multipart/form-data' } } + ) + } + + this.$toast?.success(`Model Pack ${this.editingModelPack ? 'updated' : 'created'} successfully`) + this.closeModelPackForm() + await this.fetchModelPacks() + } catch (error) { + console.error('Error saving model pack:', error) + const errorMsg = error.response?.data?.message || error.response?.data?.detail || 'Failed to save model pack' + this.$toast?.error(errorMsg) + } finally { + this.saving = false + } + }, + confirmDeleteModelPack(modelPack) { + this.modelPackToDelete = modelPack + }, + async deleteModelPack() { + try { + await this.$http.delete(`/api/modelpacks/${this.modelPackToDelete.id}/`) + this.$toast?.success('Model Pack deleted successfully') + this.modelPackToDelete = null + await this.fetchModelPacks() + } catch (error) { + console.error('Error deleting model pack:', error) + this.$toast?.error('Failed to delete model pack') + } + }, + // Dataset methods + selectDataset(event, { item }) { + this.editDataset(item) + }, + editDataset(dataset) { + this.editingDataset = dataset + this.datasetForm = { + name: dataset.name || '', + description: dataset.description || '', + original_file: null + } + this.showDatasetForm = true + }, + closeDatasetForm() { + this.showDatasetForm = false + this.editingDataset = null + this.datasetForm = { + name: '', + description: '', + original_file: null + } + }, + handleDatasetFileChange(event) { + const file = event.target.files[0] + if (file) { + this.datasetForm.original_file = file + } + }, + async saveDataset() { + this.saving = true + try { + const formData = new FormData() + formData.append('name', this.datasetForm.name) + formData.append('description', this.datasetForm.description || '') + if (this.datasetForm.original_file) { + formData.append('original_file', this.datasetForm.original_file) + } + + if (this.editingDataset) { + await this.$http.put( + `/api/datasets/${this.editingDataset.id}/`, + formData, + { headers: { 'Content-Type': 'multipart/form-data' } } + ) + } else { + await this.$http.post( + '/api/datasets/', + formData, + { headers: { 'Content-Type': 'multipart/form-data' } } + ) + } + + this.$toast?.success(`Dataset ${this.editingDataset ? 'updated' : 'created'} successfully`) + this.closeDatasetForm() + await this.fetchDatasets() + } catch (error) { + console.error('Error saving dataset:', error) + const errorMsg = error.response?.data?.message || error.response?.data?.detail || 'Failed to save dataset' + this.$toast?.error(errorMsg) + } finally { + this.saving = false + } + }, + confirmDeleteDataset(dataset) { + this.datasetToDelete = dataset + }, + async deleteDataset() { + try { + await this.$http.delete(`/api/datasets/${this.datasetToDelete.id}/`) + this.$toast?.success('Dataset deleted successfully') + this.datasetToDelete = null + await this.fetchDatasets() + } catch (error) { + console.error('Error deleting dataset:', error) + this.$toast?.error('Failed to delete dataset') + } + }, + // User methods + selectUser(event, { item }) { + this.editUser(item) + }, + editUser(user) { + this.editingUser = user + this.userForm = { + username: user.username || '', + email: user.email || '', + password: '', + is_staff: user.is_staff || false, + is_superuser: user.is_superuser || false + } + this.showUserForm = true + }, + closeUserForm() { + this.showUserForm = false + this.editingUser = null + this.userForm = { + username: '', + email: '', + password: '', + is_staff: false, + is_superuser: false + } + }, + async saveUser() { + this.saving = true + try { + const data = { + username: this.userForm.username, + email: this.userForm.email || '', + is_staff: this.userForm.is_staff, + is_superuser: this.userForm.is_superuser + } + + // Note: Password is not included in UserSerializer, so it cannot be set via API + // User creation/update will need to be done through Django admin or a custom endpoint + if (this.editingUser) { + await this.$http.put(`/api/users/${this.editingUser.id}/`, data) + } else { + // For new users, password cannot be set via this API + // Users should be created through Django admin or password reset flow + await this.$http.post('/api/users/', data) + } + + this.$toast?.success(`User ${this.editingUser ? 'updated' : 'created'} successfully`) + this.closeUserForm() + await this.fetchUsers() + } catch (error) { + console.error('Error saving user:', error) + const errorMsg = error.response?.data?.message || error.response?.data?.detail || 'Failed to save user' + this.$toast?.error(errorMsg) + } finally { + this.saving = false + } + }, }, watch: { 'formData.cuis'(newVal) { @@ -858,6 +1490,11 @@ export default { gap: 20px; } + .header-actions { + display: flex; + gap: 10px; + } + .header-text { flex: 1; @@ -1134,7 +1771,7 @@ export default { // Project Form Section (Full Screen) .project-form-section { background: white; - border-radius: 8px; + border-radius: 12px; padding: 0; box-shadow: 0 2px 8px rgba(0, 0, 0, 0.08); display: flex; @@ -1150,6 +1787,7 @@ export default { display: flex; align-items: center; gap: 12px; + border-radius: 12px 12px 0 0; .btn-back { background: rgba(255, 255, 255, 0.2); @@ -1206,56 +1844,68 @@ export default { overflow-y: auto; overflow-x: hidden; min-height: 0; - padding-bottom: 20px; + padding: 20px; + background: #f8f9fa; } .form-section { - margin-bottom: 20px; - padding-bottom: 16px; - border-bottom: 1px solid var(--color-border); + margin-bottom: 24px; + padding: 20px; + background: white; + border: 1px solid #e0e0e0; + border-radius: 12px; + box-shadow: 0 1px 2px rgba(0, 0, 0, 0.05); flex-shrink: 0; &:last-child { - border-bottom: none; margin-bottom: 0; - padding-bottom: 0; } h4 { - margin-bottom: 12px; + margin-bottom: 16px; + margin-top: 0; color: var(--color-heading); - font-size: 1rem; + font-size: 1.05rem; font-weight: 600; + padding-bottom: 12px; + border-bottom: 1px solid #f0f0f0; } &.form-section-horizontal { .form-row { display: flex; - gap: 16px; - align-items: flex-start; + gap: 20px; + align-items: flex-end; flex-wrap: wrap; .form-group-inline { flex: 1; min-width: 200px; - margin-bottom: 12px; + margin-bottom: 0; + } + + // Align checkboxes with inputs + .checkbox-group.form-group-inline { + align-self: flex-end; + margin-bottom: 0; + padding-bottom: 0; } } .backup-options { - margin-top: 12px; - padding-top: 12px; - border-top: 1px solid var(--color-border); + margin-top: 16px; + padding-top: 16px; + border-top: 1px solid #f0f0f0; } } } .form-group { - margin-bottom: 12px; + margin-bottom: 16px; label { display: block; - margin-bottom: 4px; + margin-bottom: 6px; font-weight: 500; color: var(--color-heading); font-size: 0.9rem; @@ -1267,55 +1917,88 @@ export default { .form-control { width: 100%; - padding: 6px 10px; - border: 1px solid var(--color-border); - border-radius: 4px; + padding: 8px 12px; + border: 1px solid #d0d0d0; + border-radius: 8px; font-size: 0.9rem; transition: all 0.2s ease; background: white; + box-shadow: inset 0 1px 2px rgba(0, 0, 0, 0.02); + + &:hover { + border-color: #b0b0b0; + } &:focus { outline: none; border-color: $primary; - box-shadow: 0 0 0 2px rgba(0, 114, 206, 0.1); + box-shadow: 0 0 0 3px rgba(0, 114, 206, 0.1), inset 0 1px 2px rgba(0, 0, 0, 0.02); } &:disabled { - background-color: #f8f9fa; + background-color: #f5f5f5; + border-color: #e0e0e0; cursor: not-allowed; - opacity: 0.6; + opacity: 0.7; } &::placeholder { - color: var(--color-text); - opacity: 0.5; + color: #999; + opacity: 0.7; } } + // Ensure select elements have consistent styling + select.form-control { + cursor: pointer; + background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 16 16'%3e%3cpath fill='none' stroke='%23343a40' stroke-linecap='round' stroke-linejoin='round' stroke-width='2' d='M2 5l6 6 6-6'/%3e%3c/svg%3e"); + background-repeat: no-repeat; + background-position: right 8px center; + background-size: 16px 12px; + padding-right: 32px; + } + textarea.form-control { resize: vertical; - min-height: 60px; + min-height: 80px; + font-family: inherit; + line-height: 1.5; + border-radius: 8px; } select[multiple].form-control { min-height: 120px; + padding: 8px; + border-radius: 8px; + option { + padding: 6px 8px; + } } .file-input { padding: 8px; cursor: pointer; + border: 1px solid #d0d0d0; + border-radius: 8px; + background: white; + + &:hover { + border-color: #b0b0b0; + } &::file-selector-button { - padding: 8px 16px; + padding: 6px 14px; margin-right: 12px; - border: 1px solid var(--color-border); - border-radius: 4px; + border: 1px solid #d0d0d0; + border-radius: 6px; background: #f8f9fa; cursor: pointer; transition: all 0.2s ease; + font-size: 0.85rem; &:hover { background: #e9ecef; + border-color: #b0b0b0; } } } @@ -1330,16 +2013,17 @@ export default { } .checkbox-group { - margin-bottom: 8px; + margin-bottom: 12px; .checkbox-label { display: flex; align-items: center; - gap: 8px; + gap: 10px; cursor: pointer; - padding: 4px 0; - transition: opacity 0.2s ease; + padding: 8px 0; + transition: all 0.2s ease; margin-bottom: 0; + min-height: 36px; // Match input height for alignment &:hover { opacity: 0.8; @@ -1347,25 +2031,28 @@ export default { .checkbox-input { margin: 0; - width: 16px; - height: 16px; + width: 18px; + height: 18px; cursor: pointer; accent-color: $primary; flex-shrink: 0; + border: 1px solid #d0d0d0; + border-radius: 3px; } .checkbox-text { flex: 1; font-weight: 400; color: var(--color-text); - font-size: 0.85rem; - line-height: 1.3; + font-size: 0.9rem; + line-height: 1.4; } } &.form-group-inline { margin-bottom: 0; - align-self: center; + align-self: flex-end; + padding-bottom: 0; } } @@ -1377,7 +2064,11 @@ export default { // Concept Filtering Styles (from Demo.vue) .cui-filter-controls { - margin: -4px 0 8px 0; + margin: 0 0 16px 0; + padding: 12px; + background: #f8f9fa; + border: 1px solid #e0e0e0; + border-radius: 8px; display: flex; align-items: center; justify-content: space-between; @@ -1387,42 +2078,71 @@ export default { .cui-filter-checkbox { display: flex; align-items: center; - gap: 6px; + gap: 8px; margin: 0; font-size: 0.9rem; cursor: pointer; font-weight: 400; color: var(--color-text); + + input[type="checkbox"] { + width: 16px; + height: 16px; + cursor: pointer; + accent-color: $primary; + border: 1px solid #d0d0d0; + } } .cui-filter-paste-toggle { - padding: 0; + padding: 4px 8px; font-size: 0.85rem; + border: 1px solid #d0d0d0; + border-radius: 6px; + background: white; + transition: all 0.2s ease; + + &:hover { + background: #f0f0f0; + border-color: #b0b0b0; + } } .cui-filter-row { display: flex; - gap: 16px; + gap: 20px; align-items: flex-start; - margin: 6px 0 10px 0; + margin: 0 0 16px 0; } .cui-filter-picker { flex: 0 0 50%; max-width: 50%; + padding: 12px; + background: #fafafa; + border: 1px solid #e0e0e0; + border-radius: 8px; } .cui-file-picker { - flex: 0 0 calc(50% - 16px); - max-width: calc(50% - 16px); + flex: 0 0 calc(50% - 20px); + max-width: calc(50% - 20px); + padding: 12px; + background: #fafafa; + border: 1px solid #e0e0e0; + border-radius: 8px; label { display: block; - margin-bottom: 4px; + margin-bottom: 6px; font-weight: 500; color: var(--color-heading); font-size: 0.9rem; } + + .form-control { + border: 1px solid #d0d0d0; + } } .cui-pill-row { @@ -1607,34 +2327,48 @@ export default { } .form-content { - padding: 12px 16px; + padding: 0; + } + + .form-sections-wrapper { + padding: 16px; } .form-section { - margin-bottom: 16px; - padding-bottom: 12px; + margin-bottom: 20px; + padding: 16px; h4 { - font-size: 0.95rem; - margin-bottom: 8px; + font-size: 1rem; + margin-bottom: 12px; + padding-bottom: 10px; } .form-row { flex-direction: column; - gap: 12px; + gap: 16px; .form-group-inline { min-width: 100%; + margin-bottom: 0; + } + + .checkbox-group.form-group-inline { + align-self: flex-start; } } } .form-group { - margin-bottom: 10px; + margin-bottom: 14px; label { - font-size: 0.85rem; - margin-bottom: 3px; + font-size: 0.9rem; + margin-bottom: 6px; + } + + .form-control { + padding: 8px 10px; } } } @@ -1645,7 +2379,7 @@ export default { .cui-filter-row { flex-direction: column; - gap: 12px; + gap: 16px; .cui-filter-picker, .cui-file-picker { @@ -1654,4 +2388,137 @@ export default { } } } + +// Tab Navigation Styles +.admin-tabs { + display: flex; + gap: 8px; + margin: 20px 0; + border-bottom: 2px solid var(--color-border); + padding-bottom: 0; +} + +.tab-button { + padding: 12px 24px; + background: transparent; + border: none; + border-bottom: 3px solid transparent; + cursor: pointer; + font-size: 0.95rem; + font-weight: 500; + color: var(--color-text-secondary); + display: flex; + align-items: center; + gap: 8px; + transition: all 0.2s ease; + margin-bottom: -2px; + + &:hover { + color: var(--color-primary); + background: rgba(0, 0, 0, 0.02); + } + + &.active { + color: var(--color-primary); + border-bottom-color: var(--color-primary); + font-weight: 600; + } + + svg { + font-size: 1rem; + } +} + +// Admin Section Styles (for Model Packs, Datasets, Users) +.admin-section { + .list-section { + .section-header { + margin-bottom: 20px; + + h3 { + font-size: 1.5rem; + font-weight: 600; + color: var(--color-heading); + margin: 0; + } + + .item-count { + font-weight: 400; + color: var(--color-text-secondary); + font-size: 1rem; + } + } + + .table-container { + background: white; + border-radius: 8px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + overflow: hidden; + } + } + + .admin-table { + .action-buttons { + display: flex; + gap: 6px; + justify-content: flex-end; + } + } + + .form-section { + background: white; + border-radius: 8px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + overflow: hidden; + display: flex; + flex-direction: column; + height: calc(100vh - 250px); + min-height: 600px; + } + + .admin-form { + display: flex; + flex-direction: column; + height: 100%; + overflow: hidden; + + .form-sections-wrapper { + flex: 1; + overflow-y: auto; + overflow-x: hidden; + min-height: 0; + padding: 20px; + } + + .form-actions { + margin-top: auto; + flex-shrink: 0; + padding: 20px; + border-top: 1px solid var(--color-border); + display: flex; + gap: 12px; + justify-content: flex-end; + background: var(--color-background-light); + } + } +} + +.empty-state { + text-align: center; + padding: 60px 20px; + background: white; + border-radius: 8px; + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.1); + + h4 { + font-size: 1.25rem; + color: var(--color-heading); + margin-bottom: 8px; + } + + p { + color: var(--color-text-secondary); + margin-bottom: 20px; + } +} From 51dfbff0ea00cda88c151dc0d91c157ff61f83c9 Mon Sep 17 00:00:00 2001 From: Tom Searle Date: Mon, 26 Jan 2026 23:48:12 +0000 Subject: [PATCH 18/22] feat(medcat-trainer): split project admin into components, help text around dataset schema --- medcat-trainer/webapp/api/api/serializers.py | 20 + medcat-trainer/webapp/api/api/views.py | 14 + .../src/components/admin/DatasetForm.vue | 387 +++++++++++ .../src/components/admin/DatasetsList.vue | 129 ++++ .../src/components/admin/ModelPackForm.vue | 315 +++++++++ .../src/components/admin/ModelPacksList.vue | 156 +++++ .../src/components/admin/ProjectsList.vue | 235 +++++++ .../src/components/admin/UserForm.vue | 319 +++++++++ .../src/components/admin/UsersList.vue | 134 ++++ .../frontend/src/views/ProjectAdmin.vue | 616 +++++------------- 10 files changed, 1886 insertions(+), 439 deletions(-) create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/DatasetForm.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/DatasetsList.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/ModelPackForm.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/ProjectsList.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/UserForm.vue create mode 100644 medcat-trainer/webapp/frontend/src/components/admin/UsersList.vue diff --git a/medcat-trainer/webapp/api/api/serializers.py b/medcat-trainer/webapp/api/api/serializers.py index 89ff70fea..49fd2cd16 100644 --- a/medcat-trainer/webapp/api/api/serializers.py +++ b/medcat-trainer/webapp/api/api/serializers.py @@ -60,6 +60,26 @@ class Meta: class DatasetSerializer(serializers.ModelSerializer): + """ + Serializer for Dataset model. + + Schema Requirements: + - File format: .csv or .xlsx + - Required columns: + * name: A unique identifier for each document (string) + * text: The free-text content to annotate (string) + - Additional columns are allowed but will be ignored + + Example CSV structure: + name,text + doc001,"This is the first document to annotate." + doc002,"This is the second document with medical text." + """ + + original_file = serializers.FileField( + help_text="Upload a .csv or .xlsx file with two required columns: 'name' (unique document identifier) and 'text' (free-text to annotate)." + ) + class Meta: model = Dataset fields = '__all__' diff --git a/medcat-trainer/webapp/api/api/views.py b/medcat-trainer/webapp/api/api/views.py index 74beab16f..506960097 100644 --- a/medcat-trainer/webapp/api/api/views.py +++ b/medcat-trainer/webapp/api/api/views.py @@ -199,6 +199,20 @@ class ModelPackViewSet(viewsets.ModelViewSet): class DatasetViewSet(viewsets.ModelViewSet): + """ + ViewSet for managing datasets. + + File Schema Requirements: + - Format: .csv or .xlsx file + - Required columns: + * name: A unique identifier for each document + * text: The free-text content to annotate + + Example CSV: + name,text + doc001,"First document text" + doc002,"Second document text" + """ permission_classes = [permissions.IsAuthenticated] http_method_names = ['get', 'post'] queryset = Dataset.objects.all() diff --git a/medcat-trainer/webapp/frontend/src/components/admin/DatasetForm.vue b/medcat-trainer/webapp/frontend/src/components/admin/DatasetForm.vue new file mode 100644 index 000000000..e9e9dcf18 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/DatasetForm.vue @@ -0,0 +1,387 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/DatasetsList.vue b/medcat-trainer/webapp/frontend/src/components/admin/DatasetsList.vue new file mode 100644 index 000000000..6ecdd0d77 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/DatasetsList.vue @@ -0,0 +1,129 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/ModelPackForm.vue b/medcat-trainer/webapp/frontend/src/components/admin/ModelPackForm.vue new file mode 100644 index 000000000..597006be6 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/ModelPackForm.vue @@ -0,0 +1,315 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue b/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue new file mode 100644 index 000000000..063fb7f60 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue @@ -0,0 +1,156 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/ProjectsList.vue b/medcat-trainer/webapp/frontend/src/components/admin/ProjectsList.vue new file mode 100644 index 000000000..4ef824f9c --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/ProjectsList.vue @@ -0,0 +1,235 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/UserForm.vue b/medcat-trainer/webapp/frontend/src/components/admin/UserForm.vue new file mode 100644 index 000000000..9a9e0bc79 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/UserForm.vue @@ -0,0 +1,319 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/components/admin/UsersList.vue b/medcat-trainer/webapp/frontend/src/components/admin/UsersList.vue new file mode 100644 index 000000000..0201ef506 --- /dev/null +++ b/medcat-trainer/webapp/frontend/src/components/admin/UsersList.vue @@ -0,0 +1,134 @@ + + + + + diff --git a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue index 35240cd4e..4cbdae94a 100644 --- a/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue +++ b/medcat-trainer/webapp/frontend/src/views/ProjectAdmin.vue @@ -68,75 +68,16 @@
-
-
-

- Your Projects - ({{ projects.length }}) -

-
- -
- - - - - - -
- -
-
-

No Projects Yet

-

You don't have any projects yet. Create one to get started!

- -
-
-
+
@@ -445,271 +386,70 @@
-
-
-

Model Packs ({{ modelPacks.length }})

-
-
- - - - - -
-
-

No Model Packs

-

Add a model pack to get started.

-
-
+ -
-
- -

{{ editingModelPack ? 'Edit Model Pack' : 'Add Model Pack' }}

-
-
-
-
-
-
- - -
-
- - - Upload a .zip file containing the model pack -
-
-
-
- - -
-
- - -
-
-
-
- - -
-
-
-
+
-
-
-

Datasets ({{ datasets.length }})

-
-
- - - -
-
-

No Datasets

-

Add a dataset to get started.

-
-
+ -
-
- -

{{ editingDataset ? 'Edit Dataset' : 'Add Dataset' }}

-
-
-
-
-
-
- - -
-
- - -
-
-
-
- - - Upload a .csv or .xlsx file. Must contain 'name' and 'text' columns. -
-
-
-
- - -
-
-
-
+
-
-
-

Users ({{ users.length }})

-
-
- - - - - -
-
-

No Users

-

Add a user to get started.

-
-
+ -
-
- -

{{ editingUser ? 'Edit User' : 'Add User' }}

-
-
-
-
-
-
- - -
-
- - -
-
-
-
- - - Note: Password cannot be set via API. Users should set their password through password reset or Django admin. -
-
-
-
-
- -
-
- -
-
-
-
-
- - -
-
-
-
+
@@ -749,12 +489,26 @@ diff --git a/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue b/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue index 063fb7f60..3d06a392e 100644 --- a/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue +++ b/medcat-trainer/webapp/frontend/src/components/admin/ModelPacksList.vue @@ -8,7 +8,7 @@ :items="modelPacks" :headers="headers" :hover="true" - @click:row="$emit('select-model-pack', $event)" + @click:row="handleRowClick" hide-default-footer :items-per-page="-1" class="admin-table" @@ -21,9 +21,6 @@