Merge pull request #643 from BryanGsep/gsoc/ml_assistant

Machine learning training assistant GSoC 2023 project main pull request code
camicroscope · Oct 10, 2023 · 3a7f5f2 · 3a7f5f2
2 parents 2293839 + bf91174
commit 3a7f5f2
Show file tree

Hide file tree

Showing 14 changed files with 2,971 additions and 148 deletions.
diff --git a/apps/viewer/init.js b/apps/viewer/init.js
diff --git a/apps/viewer/turf.min.js b/apps/viewer/turf.min.js
diff --git a/apps/viewer/uicallbacks.js b/apps/viewer/uicallbacks.js
@@ -279,6 +279,7 @@ function draw(e) {
     }
     if ($CAMIC.status == 'normal') {
       annotationOn.call(this, state, target);
+      mlAsisstantOn();
       return;
     }
     toolsOff();
@@ -288,13 +289,15 @@ function draw(e) {
           // all tool has turn off
             clearInterval(checkAllToolsOff);
             annotationOn.call(this, state, target);
+            mlAsisstantOn();
           }
         }.bind(this),
         100,
     );
   } else {
     // off
     annotationOff();
+    mlAsisstantOff();
   }
 }
 
@@ -311,9 +314,11 @@ function toolsOff() {
       break;
     case 'normal':
       annotationOff();
+      mlAsisstantOff();
       break;
     case 'label':
       presetLabelOff();
+      mlAsisstantOff();
       break;
     case 'download_selection':
       downloadSelectionOff();
@@ -331,6 +336,7 @@ function annotationOn(state, target) {
   canvasDraw.style.color = style.color;
 
   li.appendChild(label);
+  $UI.AssistantViewer.undoBtn.onclick=()=>canvasDraw.__align_undo();
   switch (state) {
     case 1:
       spen.menu(65, 0.2);
@@ -660,6 +666,7 @@ function mainMenuChange(data) {
     $UI.labelsSideMenu.open();
   } else {
     presetLabelOff();
+    mlAsisstantOff();
   }
 }
 
@@ -971,6 +978,31 @@ function annoCallback(data) {
       .finally(() => {});
 }
 
+function editAnnoCallback(id, slide, annotJson) {
+    // save edit annotation
+    $CAMIC.store
+      .updateMaskEdit(id, slide, annotJson)
+      .then((data) => {
+        // server error
+        if (data.error) {
+          $UI.message.addError(`${data.text}:${data.url}`);
+          Loading.close();
+          return;
+        }
+
+        // no data added
+        if (data.count < 1) {
+          Loading.close();
+          $UI.message.addWarning(`Edit Annotation Failed`);
+          return;
+        }
+      })
+      .catch((e) => {
+        Loading.close();
+      })
+      .finally(() => {});
+}
+
 function saveAnnotCallback() {
   /* reset as default */
   // clear draw data and UI
@@ -1943,6 +1975,7 @@ function drawLabel(e) {
   if (e.checked) {
     if ($CAMIC.status == 'label') {
       presetLabelOn.call(this, labels);
+      mlAsisstantOn(false);
       return;
     }
     // turn off annotation
@@ -1954,13 +1987,15 @@ function drawLabel(e) {
           // all tool has turn off
             clearInterval(checkAllToolsOff);
             presetLabelOn.call(this, labels);
+            mlAsisstantOn(false);
           }
         }.bind(this),
         100,
     );
   } else {
     // off preset label
     presetLabelOff();
+    mlAsisstantOff();
   }
 }
 
@@ -2014,6 +2049,21 @@ function presetLabelOff() {
   }
 }
 
+function mlAsisstantOff() {
+  $UI.AssistantViewer.enableBtn.checked = false;
+  $UI.AssistantViewer.elt.style.display = 'none';
+  $UI.AssistantSideMenu.close();
+}
+
+function mlAsisstantOn(enableUndo = true) {
+  if (!enableUndo) {
+    $UI.AssistantViewer.disableUndo();
+  } else {
+    $UI.AssistantViewer.enableUndo();
+  }
+  $UI.AssistantSideMenu.open();
+}
+
 function savePresetLabel() {
   if ($CAMIC.viewer.canvasDrawInstance._path_index === 0) {
     // toast
@@ -2026,6 +2076,7 @@ function savePresetLabel() {
     $UI.message.addWarning('No Label Selected. Please select One.', 4000);
     return;
   }
+  const features = $CAMIC.viewer.canvasDrawInstance.getImageFeatureCollection().features;
   const execId = randomId();
   const labelId = data.id;
   const labelName = data.type;
@@ -2036,14 +2087,48 @@ function savePresetLabel() {
     name: labelName,
     notes: data.type,
   };
-  const feature = $CAMIC.viewer.canvasDrawInstance.getImageFeatureCollection()
-      .features[0];
   let annotJson;
-  if (feature.properties.size) {
+  if (Array.isArray(features) && features[0].properties.size) {
+    // many brush
+    const values = features.reduce((p, f) => {
+      return p.concat(getGrids(
+        f.geometry.coordinates[0],
+        f.properties.size,
+      ));
+    },[]);
+    const set = new Set();
+    values.map((i) => i.toString()).forEach((v) => set.add(v));
+    const points = Array.from(set).map((d) => d.split(','));
+    annotJson = {
+      creator: getUserId(),
+      created_date: new Date(),
+      provenance: {
+        image: {
+          slide: $D.params.slideId,
+        },
+        analysis: {
+          source: 'human',
+          execution_id: execId, // randomId
+          name: labelName, // labelName
+          labelId: labelId,
+          type: 'label',
+          isGrid: true,
+        },
+      },
+      properties: {
+        annotations: noteData,
+      },
+      geometries: convertGeometries(points, {
+        note: data.type,
+        size: features[0].properties.size,
+        color: features[0].properties.style.color,
+      }),
+    };
+  } else if (!Array.isArray(features) && features.properties.size) {
     // brush
     const values = getGrids(
-        feature.geometry.coordinates[0],
-        feature.properties.size,
+        features.geometry.coordinates[0],
+        features.properties.size,
     );
     const set = new Set();
     values.map((i) => i.toString()).forEach((v) => set.add(v));
@@ -2069,8 +2154,8 @@ function savePresetLabel() {
       },
       geometries: convertGeometries(points, {
         note: data.type,
-        size: feature.properties.size,
-        color: feature.properties.style.color,
+        size: features.properties.size,
+        color: features.properties.style.color,
       }),
     };
   } else {

diff --git a/apps/viewer/viewer.html b/apps/viewer/viewer.html
@@ -72,6 +72,13 @@
       media="all"
       href="../../components/layersviewer/layersviewer.css"
     />
+    <!-- machine learning assistant css -->
+    <link
+      rel="stylesheet"
+      type="text/css"
+      media="all"
+      href="../../components/ml-assistant/ml-assistant.css"
+    />
     <!-- labels controller css -->
     <link
       rel="stylesheet"
@@ -171,6 +178,7 @@
     <link href="../../common/bootstrap-tour-standalone/bootstrap-tour-standalone.min.css" rel="stylesheet">
     <!-- tippy theme -->
     <link rel="stylesheet" href="https://unpkg.com/tippy.js@6/themes/light-border.css" />
+    <link rel="stylesheet" href="https://unpkg.com/tippy.js@6/themes/translucent.css" />
 
     <!--  common js START -->
     <!-- Smartpen -->
@@ -179,6 +187,10 @@
     <link rel="stylesheet" type="text/css" media="all" href="../../common/smartpen/autoalign.css"/>
     <!-- Smartpen end -->
 
+    <!-- machine learning tool start -->
+    <script type="text/javascript" src="../../components/ml-assistant/ml-tool.js"></script>
+    <!-- machine learning tool end -->
+
     <!-- util.js -->
     <script type="text/javascript" src="../../common/util.js"></script>
     <script type="text/javascript" src="../../common/DrawHelper.js"></script>
@@ -283,6 +295,11 @@
       type="text/javascript"
       src="../../components/modalbox/modalbox.js"
     ></script>
+    <!-- ml-assistant -->
+    <script
+      type="text/javascript"
+      src="../../components/ml-assistant/ml-assistant.js"
+    ></script>
     <!--  components js END -->
 
     <!-- osd & core js START -->
@@ -310,6 +327,15 @@
     <script type="text/javascript" src="../../core/Store.js"></script>
     <script type="text/javascript" src="../../core/CaMic.js"></script>
 
+    <!-- opencv.js -->
+    <script type='text/javascript' src='../segment/opencv.js'></script>
+    <!-- turf.js -->
+    <script type='text/javascript' src='turf.min.js'></script>
+    <!-- tensorflow.js -->
+    <script type='text/javascript' src='../../common/tf.min.js'></script>
+    <!-- tfjs visulisation -->
+    <script type='text/javascript' src='../../common/tfjs-vis.min.js'></script>
+
     <script
       type="text/javascript"
       src="../../core/extension/openseadragon-canvas-draw-overlay.js"
@@ -370,6 +396,9 @@
       <button class="reset" style="float: left;">Clear</button
       ><button class="action" style="float: right;">Save</button>
     </div> -->
+    <!-- modalbox -->
+    <div id="model_info"></div>
+    <div id="upload_panel"></div>
     <!-- side menu for apps-->
     <div id="side_apps" style="z-index:600"></div>
 
@@ -378,6 +407,8 @@
 
     <!-- side menu for labels -->
     <div id="labels_layers" style="z-index:600"></div>
+    <!-- side menu for labels -->
+    <div id="ml_assistant_layers" style="z-index:600"></div>
 
     <!-- collapsible list - annotation and analytics -->
     <div id="collapsiblelist" style="z-index:600"></div>
@@ -390,6 +421,9 @@
 
     <!-- labels manager -->
     <div id="labelmanager" style="z-index:600"></div>
+    <!-- machine learning assistant -->
+    <div id="ml_assistant" style="z-index:600"></div>
+    <div id="popup-container"></div>
     <!-- Popper & tippy -->
     <script src="https://unpkg.com/@popperjs/core@2"></script>
     <script src="https://unpkg.com/tippy.js@6"></script>

diff --git a/common/LocalStore.js b/common/LocalStore.js
@@ -174,6 +174,16 @@ function init_LocalStore(){
       res(putInLocalStorage('mark', json))
     })
   }
+  Store.prototype.updateMaskEdit = function(maskId, slide, data) {
+    if (!this.validation.mark(data)){
+      console.warn(this.validation.mark.errors)
+    }
+    return new Promise(function(res, rej){
+      removeFromLocalStorage('mark', maskId).then(x => {
+        res(putInLocalStorage('mark', data))
+      })
+    })
+  }
   Store.prototype.deleteMark = function(id, slide){
     return new Promise((res, rej)=>{
       res(removeFromLocalStorage('mark', id))