Merge pull request #102 from CatalystCode/uiupdates

UI updates

- Changed name to VOTT (Visual Object Tagging Tool)
- Optimized Configuration and Dialogs for Export and review
- Fixed export to cntk bug that was causing labels to be swapped with bounding boxes

README.md

@@ -47,7 +47,7 @@ The tool supports the following **features**:
 
  1. Load an MP4 video file either by dragging it into the app or clicking on and selecting it.
 
-  ![](media/2_load.jpg)
+    ![](media/2_load.jpg)
 
  2. Configure the tagging job and specify the settings in the screenshot below:
 
@@ -59,15 +59,6 @@ The tool supports the following **features**:
       - *Rectangle*: tag bounding boxes of any dimension
       - *Square*: tag bounding boxes of auto-fixed dimensions
 
-    **Export Frames Until**: how far into the video the export operation will proceed<br>
-      - *Last Tagged Region*: exports frames up until the last frame containing tags
-      - *Last Visited Frame*: exports frames up until the last frame that the user explicitly visited
-      - *Last Frame*: exports all video frames
-
-    **Output directory**: directory path for exporting training data<br>
-
-    **Model Path**: file path of the trained Fast-RCNN model file (optional)<br>
-
     **Labels**: labels of the tagged regions (e.g. `Cat`, `Dog`, `Horse`, `Person`)<br>
 
  3. Tag the video frame by frame
@@ -86,22 +77,34 @@ The tool supports the following **features**:
      - Since the [camshift algorithm](http://opencv.jp/opencv-1.0.0_org/docs/papers/camshift.pdf) has some known limitations, you can disable tracking for certain sets of frames. To toggle tracking *on* and *off* use the file menu setting, or the keyboard shortcut Ctrl/Cmd + T.
 
 
- 4. Export video to CNTK Format using Menu or Ctrl/Cmd + E
-
-    *Note on exporting: the tool reserves a random 20% sample of the tagged frames as a test set.*
+ 4. Export video to CNTK Format using Object Detection Menu or Ctrl/Cmd + E
 
     ![]( media/5_Export.jpg)
-
-
+
+    *Note on exporting: the tool reserves a random 20% sample of the tagged frames as a test set.*
+
+    Specify the following export configuration settings:
+
+    ![]( media/5a_Export.jpg)
+
+    - **Export Format**: What framework to export to defaults to *CNTK*<br>
+    - **Export Frames Until**: how far into the video the export operation will proceed<br>
+      - *Last Tagged Region*: exports frames up until the last frame containing tags
+      - *Last Visited Frame*: exports frames up until the last frame that the user explicitly visited
+      - *Last Frame*: exports all video frames<br>
+    - **Output directory**: directory path for exporting training data<br>
+
 ---
 ## Reviewing and Improving a CNTK Object Detection Model
 
  1. Train model with [Object Detection using FastRCNN](https://github.com/Microsoft/CNTK/wiki/Object-Detection-using-Fast-R-CNN#train-on-your-own-data)<br> *Note: the data is already in CNTK format, so you do not have to run `C1_DrawBboxesOnImages.py` or `C2_AssignLabelsToBboxes.py`*
  2. Load a new video that the model has not been trained on
- 3. Configure the tagging job specifying the following preferences
+ 3. Configure a new or load a previous tagging job 
  4. Apply model to new video using Ctrl/Cmd + R
- 5. When the model finishes processing, validate tags, re-export and retrain it
- 6. Repeat step 1 on new videos until the model performance is satisfactory
+ 5. Specify a model path and temporary output directory<br> 
+    ![](media/6_Review.jpg)
+ 6. When the model finishes processing, validate tags, re-export and retrain it
+ 7. Repeat step 1 on new videos until the model performance is satisfactory
 
 ## Upcoming Features 
 

main.js

@@ -27,7 +27,7 @@ function createWindow () {
       y: mainWindowState.y,
       minHeight: 480,
       minWidth: 480,
-      icon: __dirname + 'src/public/images/icon.png',
+      icon: __dirname + '/src/public/images/icon.png',
       show: false
   });
 
@@ -51,6 +51,52 @@ function createWindow () {
     menu.items[p+1].submenu.items[1].enabled = true;
   });
 
+
+  // do this independently for each object
+  ipcMain.on('show-popup', function(event, arg) { 
+      var popup = new BrowserWindow({
+        parent: mainWindow, 
+        modal: true, 
+        show: false, 
+        frame: false,
+        autoHideMenuBar : true
+      });
+      switch (arg) {
+        case "export":
+            popup.setSize(359, 300);
+            popup.loadURL(url.format({
+              pathname: path.join(__dirname, 'src/public/html/export-configuration.html'),
+              protocol: 'file:',
+              slashes: true
+            }));
+          break;
+
+        case "review":
+          popup.setSize(359, 310);
+          popup.loadURL(url.format({
+            pathname: path.join(__dirname, 'src/public/html/review-configuration.html'),
+            protocol: 'file:',
+            slashes: true
+          }));
+          break;
+
+        default: return; 
+      }
+
+      popup.once('ready-to-show', () => {
+        popup.show();
+//        child.webContents.toggleDevTools();
+      });
+
+      ipcMain.on('export-tags', (event, arg) => {
+        mainWindow.send('export-tags', arg);
+      });
+
+      ipcMain.on('review-model', (event, arg) => {
+        mainWindow.send('review-model', arg);
+      });
+  });
+
   mainWindow.on('ready-to-show', function() {
       mainWindow.show();
       mainWindow.focus();
@@ -89,19 +135,19 @@ function createWindow () {
       ]
     },
     {
-      label: 'CNTK',
+      label: 'Object Detection',
       submenu: [
         {
-          label: 'Export Tags to CNTK',
+          label: 'Export Tags',
           accelerator: 'CmdOrCtrl+E',
           enabled: false,
-          click () { mainWindow.webContents.send('exportCNTK'); }
+          click () { mainWindow.webContents.send('export'); }
         },
         {
-          label: 'Review CNTK Model',
+          label: 'Review Detection Model',
           accelerator: 'CmdOrCtrl+R',
           enabled: false,
-          click () { mainWindow.webContents.send('reviewCNTK'); }
+          click () { mainWindow.webContents.send('review'); }
         }
       ]
     },
@@ -158,6 +204,9 @@ if (process.platform === 'darwin') {
 
   const menu = Menu.buildFromTemplate(template);
   Menu.setApplicationMenu(menu);
+
+
+
 }
 
 // This method will be called when Electron has finished

package.json

@@ -1,12 +1,12 @@
 {
-  "name": "cntk-video-tagging-tool",
-  "version": "1.0.2",
+  "name": "vott",
+  "version": "1.0.0",
   "description": "An electron app for building end to end Object Detection Models with CNTK from Sample Videos.",
   "main": "main.js",
   "scripts": {
     "start": "electron ."
   },
-  "repository": "https://github.com/CatalystCode/CNTK-Video-Tagging-Tool/",
+  "repository": "https://github.com/CatalystCode/VOTT",
   "keywords": [
     "Video-Tagging",
     "CNTK",

src/index.html

@@ -8,7 +8,7 @@
   <head>
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, minimum-scale=1.0, initial-scale=1.0, user-scalable=yes">
-    <title>Video Tagging Tool</title>
+    <title>Visual Object Tagging Tool</title>
     <!-- Insert this line above script imports for jquery -->
     <script>if (typeof module === 'object') {window.module = module; module = undefined;}</script>
     <script src="./bower_components/webcomponentsjs/webcomponents.min.js"></script>
@@ -57,26 +57,7 @@ <h2>Tagging Job Configuration</h2>
           </div>
 
           <div class="form-group">
-           <label for="exampleTextarea" title="(Which frame to Export to.)">Export Until: </label>
-            <select id="exportTo" class="form-control" id="text" >
-                  <option value="tagged" selected="selected">Last Tagged Region</option>
-                  <option value="visited">Last Visited Frame</option>
-                  <option value="last">Last Frame</option>
-            </select>
-          </div>
-
-          <div class="form-group">
-            <label for="Out"> Output Path</label>
-            <input id="output" class="form-control" type="text" />
-          </div>
-
-          <div class="form-group">
-            <label for="Model"> Model Path</label>
-            <input id="model" class="form-control" type="text" />
-          </div>
-
-          <div class="form-group">
-            <label for="Tags"> Labels (Required)</label>
+            <label for="Tags"> Labels* (Comma Seperated)</label>
             <input id="inputtags" class="form-control" type="text"  data-role="tagsinput" required/>
           </div>
 

src/index.js

@@ -14,6 +14,10 @@ var visitedFrames, //keep track of the visited frames
     CNTKExtension,
     trackingExtension; 
 
+$(document).ready(() => {//init confirm keys figure out why this doesn't work
+  $('#inputtags').tagsinput({confirmKeys: [13, 32, 44, 45, 46, 59, 188]});
+});
+
 //ipc rendering
 ipcRenderer.on('openVideo', (event, message) => {
   fileSelected();
@@ -26,24 +30,57 @@ ipcRenderer.on('saveVideo', (event, message) => {
   });
 });
 
-ipcRenderer.on('exportCNTK', (event, message) => {
+ipcRenderer.on('export', (event, message) => {
+   ipcRenderer.send('show-popup', 'export');
+});
+
+ipcRenderer.on('export-tags', (event, exportConfig) => {
+  //add logic for supporting alternate export methods besides cntk
+
+  //init cntk extensions
+  CNTKExtension = new VideoTaggingCNTKExtension({
+      videotagging: videotagging,
+      cntkPath: cntkConfig.cntkPath,
+      visitedFrames: visitedFrames,
+      exportUntil: exportConfig.exportUntil,
+      exportPath: exportConfig.exportPath
+  });
+
   addLoader();
-  CNTKExtension.exportCNTK(testSetSize, removeLoader);
+  CNTKExtension.exportCNTK(testSetSize, () => {
+     $(".loader").remove();
+  });
 });
 
-ipcRenderer.on('reviewCNTK', (event, message) => {
-    if (fs.existsSync(cntkConfig.cntkPath)) {
-        var modelLocation = $('#model').val();
-        if (fs.existsSync(modelLocation)) {
-          addLoader();
-          CNTKExtension.reviewCNTK(modelLocation, removeLoader);
-        } else {
-            alert(`No model found! Please make sure you put your model in the following directory: ${modelLocation}`)
-        }
-
-    } else {
-      alert("This feature isn't supported by your system please check your CNTK configuration and try again later.");
-    }
+ipcRenderer.on('review', (event, message) => {
+     ipcRenderer.send('show-popup', 'review');
+});
+
+ipcRenderer.on('review-model', (event, reviewModelConfig) => {
+  //add logic for supporting alternate review model methods besides cntk
+
+  //init cntk extensions
+  CNTKExtension = new VideoTaggingCNTKExtension({
+        videotagging: videotagging,
+        cntkPath: cntkConfig.cntkPath,
+        visitedFrames: visitedFrames,
+        exportPath: reviewModelConfig.exportPath
+  });
+
+  if (fs.existsSync(cntkConfig.cntkPath)) {
+      var modelLocation = reviewModelConfig.modelPath;
+      if (fs.existsSync(modelLocation)) {
+        addLoader();
+        CNTKExtension.reviewCNTK(modelLocation, () => {
+           $(".loader").remove();
+        });
+      } else {
+          alert(`No model found! Please make sure you put your model in the following directory: ${modelLocation}`)
+      }
+
+  } else {
+    alert("This feature isn't supported by your system please check your CNTK configuration and try again later.");
+  }
 });
 
 ipcRenderer.on('toggleTracking', (event, message) => {
@@ -97,10 +134,6 @@ function addLoader() {
   }
 }
 
-function removeLoader() {
-   $(".loader").remove();
-}
-
 //managed the visited frames
 function updateVisitedFrames(){
   visitedFrames.add(videotagging.getCurrentFrame());
@@ -137,15 +170,13 @@ function fileSelected(path) {
 
     // show configuration
     $('#load-message').hide();
-    $('#video-tagging-container').hide()
+    $('#video-tagging-container').hide();
     $('#load-form-container').show();
     $('#framerateGroup').show();
 
     //set title indicator
     $('title').text(`Video Tagging Job Configuration: ${pathJS.basename(pathName, pathJS.extname(pathName))}`);
-
     $('#inputtags').tagsinput('removeAll');//remove all previous tag labels
-    $('#output').val(`${basepath}/cntk`);
     $('#model').val(`${basepath}/cntk/Fast-RCNN.model`);
 
     try {
@@ -159,9 +190,7 @@ function fileSelected(path) {
       console.log(`Error loading save file ${e.message}`);
     }
 
-    // REPLACE implementation, use jQuery to remove all event handlers on loadbutton
-    document.getElementById('loadButton').parentNode.replaceChild(document.getElementById('loadButton').cloneNode(true), document.getElementById('loadButton'));
-    document.getElementById('loadButton').addEventListener('click', loadTagger);
+    document.getElementById('loadButton').onclick = loadTagger;
 
     function loadTagger (e) {
       if(framerate.validity.valid && inputtags.validity.valid) {
@@ -198,15 +227,6 @@ function fileSelected(path) {
         videotagging.video.removeEventListener("canplay", updateVisitedFrames); //remove old listener
         videotagging.video.addEventListener("canplay",updateVisitedFrames);
 
-        //init cntk extensions
-        CNTKExtension = new VideoTaggingCNTKExtension({
-            videotagging: videotagging,
-            cntkPath: cntkConfig.cntkPath,
-            visitedFrames: visitedFrames,
-            exportUntil: $('#exportTo').val(),
-            exportPath: $('#output').val()
-        });
-
         //init region tracking
         trackingExtension = new VideoTaggingTrackingExtension({
             videotagging: videotagging, 
@@ -231,10 +251,8 @@ function save() {
     var saveObject = {
       "frames" : videotagging.frames,
       "inputTags": $('#inputtags').val(),
-      "exportTo": $('#exportTo').val(),
       "visitedFrames": Array.from(visitedFrames),
     };
 
     fs.writeFileSync(`${videotagging.src}.json`, JSON.stringify(saveObject));
 }
-

src/public/css/styles.css

@@ -89,7 +89,7 @@ html, body {
     100% { transform: rotate(360deg); }
 }
 
-#loadButton, .label-info {
+.btn-primary, .label-info {
   color: #3c3c3c !important;
   margin-right: 5px;
   background-color: #d5d5d5 !important;