Adding GUI features to enable the marker editor in Python

keenon · Jul 12, 2022 · 9330749 · 9330749
1 parent 179cefa
commit 9330749
Show file tree

Hide file tree

Showing 23 changed files with 869 additions and 217 deletions.
diff --git a/VERSION.txt b/VERSION.txt
@@ -1 +1 @@
-0.8.7
+0.8.8
diff --git a/dart/biomechanics/OpenSimParser.cpp b/dart/biomechanics/OpenSimParser.cpp
@@ -10,6 +10,8 @@
 #include <utility>
 #include <vector>
 
+#include <tinyxml2.h>
+
 #include "dart/biomechanics/ForcePlate.hpp"
 #include "dart/biomechanics/IKErrorReport.hpp"
 #include "dart/common/Uri.hpp"
@@ -1192,6 +1194,119 @@ void OpenSimParser::rationalizeJoints(
   newFile.SaveFile(outputPath.c_str());
 }
 
+/// Read an *.osim file, overwrite all the markers, and write it out
+/// to a new *.osim file
+void OpenSimParser::replaceOsimMarkers(
+    const common::Uri& uri,
+    const std::map<std::string, std::pair<std::string, Eigen::Vector3s>>&
+        markers,
+    const std::map<std::string, bool> isAnatomical,
+    const std::string& outputPath,
+    const common::ResourceRetrieverPtr& nullOrRetriever)
+{
+  const common::ResourceRetrieverPtr retriever
+      = ensureRetriever(nullOrRetriever);
+
+  //--------------------------------------------------------------------------
+  // Load xml and create Document
+  tinyxml2::XMLDocument originalFile;
+  try
+  {
+    openXMLFile(originalFile, uri, retriever);
+  }
+  catch (std::exception const& e)
+  {
+    std::cout << "LoadFile [" << uri.toString() << "] Fails: " << e.what()
+              << std::endl;
+    return;
+  }
+
+  //--------------------------------------------------------------------------
+  // Deep copy document
+
+  tinyxml2::XMLDocument newFile;
+  originalFile.DeepCopy(&newFile);
+
+  //--------------------------------------------------------------------------
+  tinyxml2::XMLElement* docElement
+      = newFile.FirstChildElement("OpenSimDocument");
+  if (docElement == nullptr)
+  {
+    dterr << "OpenSim file[" << uri.toString()
+          << "] does not contain <OpenSimDocument> as the root element.\n";
+    return;
+  }
+  tinyxml2::XMLElement* modelElement = docElement->FirstChildElement("Model");
+  if (modelElement == nullptr)
+  {
+    dterr << "OpenSim file[" << uri.toString()
+          << "] does not contain <Model> as the child of the root "
+             "<OpenSimDocument> element.\n";
+    return;
+  }
+
+  //--------------------------------------------------------------------------
+  // Go through the body nodes and adjust the scaling
+  tinyxml2::XMLElement* markerSet
+      = modelElement->FirstChildElement("MarkerSet");
+
+  if (markerSet == nullptr)
+  {
+    dterr << "OpenSim file[" << uri.toString()
+          << "] does not contain <MarkerSet> as the child of the root "
+             "<Model> element.\n";
+    return;
+  }
+
+  tinyxml2::XMLElement* markerSetObjects
+      = markerSet->FirstChildElement("objects");
+  if (markerSetObjects == nullptr)
+  {
+    dterr << "OpenSim file[" << uri.toString()
+          << "] does not contain <objects> as the child of the root "
+             "<MarkerSet> element.\n";
+    return;
+  }
+
+  markerSetObjects->DeleteChildren();
+
+  for (auto& pair : markers)
+  {
+    tinyxml2::XMLElement* marker
+        = markerSetObjects->InsertNewChildElement("Marker");
+    /*
+            <Marker name="RACR">
+                <!--Body segment in the model on which the marker resides.-->
+                <body>torso</body>
+                <!--Location of a marker on the body segment.-->
+                <location> -0.003000 0.425000 0.130000</location>
+                <!--Flag (true or false) specifying whether or not a marker
+       should be kept fixed in the marker placement step.  i.e. If false, the
+       marker is allowed to move.--> <fixed>false</fixed>
+            </Marker>
+    */
+    marker->SetAttribute("name", pair.first.c_str());
+
+    tinyxml2::XMLElement* body = marker->InsertNewChildElement("body");
+    body->SetText(pair.second.first.c_str());
+
+    tinyxml2::XMLElement* location = marker->InsertNewChildElement("location");
+    Eigen::Vector3s markerOffset = pair.second.second;
+    location->SetText((" " + std::to_string(markerOffset(0)) + " "
+                       + std::to_string(markerOffset(1)) + " "
+                       + std::to_string(markerOffset(2)))
+                          .c_str());
+
+    tinyxml2::XMLElement* fixed = marker->InsertNewChildElement("fixed");
+    fixed->SetText(
+        (isAnatomical.count(pair.first) > 0 && isAnatomical.at(pair.first))
+            ? "true"
+            : "false");
+  }
+
+  newFile.SaveFile(outputPath.c_str());
+}
+
 //==============================================================================
 /// Read an *.osim file, rescale the body, and write it out to a new *.osim
 /// file

diff --git a/dart/biomechanics/OpenSimParser.hpp b/dart/biomechanics/OpenSimParser.hpp
@@ -136,6 +136,16 @@ class OpenSimParser
       const std::string& outputPath,
       const common::ResourceRetrieverPtr& retriever = nullptr);
 
+  /// Read an *.osim file, overwrite all the markers, and write it out
+  /// to a new *.osim file
+  static void replaceOsimMarkers(
+      const common::Uri& uri,
+      const std::map<std::string, std::pair<std::string, Eigen::Vector3s>>&
+          markers,
+      const std::map<std::string, bool> isAnatomical,
+      const std::string& outputPath,
+      const common::ResourceRetrieverPtr& retriever = nullptr);
+
   /// Read an *.osim file, move the markers to new locations, and write it out
   /// to a new *.osim file
   static void moveOsimMarkers(

diff --git a/dart/proto/GUI.proto b/dart/proto/GUI.proto
@@ -23,7 +23,8 @@ message Command {
     SetObjectScale set_object_scale = 8;
     SetObjectTooltip set_object_tooltip = 32;
     DeleteObjectTooltip delete_object_tooltip = 33;
-    EnableMouseInteraction enable_mouse_interaction = 18;
+    EnableDrag enable_drag = 18;
+    EnableEditTooltip enable_edit_tooltip = 34;
     CreateText text = 12;
     CreateButton button = 13;
     CreateSlider slider = 14;
@@ -142,7 +143,11 @@ message DeleteObjectTooltip {
   int32 key = 1;
 }
 
-message EnableMouseInteraction {
+message EnableDrag {
+  int32 key = 1;
+}
+
+message EnableEditTooltip {
   int32 key = 1;
 }
 

diff --git a/dart/server/GUIStateMachine.cpp b/dart/server/GUIStateMachine.cpp
@@ -96,9 +96,13 @@ std::string GUIStateMachine::getCurrentStateAsJson()
       encodeSetRichPlotData(list, pair.second.key, dataPair.second);
     }
   }
-  for (auto key : mMouseInteractionEnabled)
+  for (auto key : mDragEnabled)
   {
-    encodeEnableMouseInteraction(list, key);
+    encodeEnableDrag(list, key);
+  }
+  for (auto key : mTooltipEditable)
+  {
+    encodeEnableEditTooltip(list, key);
   }
 
   return list.SerializeAsString();
@@ -1140,13 +1144,24 @@ void GUIStateMachine::deleteObjectTooltip(const std::string& key)
   });
 }
 
-/// This sets an object to allow mouse interaction on the GUI
-void GUIStateMachine::setObjectMouseInteractionEnabled(const std::string& key)
+/// This sets an object to allow dragging around by the mouse on the GUI
+void GUIStateMachine::setObjectDragEnabled(const std::string& key)
 {
-  mMouseInteractionEnabled.emplace(key);
+  mDragEnabled.emplace(key);
   queueCommand([&](proto::CommandList& list) {
     proto::Command* command = list.add_command();
-    command->mutable_enable_mouse_interaction()->set_key(getStringCode(key));
+    command->mutable_enable_drag()->set_key(getStringCode(key));
+  });
+}
+
+/// This sets an object to allow editing the tooltip by double-clicking on the
+/// object
+void GUIStateMachine::setObjectTooltipEditable(const std::string& key)
+{
+  mTooltipEditable.emplace(key);
+  queueCommand([&](proto::CommandList& list) {
+    proto::Command* command = list.add_command();
+    command->mutable_enable_edit_tooltip()->set_key(getStringCode(key));
   });
 }
 
@@ -1938,11 +1953,18 @@ void GUIStateMachine::encodeCreateTexture(
   command->mutable_texture()->set_base64(texture.base64);
 }
 
-void GUIStateMachine::encodeEnableMouseInteraction(
+void GUIStateMachine::encodeEnableDrag(
+    proto::CommandList& list, const std::string& key)
+{
+  proto::Command* command = list.add_command();
+  command->mutable_enable_drag()->set_key(getStringCode(key));
+}
+
+void GUIStateMachine::encodeEnableEditTooltip(
     proto::CommandList& list, const std::string& key)
 {
   proto::Command* command = list.add_command();
-  command->mutable_enable_mouse_interaction()->set_key(getStringCode(key));
+  command->mutable_enable_edit_tooltip()->set_key(getStringCode(key));
 }
 
 void GUIStateMachine::encodeCreateText(proto::CommandList& list, Text& text)

diff --git a/dart/server/GUIStateMachine.hpp b/dart/server/GUIStateMachine.hpp
@@ -251,8 +251,12 @@ class GUIStateMachine
   /// This removes a tooltip for the object at key.
   void deleteObjectTooltip(const std::string& key);
 
-  /// This sets an object to allow mouse interaction on the GUI
-  void setObjectMouseInteractionEnabled(const std::string& key);
+  /// This sets an object to allow dragging around by the mouse on the GUI
+  void setObjectDragEnabled(const std::string& key);
+
+  /// This sets an object to allow editing the tooltip by double-clicking on the
+  /// object
+  void setObjectTooltipEditable(const std::string& key);
 
   /// This deletes an object by key
   void deleteObject(const std::string& key);
@@ -392,7 +396,8 @@ class GUIStateMachine
   proto::CommandList mCommandList;
   std::string mCommandListOutputBuffer;
   // This is a list of all the objects with mouse interaction enabled
-  std::unordered_set<std::string> mMouseInteractionEnabled;
+  std::unordered_set<std::string> mDragEnabled;
+  std::unordered_set<std::string> mTooltipEditable;
 
   std::unordered_map<std::string, int> mStringCodes;
   std::unordered_map<int, std::string> mCodeStrings;
@@ -572,7 +577,8 @@ class GUIStateMachine
   void encodeCreateMesh(proto::CommandList& list, Mesh& mesh);
   void encodeSetTooltip(proto::CommandList& list, Tooltip& tooltip);
   void encodeCreateTexture(proto::CommandList& list, Texture& texture);
-  void encodeEnableMouseInteraction(
+  void encodeEnableDrag(proto::CommandList& list, const std::string& key);
+  void encodeEnableEditTooltip(
       proto::CommandList& list, const std::string& key);
   void encodeCreateText(proto::CommandList& list, Text& text);
   void encodeCreateButton(proto::CommandList& list, Button& button);

diff --git a/dart/server/GUIWebsocketServer.cpp b/dart/server/GUIWebsocketServer.cpp
@@ -176,7 +176,7 @@ void GUIWebsocketServer::serve(int port)
     }
     else if (args["type"].asString() == "drag")
     {
-      std::string key = args["key"].asString();
+      std::string key = this->getCodeString(args["key"].asInt());
       Eigen::Vector3s pos = Eigen::Vector3s(
           static_cast<s_t>(args["pos"][0].asDouble()),
           static_cast<s_t>(args["pos"][1].asDouble()),
@@ -187,6 +187,24 @@ void GUIWebsocketServer::serve(int port)
         handler(pos);
       }
     }
+    else if (args["type"].asString() == "drag_end")
+    {
+      std::string key = this->getCodeString(args["key"].asInt());
+      for (auto handler : mDragEndListeners[key])
+      {
+        handler();
+      }
+    }
+    else if (args["type"].asString() == "edit_tooltip")
+    {
+      std::string key = this->getCodeString(args["key"].asInt());
+      std::string tooltip = args["tooltip"].asString();
+
+      for (auto handler : mTooltipChangeListeners[key])
+      {
+        handler(tooltip);
+      }
+    }
   });
 
   // unblock signals in this thread
@@ -420,12 +438,27 @@ void GUIWebsocketServer::clear()
 /// "listener" whenever the object is dragged with the desired drag
 /// coordinates
 GUIWebsocketServer& GUIWebsocketServer::registerDragListener(
-    const std::string& key, std::function<void(Eigen::Vector3s)> listener)
+    const std::string& key,
+    std::function<void(Eigen::Vector3s)> listener,
+    std::function<void()> endDrag)
 {
   const std::lock_guard<std::recursive_mutex> lock(this->globalMutex);
 
-  setObjectMouseInteractionEnabled(key);
+  setObjectDragEnabled(key);
   mDragListeners[key].push_back(listener);
+  mDragEndListeners[key].push_back(endDrag);
+  return *this;
+}
+
+/// This enables the user to edit the tooltip on an object, and calls this
+/// listener when the tooltip changes.
+GUIWebsocketServer& GUIWebsocketServer::registerTooltipChangeListener(
+    const std::string& key, std::function<void(std::string)> listener)
+{
+  const std::lock_guard<std::recursive_mutex> lock(this->globalMutex);
+
+  setObjectTooltipEditable(key);
+  mTooltipChangeListeners[key].push_back(listener);
   return *this;
 }
 

diff --git a/dart/server/GUIWebsocketServer.hpp b/dart/server/GUIWebsocketServer.hpp
@@ -94,9 +94,17 @@ class GUIWebsocketServer : public GUIStateMachine
 
   /// This enables mouse events on an object (if they're not already), and
   /// calls "listener" whenever the object is dragged with the desired drag
-  /// coordinates
+  /// coordinates. The "endDrag" function is called whenever the user releases
+  /// their mouse.
   GUIWebsocketServer& registerDragListener(
-      const std::string& key, std::function<void(Eigen::Vector3s)> listener);
+      const std::string& key,
+      std::function<void(Eigen::Vector3s)> listener,
+      std::function<void()> endDrag);
+
+  /// This enables the user to edit the tooltip on an object, and calls this
+  /// listener when the tooltip changes.
+  GUIWebsocketServer& registerTooltipChangeListener(
+      const std::string& key, std::function<void(std::string)> listener);
 
   /// This gets the current screen size
   Eigen::Vector2i getScreenSize();
@@ -127,6 +135,10 @@ class GUIWebsocketServer : public GUIStateMachine
       std::string,
       std::vector<std::function<void(Eigen::Vector3s)>>>
       mDragListeners;
+  std::unordered_map<std::string, std::vector<std::function<void()>>>
+      mDragEndListeners;
+  std::unordered_map<std::string, std::vector<std::function<void(std::string)>>>
+      mTooltipChangeListeners;
   std::vector<std::function<void(Eigen::Vector2i)>> mScreenResizeListeners;
   // This is a list of all the objects with mouse interaction enabled
   std::unordered_set<std::string> mMouseInteractionEnabled;

diff --git a/javascript/src/NimbleRemote.ts b/javascript/src/NimbleRemote.ts
@@ -42,6 +42,27 @@ class DARTRemote {
         this.socket.send(message);
       }
     });
+
+    this.view.addDragEndListener((key: number) => {
+      const message = JSON.stringify({
+        type: "drag_end",
+        key
+      });
+      if (this.socket != null && this.socket.readyState == WebSocket.OPEN) {
+        this.socket.send(message);
+      }
+    });
+
+    this.view.addTooltipEditListener((key: number, tooltip: string) => {
+      const message = JSON.stringify({
+        type: "edit_tooltip",
+        key,
+        tooltip,
+      });
+      if (this.socket != null && this.socket.readyState == WebSocket.OPEN) {
+        this.socket.send(message);
+      }
+    });
   }
 
   /**