ENH: Add OpenXR Remoting support

This commit updates the view node by adding "Remoting" and "PlayerIPAddress"
properties. The corresponding UI updates include a checkbox for toggling remoting
and a `QLineEdit` for entering the IP address.

The macro `SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT` is added to
`vtkMRMLVirtualRealityConfigure.h.in` to conditionally include code specific
to OpenXR remoting.

The "Remoting" checkbox and "PlayerIPAddress" line edit are disabled if an
active connection has been established.

To fix OpenXRRemoting "wglDXRegisterObjectNV failed in RegisterSharedTexture()",
the OpenXRRemoting HelperWindow MultiSamples property is explicitly set to 0.

To ensure no background is displayed, the background alpha/color/gradient is
explicitly set when remoting is used.

The last "Remoting" and "PlayerIPAddress" properties associated with a successful
hardware connection are saved to and restored from the settings.

Co-authored-by: Lucas Gandel <[email protected]>

VirtualReality/CMakeLists.txt

@@ -13,6 +13,9 @@ endif()
 if(SlicerVirtualReality_HAS_OPENXR_SUPPORT)
   find_package(vtkRenderingOpenXR REQUIRED)
 endif()
+if(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+  find_package(vtkRenderingOpenXRRemoting REQUIRED)
+endif()
 
 #-----------------------------------------------------------------------------
 add_subdirectory(MRML)

VirtualReality/MRML/CMakeLists.txt

@@ -37,6 +37,11 @@ if(SlicerVirtualReality_HAS_OPENXR_SUPPORT)
     VTK::RenderingOpenXR
     )
 endif()
+if(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+  list(APPEND ${KIT}_TARGET_LIBRARIES
+    VTK::RenderingOpenXRRemoting
+    )
+endif()
 
 #-----------------------------------------------------------------------------
 SlicerMacroBuildModuleMRML(

VirtualReality/MRML/vtkMRMLVirtualRealityConfigure.h.in

@@ -3,5 +3,6 @@
 
 #cmakedefine SlicerVirtualReality_HAS_OPENVR_SUPPORT
 #cmakedefine SlicerVirtualReality_HAS_OPENXR_SUPPORT
+#cmakedefine SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT
 
 #endif

VirtualReality/MRML/vtkMRMLVirtualRealityViewNode.cxx

@@ -83,6 +83,9 @@ void vtkMRMLVirtualRealityViewNode::WriteXML(ostream& of, int nIndent)
   vtkMRMLWriteXMLBooleanMacro(hmdTransformUpdate, HMDTransformUpdate);
   vtkMRMLWriteXMLBooleanMacro(controllerModelsVisible, ControllerModelsVisible);
   vtkMRMLWriteXMLBooleanMacro(lighthouseModelsVisible, LighthouseModelsVisible);
+  // OpenXRRemoting
+  vtkMRMLWriteXMLBooleanMacro(remoting, Remoting);
+  vtkMRMLWriteXMLStdStringMacro(playerIPAddress, PlayerIPAddress);
   vtkMRMLWriteXMLEndMacro();
 }
 
@@ -105,6 +108,9 @@ void vtkMRMLVirtualRealityViewNode::ReadXMLAttributes(const char** atts)
   vtkMRMLReadXMLBooleanMacro(hmdTransformUpdate, HMDTransformUpdate);
   vtkMRMLReadXMLBooleanMacro(controllerModelsVisible, ControllerModelsVisible);
   vtkMRMLReadXMLBooleanMacro(lighthouseModelsVisible, LighthouseModelsVisible);
+  // OpenXRRemoting
+  vtkMRMLReadXMLBooleanMacro(remoting, Remoting);
+  vtkMRMLReadXMLStdStringMacro(playerIPAddress, PlayerIPAddress);
   vtkMRMLReadXMLEndMacro();
 
   this->EndModify(disabledModify);
@@ -131,6 +137,9 @@ void vtkMRMLVirtualRealityViewNode::Copy(vtkMRMLNode* anode)
   vtkMRMLCopyBooleanMacro(HMDTransformUpdate);
   vtkMRMLCopyBooleanMacro(ControllerModelsVisible);
   vtkMRMLCopyBooleanMacro(LighthouseModelsVisible);
+  // OpenXRRemoting
+  vtkMRMLCopyBooleanMacro(Remoting);
+  vtkMRMLCopyStringMacro(PlayerIPAddress);
   vtkMRMLCopyEndMacro();
 
   this->EndModify(disabledModify);
@@ -153,6 +162,9 @@ void vtkMRMLVirtualRealityViewNode::PrintSelf(ostream& os, vtkIndent indent)
   vtkMRMLPrintBooleanMacro(HMDTransformUpdate);
   vtkMRMLPrintBooleanMacro(ControllerModelsVisible);
   vtkMRMLPrintBooleanMacro(LighthouseModelsVisible);
+  // OpenXRRemoting
+  vtkMRMLPrintBooleanMacro(Remoting);
+  vtkMRMLPrintStdStringMacro(PlayerIPAddress);
   vtkMRMLPrintEndMacro();
 }
 

VirtualReality/MRML/vtkMRMLVirtualRealityViewNode.h

@@ -242,6 +242,19 @@ class VTK_SLICER_VIRTUALREALITY_MODULE_MRML_EXPORT vtkMRMLVirtualRealityViewNode
   static int GetXRRuntimeFromString(const char* name);
   ///@}
 
+  ///@{
+  /// Get/Set if remoting is enabled.
+  vtkGetMacro(Remoting, bool);
+  vtkSetMacro(Remoting, bool);
+  vtkBooleanMacro(Remoting, bool);
+  ///@}
+
+  ///@{
+  /// OpenXR remoting IP address to connect to.
+  vtkSetMacro(PlayerIPAddress, const std::string);
+  vtkGetMacro(PlayerIPAddress, std::string);
+  ///@}
+
   /// Return true if an error has occurred.
   /// "Connected" member requests connection but this method can tell if the
   /// hardware connection has been actually successfully established.
@@ -273,6 +286,10 @@ class VTK_SLICER_VIRTUALREALITY_MODULE_MRML_EXPORT vtkMRMLVirtualRealityViewNode
 
   std::string LastErrorMessage;
 
+  // OpenXRRemoting
+  bool Remoting{false};
+  std::string PlayerIPAddress;
+
   vtkMRMLVirtualRealityViewNode();
   ~vtkMRMLVirtualRealityViewNode() override;
   vtkMRMLVirtualRealityViewNode(const vtkMRMLVirtualRealityViewNode&);

VirtualReality/Resources/UI/qSlicerVirtualRealityModuleWidget.ui

@@ -37,13 +37,41 @@
        <widget class="QComboBox" name="XRRuntimeComboBox"/>
       </item>
       <item row="1" column="0">
+       <widget class="QLabel" name="EnableRemotingLabel">
+        <property name="text">
+         <string>Enable Remoting:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <layout class="QHBoxLayout" name="RemotingLayout">
+        <item>
+         <widget class="QCheckBox" name="RemotingEnabledCheckBox">
+          <property name="text">
+           <string/>
+          </property>
+          <property name="checked">
+           <bool>false</bool>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QLineEdit" name="PlayerIPAddressLineEdit">
+          <property name="inputMask">
+           <string>000.000.000.000</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="2" column="0">
        <widget class="QLabel" name="ConnectToHardwareLabel">
         <property name="text">
          <string>Connect to hardware:</string>
         </property>
        </widget>
       </item>
-      <item row="1" column="1">
+      <item row="2" column="1">
        <layout class="QHBoxLayout" name="ConnectionLayout">
         <item>
          <widget class="ctkCheckBox" name="ConnectCheckBox"/>
@@ -100,14 +128,14 @@
         </item>
        </layout>
       </item>
-      <item row="2" column="0">
+      <item row="3" column="0">
        <widget class="QLabel" name="EnableRenderingLabel">
         <property name="text">
          <string>Enable rendering:</string>
         </property>
        </widget>
       </item>
-      <item row="2" column="1">
+      <item row="3" column="1">
        <widget class="ctkCheckBox" name="RenderingEnabledCheckBox"/>
       </item>
      </layout>

VirtualReality/Widgets/qMRMLVirtualRealityView.cxx

@@ -21,6 +21,7 @@
 // For:
 //  - SlicerVirtualReality_HAS_OPENVR_SUPPORT
 //  - SlicerVirtualReality_HAS_OPENXR_SUPPORT
+//  - SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT
 #include "vtkMRMLVirtualRealityConfigure.h"
 
 // VR Logic includes
@@ -93,6 +94,11 @@
 #include <vtkOpenXRRenderer.h>
 #endif
 
+#if defined(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+// VTK Rendering/OpenXRRemoting includes
+#include <vtkOpenXRRemotingRenderWindow.h>
+#endif
+
 // VTK Rendering/VR includes
 #include <vtkVRCamera.h>
 #include <vtkVRRenderer.h>
@@ -242,7 +248,20 @@ void qMRMLVirtualRealityViewPrivate::createRenderWindow(vtkMRMLVirtualRealityVie
     vtkNew<vtkVirtualRealityViewOpenXRInteractorStyle> interactorStyle;
     interactorStyle->SetInteractorStyleDelegate(this->InteractorStyleDelegate);
 
-    this->RenderWindow = vtkSmartPointer<vtkOpenXRRenderWindow>::New();
+#if defined(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+    if (this->MRMLVirtualRealityViewNode->GetRemoting())
+    {
+      vtkSmartPointer<vtkOpenXRRemotingRenderWindow> xrRemotingRenderWindow = vtkSmartPointer<vtkOpenXRRemotingRenderWindow>::New();
+      xrRemotingRenderWindow->SetRemotingIPAddress(this->MRMLVirtualRealityViewNode->GetPlayerIPAddress().c_str());
+      // Address "wglDXRegisterObjectNV failed in RegisterSharedTexture()" reported when using "OpenXRRemoting"
+      xrRemotingRenderWindow->GetHelperWindow()->SetMultiSamples(0);
+      this->RenderWindow = xrRemotingRenderWindow;
+    }
+    else
+#endif
+    {
+      this->RenderWindow = vtkSmartPointer<vtkOpenXRRenderWindow>::New();
+    }
     this->Renderer = vtkSmartPointer<vtkOpenXRRenderer>::New();
     this->InteractorStyle = interactorStyle;
     this->Interactor = vtkSmartPointer<vtkVirtualRealityViewOpenXRInteractor>::New();
@@ -386,6 +405,10 @@ void qMRMLVirtualRealityViewPrivate::createRenderWindow(vtkMRMLVirtualRealityVie
 
   // Keep track of last valid parameters in the settings
   QSettings().setValue("VirtualReality/DefaultXRRuntime", xrRuntimeAsStr);
+#if defined(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+  QSettings().setValue("VirtualReality/DefaultRemotingEnabled", this->MRMLVirtualRealityViewNode->GetRemoting());
+  QSettings().setValue("VirtualReality/DefaultPlayerIPAddress", QString::fromStdString(this->MRMLVirtualRealityViewNode->GetPlayerIPAddress()));
+#endif
 
   qDebug() << "";
   qDebug() << "XR runtime \"" << xrRuntimeAsStr << "\" initialized";
@@ -434,7 +457,11 @@ vtkMRMLVirtualRealityViewNode::XRRuntimeType qMRMLVirtualRealityViewPrivate::cur
   }
 #endif
 #if defined(SlicerVirtualReality_HAS_OPENXR_SUPPORT)
-  if (vtkOpenXRRenderWindow::SafeDownCast(this->RenderWindow) != nullptr)
+  if (vtkOpenXRRenderWindow::SafeDownCast(this->RenderWindow) != nullptr
+#if defined(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+      || vtkOpenXRRemotingRenderWindow::SafeDownCast(this->RenderWindow) != nullptr
+#endif
+      )
   {
     return vtkMRMLVirtualRealityViewNode::OpenXR;
   }
@@ -444,6 +471,22 @@ vtkMRMLVirtualRealityViewNode::XRRuntimeType qMRMLVirtualRealityViewPrivate::cur
   return vtkMRMLVirtualRealityViewNode::UndefinedXRRuntime;
 }
 
+// --------------------------------------------------------------------------
+bool qMRMLVirtualRealityViewPrivate::currentXRRuntimeRemotingEnabled() const
+{
+#if defined(SlicerVirtualReality_HAS_OPENXRREMOTING_SUPPORT)
+  return vtkOpenXRRemotingRenderWindow::SafeDownCast(this->RenderWindow) != nullptr;
+#else
+  return false;
+#endif
+}
+
+// --------------------------------------------------------------------------
+std::string qMRMLVirtualRealityViewPrivate::currentXRRuntimeRemotingIPAddress() const
+{
+  return vtkOpenXRManager::GetInstance().GetConnectionStrategy()->GetIPAddress();
+}
+
 // --------------------------------------------------------------------------
 void qMRMLVirtualRealityViewPrivate::updateWidgetFromMRML()
 {
@@ -473,7 +516,9 @@ void qMRMLVirtualRealityViewPrivate::updateWidgetFromMRMLNoModify()
   }
 
   // Reset initialization attempts and clear errors if the XR runtime has changed
-  if (this->currentXRRuntime() != this->MRMLVirtualRealityViewNode->GetXRRuntime())
+  if (this->currentXRRuntime() != this->MRMLVirtualRealityViewNode->GetXRRuntime()
+      || this->currentXRRuntimeRemotingEnabled() != this->MRMLVirtualRealityViewNode->GetRemoting()
+      || this->currentXRRuntimeRemotingIPAddress() != this->MRMLVirtualRealityViewNode->GetPlayerIPAddress())
   {
     this->InitializationAttempts = 0;
     this->MRMLVirtualRealityViewNode->ClearError();
@@ -482,6 +527,8 @@ void qMRMLVirtualRealityViewPrivate::updateWidgetFromMRMLNoModify()
   // Attempt to initialize XR runtime if the current runtime differs or is undefined, and
   // the view is visible (i.e., connected to the hardware)
   if ((this->currentXRRuntime() != this->MRMLVirtualRealityViewNode->GetXRRuntime()
+       || this->currentXRRuntimeRemotingEnabled() != this->MRMLVirtualRealityViewNode->GetRemoting()
+       || this->currentXRRuntimeRemotingIPAddress() != this->MRMLVirtualRealityViewNode->GetPlayerIPAddress()
        || this->currentXRRuntime() == vtkMRMLVirtualRealityViewNode::UndefinedXRRuntime)
       && this->MRMLVirtualRealityViewNode->GetVisibility())
   {
@@ -520,9 +567,18 @@ void qMRMLVirtualRealityViewPrivate::updateWidgetFromMRMLNoModify()
   }
 
   // Renderer properties
-  this->Renderer->SetGradientBackground(1);
-  this->Renderer->SetBackground(this->MRMLVirtualRealityViewNode->GetBackgroundColor());
-  this->Renderer->SetBackground2(this->MRMLVirtualRealityViewNode->GetBackgroundColor2());
+  if (this->MRMLVirtualRealityViewNode->GetRemoting())
+  {
+    this->Renderer->SetGradientBackground(0);
+    this->Renderer->SetBackground(0.0, 0.0, 0.0);
+    this->Renderer->SetBackgroundAlpha(0.0);
+  }
+  else
+  {
+    this->Renderer->SetGradientBackground(1);
+    this->Renderer->SetBackground(this->MRMLVirtualRealityViewNode->GetBackgroundColor());
+    this->Renderer->SetBackground2(this->MRMLVirtualRealityViewNode->GetBackgroundColor2());
+  }
 
   this->Renderer->SetTwoSidedLighting(this->MRMLVirtualRealityViewNode->GetTwoSidedLighting());
 

VirtualReality/Widgets/qMRMLVirtualRealityView_p.h

@@ -90,6 +90,8 @@ class qMRMLVirtualRealityViewPrivate: public QObject
   double stillUpdateRate();
 
   vtkMRMLVirtualRealityViewNode::XRRuntimeType currentXRRuntime() const;
+  bool currentXRRuntimeRemotingEnabled() const;
+  std::string currentXRRuntimeRemotingIPAddress() const;
 
 public slots:
   void updateWidgetFromMRML();

VirtualReality/qSlicerVirtualRealityModule.cxx

@@ -469,6 +469,16 @@ void qSlicerVirtualRealityModule::updateDefaultViewNodeFromSettings(vtkMRMLVirtu
                           settings.value("DefaultXRRuntime").toString().toUtf8().constData());
   }
   defaultViewNode->SetXRRuntime(defaultXRRuntime);
+  // Remoting
+  if (settings.contains("DefaultRemotingEnabled"))
+  {
+    defaultViewNode->SetRemoting(settings.value("DefaultRemotingEnabled").toBool());
+  }
+  // PlayerIPAddress
+  if (settings.contains("DefaultPlayerIPAddress"))
+  {
+    defaultViewNode->SetPlayerIPAddress(settings.value("DefaultPlayerIPAddress").toString().toStdString());
+  }
   settings.endGroup(); // VirtualReality
 }
 

VirtualReality/qSlicerVirtualRealityModuleWidget.cxx

@@ -93,6 +93,8 @@ void qSlicerVirtualRealityModuleWidget::setup()
 #endif
 
   connect(d->XRRuntimeComboBox, SIGNAL(currentIndexChanged(int)), this, SLOT(setVirtualRealityXRRuntime(int)));
+  connect(d->RemotingEnabledCheckBox, SIGNAL(toggled(bool)), this, SLOT(setRemotingEnabled(bool)));
+  connect(d->PlayerIPAddressLineEdit, SIGNAL(editingFinished()), this, SLOT(onPlayerIPAddressLineEditEditingFinished()));
 
   connect(d->ConnectCheckBox, SIGNAL(toggled(bool)), this, SLOT(setVirtualRealityConnected(bool)));
   connect(d->RenderingEnabledCheckBox, SIGNAL(toggled(bool)), this, SLOT(setVirtualRealityActive(bool)));
@@ -220,6 +222,25 @@ void qSlicerVirtualRealityModuleWidget::updateWidgetFromMRML()
 
   d->UpdateViewFromReferenceViewCameraButton->setEnabled(vrViewNode != nullptr
       && vrViewNode->GetReferenceViewNode() != nullptr);
+
+  // OpenXRRemoting
+  wasBlocked = d->RemotingEnabledCheckBox->blockSignals(true);
+  d->RemotingEnabledCheckBox->setChecked(vrViewNode != nullptr ? vrViewNode->GetRemoting() : false);
+  d->RemotingEnabledCheckBox->setEnabled(
+        (vrViewNode != nullptr)
+        && vrViewNode->GetXRRuntime() == vtkMRMLVirtualRealityViewNode::OpenXR
+        && !vrLogic->GetVirtualRealityConnected());
+  d->RemotingEnabledCheckBox->blockSignals(wasBlocked);
+
+  wasBlocked = d->PlayerIPAddressLineEdit->blockSignals(true);
+  d->PlayerIPAddressLineEdit->setText(
+        vrViewNode != nullptr ? QString::fromStdString(vrViewNode->GetPlayerIPAddress()) : QString());
+  d->PlayerIPAddressLineEdit->setEnabled(
+        (vrViewNode != nullptr)
+        && vrViewNode->GetXRRuntime() == vtkMRMLVirtualRealityViewNode::OpenXR
+        && vrViewNode->GetRemoting());
+  d->PlayerIPAddressLineEdit->setReadOnly(vrLogic->GetVirtualRealityConnected());
+  d->PlayerIPAddressLineEdit->blockSignals(wasBlocked);
 }
 
 
@@ -445,3 +466,26 @@ void qSlicerVirtualRealityModuleWidget::setTrackerTransformsUpdate(bool active)
     vrViewNode->SetTrackerTransformUpdate(active);
   }
 }
+
+//----------------------------------------------------------------------------
+void qSlicerVirtualRealityModuleWidget::setRemotingEnabled(bool enabled)
+{
+  vtkSlicerVirtualRealityLogic* vrLogic = vtkSlicerVirtualRealityLogic::SafeDownCast(this->logic());
+  vtkMRMLVirtualRealityViewNode* vrViewNode = vrLogic->GetVirtualRealityViewNode();
+  if (vrViewNode)
+  {
+    vrViewNode->SetRemoting(enabled);
+  }
+}
+
+//----------------------------------------------------------------------------
+void qSlicerVirtualRealityModuleWidget::onPlayerIPAddressLineEditEditingFinished()
+{
+  Q_D(qSlicerVirtualRealityModuleWidget);
+  vtkSlicerVirtualRealityLogic* vrLogic = vtkSlicerVirtualRealityLogic::SafeDownCast(this->logic());
+  vtkMRMLVirtualRealityViewNode* vrViewNode = vrLogic->GetVirtualRealityViewNode();
+  if (vrViewNode)
+  {
+    vrViewNode->SetPlayerIPAddress(d->PlayerIPAddressLineEdit->text().toStdString());
+  }
+}

VirtualReality/qSlicerVirtualRealityModuleWidget.h

@@ -54,6 +54,9 @@ public slots:
   void setControllerTransformsUpdate(bool);
   void setHMDTransformUpdate(bool);
   void setTrackerTransformsUpdate(bool);
+  // OpenXRRemoting
+  void setRemotingEnabled(bool);
+  void onPlayerIPAddressLineEditEditingFinished();
 
 protected slots:
   void updateWidgetFromMRML();