diff --git a/jdaviz/app.py b/jdaviz/app.py
index 8f0d9c154e..d987af1050 100644
--- a/jdaviz/app.py
+++ b/jdaviz/app.py
@@ -2317,11 +2317,15 @@ def _on_new_viewer(self, msg, vid=None, name=None, add_layers_to_viewer=False):
             viewer=viewer, vid=vid, name=name, reference=name
         )
 
-        ref_data = self._jdaviz_helper.default_viewer._obj.state.reference_data
-        new_viewer_item['reference_data_label'] = getattr(ref_data, 'label', None)
-
-        if hasattr(viewer, 'reference'):
-            viewer.state.reference_data = ref_data
+        if self.config == 'imviz':
+            # NOTE: if ever extending image rotation beyond imviz or adding non-image viewers
+            # to imviz: this currently assumes that the helper has a default_viewer and that is an
+            # image viewer
+            ref_data = self._jdaviz_helper.default_viewer._obj.state.reference_data
+            new_viewer_item['reference_data_label'] = getattr(ref_data, 'label', None)
+
+            if hasattr(viewer, 'reference'):
+                viewer.state.reference_data = ref_data
 
         new_stack_item = self._create_stack_item(
             container='gl-stack',