diff --git a/glue/core/units.py b/glue/core/units.py
index 576772f6e..8090a4888 100644
--- a/glue/core/units.py
+++ b/glue/core/units.py
@@ -27,19 +27,19 @@ def to_unit(self, data, cid, values, target_units):
         if target_units is None:
             return values
         original_units = self._get_units(data, cid)
-        if original_units:
-            return self.converter_helper.to_unit(data, cid, values, original_units, target_units)
-        else:
+        if original_units == target_units or not original_units:
             return values
+        else:
+            return self.converter_helper.to_unit(data, cid, values, original_units, target_units)
 
     def to_native(self, data, cid, values, original_units):
         if original_units is None:
             return values
         target_units = self._get_units(data, cid)
-        if target_units:
-            return self.converter_helper.to_unit(data, cid, values, original_units, target_units)
-        else:
+        if original_units == target_units or not target_units:
             return values
+        else:
+            return self.converter_helper.to_unit(data, cid, values, original_units, target_units)
 
     def _get_units(self, data, cid):
         data = data.data if isinstance(data, Subset) else data
diff --git a/glue/viewers/image/tests/test_state.py b/glue/viewers/image/tests/test_state.py
index ce88efb0c..5ae4be992 100644
--- a/glue/viewers/image/tests/test_state.py
+++ b/glue/viewers/image/tests/test_state.py
@@ -458,3 +458,16 @@ def test_stretch_global():
 
     assert layer_state.v_min == 49.95
     assert layer_state.v_max == 949.05
+
+
+def test_attribute_units_invalid():
+
+    # Regression test for a bug that caused a crash if a dataset had an
+    # unrecognized unit
+
+    viewer_state = ImageViewerState()
+
+    data = Data(x=np.arange(100).reshape((10, 10)))
+    data.get_component('x').units = 'banana'
+
+    ImageLayerState(layer=data, viewer_state=viewer_state)