Merge pull request #981 from onaio/key_replacement_tableau_data

Move key replacement into the streaming_response for OpenDataViewset
onaio · Mar 31, 2017 · eb8fade · eb8fade
2 parents 9a4fd93 + 8c073b4
commit eb8fade
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 17 deletions.
diff --git a/onadata/apps/api/viewsets/open_data_viewset.py b/onadata/apps/api/viewsets/open_data_viewset.py
@@ -138,11 +138,7 @@ def data(self, request, **kwargs):
             csv_df_builder = CSVDataFrameBuilder(
                 xform.user.username, xform.id_string, include_images=False)
             data = csv_df_builder._format_for_dataframe(
-                DataInstanceSerializer(instances, many=True).data,
-                key_replacement_obj={
-                    'pattern': r"(/|-|\[|\])",
-                    "replacer": r"_"
-                })
+                DataInstanceSerializer(instances, many=True).data)
 
             return self._get_streaming_response(data, length)
 
@@ -156,7 +152,9 @@ def stream_json(streaming_data, length):
             yield u"["
 
             for i, d in enumerate(streaming_data, start=1):
-                yield json.dumps(d)
+                yield json.dumps({
+                    re.sub(r"\W", r"_", a): b for a, b in d.items()
+                })
                 yield "" if i == length else ","
 
             yield u"]"

diff --git a/onadata/libs/utils/csv_builder.py b/onadata/libs/utils/csv_builder.py
@@ -1,4 +1,3 @@
-import re
 from collections import OrderedDict
 from itertools import chain
 
@@ -477,12 +476,7 @@ def _update_columns_from_data(self, cursor):
                 self._reindex(key, value, self.ordered_columns,
                               record, self.dd, include_images=image_xpaths)
 
-    def _format_for_dataframe(self, cursor, key_replacement_obj=None):
-        pattern, replacer = None, None
-        if key_replacement_obj:
-            pattern = key_replacement_obj.get('pattern')
-            replacer = key_replacement_obj.get('replacer')
-
+    def _format_for_dataframe(self, cursor):
         # TODO: check for and handle empty results
         # add ordered columns for select multiples
         if self.split_select_multiples:
@@ -510,10 +504,7 @@ def _format_for_dataframe(self, cursor, key_replacement_obj=None):
             flat_dict = {}
             # re index repeats
             for key, value in record.iteritems():
-                _key = key
-                if pattern and replacer:
-                    _key = re.sub(pattern, replacer, _key)
-                reindexed = self._reindex(_key, value, self.ordered_columns,
+                reindexed = self._reindex(key, value, self.ordered_columns,
                                           record, self.dd,
                                           include_images=image_xpaths)
                 flat_dict.update(reindexed)