Merge pull request #711 from effigies/fix/missing_metadata

FIX: Lazily load metadata, skip when missing data files are missing sidecars

bids/layout/index.py

@@ -4,6 +4,7 @@
 import json
 from collections import defaultdict
 from pathlib import Path
+from functools import partial, lru_cache
 
 from bids_validator import BIDSValidator
 
@@ -264,6 +265,18 @@ def _index_metadata(self):
         # The payload is left empty for non-JSON files.
         file_data = {}
 
+        # Memoizing JSON loader
+        # Use as a function to allow lazy loading so only read JSON files
+        # if they correspond to data files that are indexed
+        @lru_cache(maxsize=None)
+        def load_json(path):
+            with open(path, 'r', encoding='utf-8') as handle:
+                try:
+                    return json.load(handle)
+                except json.JSONDecodeError as e:
+                    msg = f"Error occurred while trying to decode JSON from file {path}"
+                    raise IOError(msg) from e
+
         for bf in all_files:
             file_ents = bf.entities.copy()
             suffix = file_ents.pop('suffix', None)
@@ -274,16 +287,9 @@ def _index_metadata(self):
                 if key not in file_data:
                     file_data[key] = defaultdict(list)
 
+                payload = None
                 if ext == dot + 'json':
-                    with open(bf.path, 'r', encoding='utf-8') as handle:
-                        try:
-                            payload = json.load(handle)
-                        except json.JSONDecodeError as e:
-                            msg = ("Error occurred while trying to decode JSON"
-                                   " from file '{}'.".format(bf.path))
-                            raise IOError(msg) from e
-                else:
-                    payload = None
+                    payload = partial(load_json, bf.path)
 
                 to_store = (file_ents, payload, bf.path)
                 file_data[key][bf.dirname].append(to_store)
@@ -359,14 +365,26 @@ def create_association_pair(src, dst, kind, kind2=None):
             if not payloads:
                 continue
 
+            # Missing data files can tolerate absent metadata files,
+            # but we will try to load it anyway
+            virtual_datafile = not os.path.exists(bf.path)
+
             # Create DB records for metadata associations
             js_file = payloads[0][1]
             create_association_pair(js_file, bf.path, 'Metadata')
 
             # Consolidate metadata by looping over inherited JSON files
             file_md = {}
             for pl, js_file in payloads[::-1]:
-                file_md.update(pl)
+                try:
+                    file_md.update(pl())
+                except FileNotFoundError:
+                    if not virtual_datafile:
+                        raise
+                    # Drop metadata if any files are missing
+                    # Otherwise missing overrides could give misleading metadata
+                    file_md = {}
+                    break
 
             # Create FileAssociation records for JSON inheritance
             n_pl = len(payloads)