Fix validation

requirements-min.txt

@@ -1,6 +1,6 @@
 # these minimum requirements specify '==' for testing; setup.py replaces '==' with '>='
 h5py==2.9  # support for setting attrs to lists of utf-8 added in 2.9
-hdmf==1.5.4,<2
+hdmf==1.6.0,<2
 numpy==1.16
 pandas==0.23
 python-dateutil==2.7

requirements.txt

@@ -1,5 +1,5 @@
 h5py==2.10.0
-hdmf==1.5.4
+hdmf==1.6.0
 numpy==1.18.1
 pandas==0.25.3
 python-dateutil==2.8.0

src/pynwb/testing/make_test_files.py

@@ -48,3 +48,13 @@ def make_nwbfile_str_pub():
     make_nwbfile()
     make_nwbfile_str_experimenter()
     make_nwbfile_str_pub()
+
+
+# 1.2.1_extension.nwb was generated using the following:
+# download ndx-miniscope.namespace.yaml and ndx-miniscope.extension.yaml from NDX Catalog
+# from pynwb import load_namespaces, get_class
+# load_namespaces('ndx-miniscope.namespace.yaml')
+# Miniscope = get_class('Miniscope', 'ndx-miniscope')
+# nwbfile.add_device(Miniscope('ADDME'))
+# test_name = 'extension'
+# _write(test_name, nwbfile)

src/pynwb/validate.py

@@ -6,7 +6,7 @@
 from hdmf.build import BuildManager
 from hdmf.build import TypeMap as TypeMap
 
-from pynwb import validate, available_namespaces, NWBHDF5IO
+from pynwb import validate, CORE_NAMESPACE, NWBHDF5IO
 from pynwb.spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace
 
 
@@ -35,14 +35,14 @@ def main():
 
     parser = ArgumentParser(description="Validate an NWB file", epilog=ep)
     parser.add_argument("paths", type=str, nargs='+', help="NWB file paths")
-    parser.add_argument('-p', '--nspath', type=str, help="the path to the namespace YAML file")
-    parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against")
+    parser.add_argument('-p', '--nspath', type=str, help="The path to the namespace YAML file")
+    parser.add_argument("-n", "--ns", type=str, help="The name of the namespace to validate against")
 
     feature_parser = parser.add_mutually_exclusive_group(required=False)
     feature_parser.add_argument("--cached-namespace", dest="cached_namespace", action='store_true',
-                                help="Use the cached namespace (default).")
+                                help="Validate against the cached namespaces (default).")
     feature_parser.add_argument('--no-cached-namespace', dest="cached_namespace", action='store_false',
-                                help="Don't use the cached namespace.")
+                                help="Don't validate against the cached namespaces.")
     parser.set_defaults(cached_namespace=True)
 
     args = parser.parse_args()
@@ -54,18 +54,17 @@ def main():
             sys.exit(1)
 
         if args.cached_namespace:
-            print("Turning off validation against cached namespace information"
-                  "as --nspath was passed.", file=sys.stderr)
+            print("Turning off validation against cached namespace information as --nspath was passed.",
+                  file=sys.stderr)
             args.cached_namespace = False
 
     for path in args.paths:
-
         if not os.path.isfile(path):
             print("The file {} does not exist.".format(path), file=sys.stderr)
             ret = 1
             continue
 
-        if args.cached_namespace:
+        if args.cached_namespace:  # validate against the cached namespaces
             catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
             ns_deps = NWBHDF5IO.load_namespaces(catalog, path)
             s = set(ns_deps.keys())       # determine which namespaces are the most
@@ -76,13 +75,13 @@ def main():
                 tm = TypeMap(catalog)
                 manager = BuildManager(tm)
                 specloc = "cached namespace information"
-            else:
+            else:  # no namespaces -- validate against NWB core namespace
                 manager = None
-                namespaces = available_namespaces()
+                namespaces = [CORE_NAMESPACE]
                 specloc = "pynwb namespace information"
                 print("The file {} has no cached namespace information. "
                       "Falling back to {}.".format(path, specloc), file=sys.stderr)
-        elif args.nspath:
+        elif args.nspath:  # validate against a given namespace YAML file (e.g., from an extension)
             catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
             namespaces = catalog.load_namespaces(args.nspath)
 
@@ -93,22 +92,22 @@ def main():
             tm = TypeMap(catalog)
             manager = BuildManager(tm)
             specloc = "--nspath namespace information"
-        else:
+        else:  # validate against NWB core namespace
             manager = None
-            namespaces = available_namespaces()
+            namespaces = [CORE_NAMESPACE]
             specloc = "pynwb namespace information"
 
-        if args.ns:
+        if args.ns:  # validate against a particular namespace out of the ones provided
             if args.ns in namespaces:
                 namespaces = [args.ns]
             else:
-                print("The namespace {} could not be found in {}.".format(args.ns, specloc), file=sys.stderr)
+                print("The namespace '{}' could not be found in {}.".format(args.ns, specloc), file=sys.stderr)
                 ret = 1
                 continue
 
         with NWBHDF5IO(path, mode='r', manager=manager) as io:
             for ns in namespaces:
-                print("Validating {} against {} using namespace {}.".format(path, specloc, ns))
+                print("Validating {} against {} using namespace '{}'.".format(path, specloc, ns))
                 ret = ret or _validate_helper(io=io, namespace=ns)
 
     sys.exit(ret)

test.py

@@ -12,7 +12,7 @@
 import unittest
 from tests.coloredtestrunner import ColoredTestRunner, ColoredTestResult
 
-flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5}
+flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5, 'validation': 6}
 
 TOTAL = 0
 FAILURES = 0
@@ -149,10 +149,13 @@ def main():
                         help='run example tests')
     parser.add_argument('-b', '--backwards', action='append_const', const=flags['backwards'], dest='suites',
                         help='run backwards compatibility tests')
+    parser.add_argument('-w', '--validation', action='append_const', const=flags['validation'], dest='suites',
+                        help='run validation tests')
     args = parser.parse_args()
     if not args.suites:
         args.suites = list(flags.values())
         args.suites.pop(args.suites.index(flags['example']))  # remove example as a suite run by default
+        args.suites.pop(args.suites.index(flags['validation']))  # remove validation as a suite run by default
 
     # set up logger
     root = logging.getLogger()
@@ -186,6 +189,9 @@ def main():
     if flags['backwards'] in args.suites:
         run_test_suite("tests/back_compat", "pynwb backwards compatibility tests", verbose=args.verbosity)
 
+    if flags['validation'] in args.suites:
+        run_test_suite("tests/validation", "pynwb validation tests", verbose=args.verbosity)
+
     final_message = 'Ran %s tests' % TOTAL
     exitcode = 0
     if ERRORS > 0 or FAILURES > 0:

tests/validation/test_validate.py

@@ -0,0 +1,139 @@
+import subprocess
+import re
+
+from pynwb.testing import TestCase
+from pynwb import validate, NWBHDF5IO
+
+
+class TestValidateScript(TestCase):
+
+    # 1.0.2_nwbfile.nwb has no cached specifications
+    # 1.0.3_nwbfile.nwb has cached "core" specification
+    # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications
+
+    def test_validate_file_no_cache(self):
+        """Test that validating a file with no cached spec against the core namespace succeeds."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.0.2_nwbfile.nwb",
+                                capture_output=True)
+
+        stderr_regex = re.compile(
+            r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*"
+            r"warnings.warn\(msg\)\s*"
+            r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. "
+            r"Falling back to pynwb namespace information\.\s*"
+        )
+        self.assertRegex(result.stderr.decode('utf-8'), stderr_regex)
+
+        stdout_regex = re.compile(
+            r"Validating tests/back_compat/1\.0\.2_nwbfile\.nwb against pynwb namespace information using namespace "
+            r"'core'\.\s* - no errors found\.\s*")
+        self.assertRegex(result.stdout.decode('utf-8'), stdout_regex)
+
+    def test_validate_file_no_cache_bad_ns(self):
+        """Test that validating a file with no cached spec against a specified, unknown namespace fails."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.0.2_nwbfile.nwb --ns notfound",
+                                capture_output=True)
+
+        stderr_regex = re.compile(
+            r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*"
+            r"warnings.warn\(msg\)\s*"
+            r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. "
+            r"Falling back to pynwb namespace information\.\s*"
+            r"The namespace 'notfound' could not be found in pynwb namespace information\.\s*"
+        )
+        self.assertRegex(result.stderr.decode('utf-8'), stderr_regex)
+
+        self.assertEqual(result.stdout.decode('utf-8'), '')
+
+    def test_validate_file_cached(self):
+        """Test that validating a file with cached spec against its cached namespace succeeds."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb",
+                                capture_output=True)
+        breakpoint()
+
+        self.assertEqual(result.stderr.decode('utf-8'), '')
+
+        stdout_regex = re.compile(
+            r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against cached namespace information using namespace "
+            r"'core'\.\s* - no errors found\.\s*")
+        self.assertRegex(result.stdout.decode('utf-8'), stdout_regex)
+
+    def test_validate_file_cached_bad_ns(self):
+        """Test that validating a file with cached spec against a specified, unknown namespace fails."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --ns notfound",
+                                capture_output=True)
+
+        stderr_regex = re.compile(
+            r"The namespace 'notfound' could not be found in cached namespace information\.\s*"
+        )
+        self.assertRegex(result.stderr.decode('utf-8'), stderr_regex)
+
+        self.assertEqual(result.stdout.decode('utf-8'), '')
+
+    def test_validate_file_cached_hdmf_common(self):
+        """Test that validating a file with cached spec against the hdmf-common namespace fails."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --ns hdmf-common",
+                                capture_output=True)
+
+        stderr_regex = re.compile(
+            r".*ValueError: data type \'NWBFile\' not found in namespace hdmf-common.\s*",
+            re.DOTALL
+        )
+        self.assertRegex(result.stderr.decode('utf-8'), stderr_regex)
+
+        stdout_regex = re.compile(
+            r"Validating tests/back_compat/1.1.2_nwbfile.nwb against cached namespace information using namespace "
+            r"'hdmf-common'.\s*"
+        )
+        self.assertRegex(result.stdout.decode('utf-8'), stdout_regex)
+
+    def test_validate_file_cached_ignore(self):
+        """Test that validating a file with cached spec against the core namespace succeeds."""
+        result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --no-cached-namespace",
+                                capture_output=True)
+
+        self.assertEqual(result.stderr.decode('utf-8'), '')
+
+        stdout_regex = re.compile(
+            r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against pynwb namespace information using namespace "
+            r"'core'\.\s* - no errors found\.\s*")
+        self.assertRegex(result.stdout.decode('utf-8'), stdout_regex)
+
+
+class TestValidateFunction(TestCase):
+
+    # 1.0.2_nwbfile.nwb has no cached specifications
+    # 1.0.3_nwbfile.nwb has cached "core" specification
+    # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications
+
+    def test_validate_file_no_cache(self):
+        """Test that validating a file with no cached spec against the core namespace succeeds."""
+        with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io:
+            errors = validate(io)
+            self.assertEqual(errors, [])
+
+    def test_validate_file_no_cache_bad_ns(self):
+        """Test that validating a file with no cached spec against a specified, unknown namespace fails."""
+        with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io:
+            with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""):
+                validate(io, 'notfound')
+
+    def test_validate_file_cached(self):
+        """Test that validating a file with cached spec against its cached namespace succeeds."""
+        with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io:
+            errors = validate(io)
+            self.assertEqual(errors, [])
+
+    def test_validate_file_cached_bad_ns(self):
+        """Test that validating a file with cached spec against a specified, unknown namespace fails."""
+        with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io:
+            with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""):
+                validate(io, 'notfound')
+
+    def test_validate_file_cached_hdmf_common(self):
+        """Test that validating a file with cached spec against the hdmf-common namespace fails."""
+        with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io:
+            # TODO this error should not be different from the error when using the validate script above
+            msg = "builder must have data type defined with attribute 'data_type'"
+            with self.assertRaisesWith(ValueError, msg):
+                validate(io, 'hdmf-common')