diff --git a/requirements-min.txt b/requirements-min.txt index d462a1954..92cab9ef8 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -1,6 +1,6 @@ # these minimum requirements specify '==' for testing; setup.py replaces '==' with '>=' h5py==2.9 # support for setting attrs to lists of utf-8 added in 2.9 -hdmf==1.5.4,<2 +hdmf==1.6.0,<2 numpy==1.16 pandas==0.23 python-dateutil==2.7 diff --git a/requirements.txt b/requirements.txt index 3d8d7d180..e86a9a1da 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ h5py==2.10.0 -hdmf==1.5.4 +hdmf==1.6.0 numpy==1.18.1 pandas==0.25.3 python-dateutil==2.8.0 diff --git a/src/pynwb/testing/make_test_files.py b/src/pynwb/testing/make_test_files.py index 161db19d0..ae6a0c0d6 100644 --- a/src/pynwb/testing/make_test_files.py +++ b/src/pynwb/testing/make_test_files.py @@ -48,3 +48,13 @@ def make_nwbfile_str_pub(): make_nwbfile() make_nwbfile_str_experimenter() make_nwbfile_str_pub() + + +# 1.2.1_extension.nwb was generated using the following: +# download ndx-miniscope.namespace.yaml and ndx-miniscope.extension.yaml from NDX Catalog +# from pynwb import load_namespaces, get_class +# load_namespaces('ndx-miniscope.namespace.yaml') +# Miniscope = get_class('Miniscope', 'ndx-miniscope') +# nwbfile.add_device(Miniscope('ADDME')) +# test_name = 'extension' +# _write(test_name, nwbfile) diff --git a/src/pynwb/validate.py b/src/pynwb/validate.py index 37abd9076..1383b5487 100644 --- a/src/pynwb/validate.py +++ b/src/pynwb/validate.py @@ -6,7 +6,7 @@ from hdmf.build import BuildManager from hdmf.build import TypeMap as TypeMap -from pynwb import validate, available_namespaces, NWBHDF5IO +from pynwb import validate, CORE_NAMESPACE, NWBHDF5IO from pynwb.spec import NWBDatasetSpec, NWBGroupSpec, NWBNamespace @@ -35,14 +35,14 @@ def main(): parser = ArgumentParser(description="Validate an NWB file", epilog=ep) parser.add_argument("paths", type=str, nargs='+', help="NWB file paths") - parser.add_argument('-p', '--nspath', type=str, help="the path to the namespace YAML file") - parser.add_argument("-n", "--ns", type=str, help="the namespace to validate against") + parser.add_argument('-p', '--nspath', type=str, help="The path to the namespace YAML file") + parser.add_argument("-n", "--ns", type=str, help="The name of the namespace to validate against") feature_parser = parser.add_mutually_exclusive_group(required=False) feature_parser.add_argument("--cached-namespace", dest="cached_namespace", action='store_true', - help="Use the cached namespace (default).") + help="Validate against the cached namespaces (default).") feature_parser.add_argument('--no-cached-namespace', dest="cached_namespace", action='store_false', - help="Don't use the cached namespace.") + help="Don't validate against the cached namespaces.") parser.set_defaults(cached_namespace=True) args = parser.parse_args() @@ -54,18 +54,17 @@ def main(): sys.exit(1) if args.cached_namespace: - print("Turning off validation against cached namespace information" - "as --nspath was passed.", file=sys.stderr) + print("Turning off validation against cached namespace information as --nspath was passed.", + file=sys.stderr) args.cached_namespace = False for path in args.paths: - if not os.path.isfile(path): print("The file {} does not exist.".format(path), file=sys.stderr) ret = 1 continue - if args.cached_namespace: + if args.cached_namespace: # validate against the cached namespaces catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) ns_deps = NWBHDF5IO.load_namespaces(catalog, path) s = set(ns_deps.keys()) # determine which namespaces are the most @@ -76,13 +75,13 @@ def main(): tm = TypeMap(catalog) manager = BuildManager(tm) specloc = "cached namespace information" - else: + else: # no namespaces -- validate against NWB core namespace manager = None - namespaces = available_namespaces() + namespaces = [CORE_NAMESPACE] specloc = "pynwb namespace information" print("The file {} has no cached namespace information. " "Falling back to {}.".format(path, specloc), file=sys.stderr) - elif args.nspath: + elif args.nspath: # validate against a given namespace YAML file (e.g., from an extension) catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace) namespaces = catalog.load_namespaces(args.nspath) @@ -93,22 +92,22 @@ def main(): tm = TypeMap(catalog) manager = BuildManager(tm) specloc = "--nspath namespace information" - else: + else: # validate against NWB core namespace manager = None - namespaces = available_namespaces() + namespaces = [CORE_NAMESPACE] specloc = "pynwb namespace information" - if args.ns: + if args.ns: # validate against a particular namespace out of the ones provided if args.ns in namespaces: namespaces = [args.ns] else: - print("The namespace {} could not be found in {}.".format(args.ns, specloc), file=sys.stderr) + print("The namespace '{}' could not be found in {}.".format(args.ns, specloc), file=sys.stderr) ret = 1 continue with NWBHDF5IO(path, mode='r', manager=manager) as io: for ns in namespaces: - print("Validating {} against {} using namespace {}.".format(path, specloc, ns)) + print("Validating {} against {} using namespace '{}'.".format(path, specloc, ns)) ret = ret or _validate_helper(io=io, namespace=ns) sys.exit(ret) diff --git a/test.py b/test.py index 2d0b91859..82a7ec017 100755 --- a/test.py +++ b/test.py @@ -12,7 +12,7 @@ import unittest from tests.coloredtestrunner import ColoredTestRunner, ColoredTestResult -flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5} +flags = {'pynwb': 2, 'integration': 3, 'example': 4, 'backwards': 5, 'validation': 6} TOTAL = 0 FAILURES = 0 @@ -149,10 +149,13 @@ def main(): help='run example tests') parser.add_argument('-b', '--backwards', action='append_const', const=flags['backwards'], dest='suites', help='run backwards compatibility tests') + parser.add_argument('-w', '--validation', action='append_const', const=flags['validation'], dest='suites', + help='run validation tests') args = parser.parse_args() if not args.suites: args.suites = list(flags.values()) args.suites.pop(args.suites.index(flags['example'])) # remove example as a suite run by default + args.suites.pop(args.suites.index(flags['validation'])) # remove validation as a suite run by default # set up logger root = logging.getLogger() @@ -186,6 +189,9 @@ def main(): if flags['backwards'] in args.suites: run_test_suite("tests/back_compat", "pynwb backwards compatibility tests", verbose=args.verbosity) + if flags['validation'] in args.suites: + run_test_suite("tests/validation", "pynwb validation tests", verbose=args.verbosity) + final_message = 'Ran %s tests' % TOTAL exitcode = 0 if ERRORS > 0 or FAILURES > 0: diff --git a/tests/back_compat/1.2.1_extension.nwb b/tests/back_compat/1.2.1_extension.nwb new file mode 100644 index 000000000..20e94a71c Binary files /dev/null and b/tests/back_compat/1.2.1_extension.nwb differ diff --git a/tests/back_compat/1.2.1_nwbfile.nwb b/tests/back_compat/1.2.1_nwbfile.nwb new file mode 100644 index 000000000..3e0ad8860 Binary files /dev/null and b/tests/back_compat/1.2.1_nwbfile.nwb differ diff --git a/tests/validation/__init__.py b/tests/validation/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/validation/test_validate.py b/tests/validation/test_validate.py new file mode 100644 index 000000000..a939b682e --- /dev/null +++ b/tests/validation/test_validate.py @@ -0,0 +1,139 @@ +import subprocess +import re + +from pynwb.testing import TestCase +from pynwb import validate, NWBHDF5IO + + +class TestValidateScript(TestCase): + + # 1.0.2_nwbfile.nwb has no cached specifications + # 1.0.3_nwbfile.nwb has cached "core" specification + # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications + + def test_validate_file_no_cache(self): + """Test that validating a file with no cached spec against the core namespace succeeds.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.0.2_nwbfile.nwb", + capture_output=True) + + stderr_regex = re.compile( + r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" + r"warnings.warn\(msg\)\s*" + r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " + r"Falling back to pynwb namespace information\.\s*" + ) + self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) + + stdout_regex = re.compile( + r"Validating tests/back_compat/1\.0\.2_nwbfile\.nwb against pynwb namespace information using namespace " + r"'core'\.\s* - no errors found\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_no_cache_bad_ns(self): + """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.0.2_nwbfile.nwb --ns notfound", + capture_output=True) + + stderr_regex = re.compile( + r".*UserWarning: No cached namespaces found in tests/back_compat/1\.0\.2_nwbfile\.nwb\s*" + r"warnings.warn\(msg\)\s*" + r"The file tests/back_compat/1\.0\.2_nwbfile\.nwb has no cached namespace information\. " + r"Falling back to pynwb namespace information\.\s*" + r"The namespace 'notfound' could not be found in pynwb namespace information\.\s*" + ) + self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) + + self.assertEqual(result.stdout.decode('utf-8'), '') + + def test_validate_file_cached(self): + """Test that validating a file with cached spec against its cached namespace succeeds.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb", + capture_output=True) + breakpoint() + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile( + r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against cached namespace information using namespace " + r"'core'\.\s* - no errors found\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_cached_bad_ns(self): + """Test that validating a file with cached spec against a specified, unknown namespace fails.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --ns notfound", + capture_output=True) + + stderr_regex = re.compile( + r"The namespace 'notfound' could not be found in cached namespace information\.\s*" + ) + self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) + + self.assertEqual(result.stdout.decode('utf-8'), '') + + def test_validate_file_cached_hdmf_common(self): + """Test that validating a file with cached spec against the hdmf-common namespace fails.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --ns hdmf-common", + capture_output=True) + + stderr_regex = re.compile( + r".*ValueError: data type \'NWBFile\' not found in namespace hdmf-common.\s*", + re.DOTALL + ) + self.assertRegex(result.stderr.decode('utf-8'), stderr_regex) + + stdout_regex = re.compile( + r"Validating tests/back_compat/1.1.2_nwbfile.nwb against cached namespace information using namespace " + r"'hdmf-common'.\s*" + ) + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + def test_validate_file_cached_ignore(self): + """Test that validating a file with cached spec against the core namespace succeeds.""" + result = subprocess.run("python -m pynwb.validate tests/back_compat/1.1.2_nwbfile.nwb --no-cached-namespace", + capture_output=True) + + self.assertEqual(result.stderr.decode('utf-8'), '') + + stdout_regex = re.compile( + r"Validating tests/back_compat/1\.1\.2_nwbfile\.nwb against pynwb namespace information using namespace " + r"'core'\.\s* - no errors found\.\s*") + self.assertRegex(result.stdout.decode('utf-8'), stdout_regex) + + +class TestValidateFunction(TestCase): + + # 1.0.2_nwbfile.nwb has no cached specifications + # 1.0.3_nwbfile.nwb has cached "core" specification + # 1.1.2_nwbfile.nwb has cached "core" and "hdmf-common" specifications + + def test_validate_file_no_cache(self): + """Test that validating a file with no cached spec against the core namespace succeeds.""" + with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + errors = validate(io) + self.assertEqual(errors, []) + + def test_validate_file_no_cache_bad_ns(self): + """Test that validating a file with no cached spec against a specified, unknown namespace fails.""" + with NWBHDF5IO('tests/back_compat/1.0.2_nwbfile.nwb', 'r') as io: + with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): + validate(io, 'notfound') + + def test_validate_file_cached(self): + """Test that validating a file with cached spec against its cached namespace succeeds.""" + with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + errors = validate(io) + self.assertEqual(errors, []) + + def test_validate_file_cached_bad_ns(self): + """Test that validating a file with cached spec against a specified, unknown namespace fails.""" + with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + with self.assertRaisesWith(KeyError, "\"'notfound' not a namespace\""): + validate(io, 'notfound') + + def test_validate_file_cached_hdmf_common(self): + """Test that validating a file with cached spec against the hdmf-common namespace fails.""" + with NWBHDF5IO('tests/back_compat/1.1.2_nwbfile.nwb', 'r') as io: + # TODO this error should not be different from the error when using the validate script above + msg = "builder must have data type defined with attribute 'data_type'" + with self.assertRaisesWith(ValueError, msg): + validate(io, 'hdmf-common')