From 6fd2a254bf45f2d916a5447a4b73aa07abdf9328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=8Dtalo=20Epif=C3=A2nio?= Date: Tue, 16 Aug 2022 11:21:13 -0300 Subject: [PATCH] Add coverage --- README.md | 5 +++ test_nbs.py | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 118 insertions(+) create mode 100644 test_nbs.py diff --git a/README.md b/README.md index 06f5e42..bd2dc7a 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,11 @@ pip install ipytest [Nbdev](https://nbdev.fast.ai/#A-Motivating-Example) uses comments on the Jupyter notebook cells, such as `#exports`, to output the library code, and cells without comments to be executed as tests. Nbdev by itself doesn't guarantee that the tests are self-contained, that's why Ipyannotator uses [pytest](https://docs.pytest.org/en/7.1.x/) and [ipytest](https://github.com/chmp/ipytest) as dev dependencies. +### Run ipyannotator coverage + +Coverage can be executed using `pytest --cov=ipyannotator`. Coverage feature adapted [from FastAI Forum](https://forums.fast.ai/t/nbdev-code-coverage-n-tests/73993). + + ### Run ipyannotator as stand-alone web app using voila Ipyannotator can be executed as a web app using the [voila](https://github.com/voila-dashboards/voila) library. The following sections describe how to run using poetry and pip. diff --git a/test_nbs.py b/test_nbs.py new file mode 100644 index 0000000..e9f1532 --- /dev/null +++ b/test_nbs.py @@ -0,0 +1,113 @@ +# Adapted from https://forums.fast.ai/t/nbdev-code-coverage-n-tests/73993 + +import os +from pathlib import Path +import time +import glob +import nbformat +from nbdev.imports import get_config,parallel +from nbdev.export import read_nb,find_default_export,is_export,split_flags_and_code +from nbdev.test import get_all_flags,NoExportPreprocessor + +def before_test(nb): + "callback that will import modules and run cells that are not exported" + default_export=find_default_export(nb['cells']) + exports = [is_export(c, default_export) for c in nb['cells']] + imports = '' + # exclude exported, notebook2script calls etc + things_to_exclude = ['notebook2script'] + cells = [(i,c,e) for i,(c,e) in enumerate(zip(nb['cells'],exports)) if c['cell_type']=='code'] + for i,c,e in cells: + if e: + c['cell_type']='exclude' # if it's exported to the library, don't run as test + for line in split_flags_and_code(c): # but we might still need to run import statements + if 'import' in line: imports += f'{line}\n' + continue + for thing_to_exclude in things_to_exclude: # TODO: is this too coarse? maybe just exclude specific lines? + if thing_to_exclude in c['source']: + c['cell_type']='exclude' + continue + + nb['cells'].insert(0,nbformat.v4.new_code_cell(imports)) + + # import everything from modules written to by this notebook + for export in {export[0] for export in exports if export}: + export_parts=export.split('.') + b=export_parts.pop() + export_parts.insert(0,get_config().lib_name) + a='.'.join(export_parts) + src=f""" +from {a} import {b} +for o in dir({b}): + exec(f'from {a}.{b} import {{o}}')""" + nb['cells'].insert(0,nbformat.v4.new_code_cell(src)) + return nb + +# uncomment to see current nbdev behaviour +# i.e. use a before test callback that does nothing +# def before_test(nb): return nb + +# If nbdev.test.test_nb knew to call our "before test" callback, the rest of this script could be just the following 3 lines +# def test_run(): +# from nbdev.cli import nbdev_test_nbs +# nbdev_test_nbs.__wrapped__() + +# until it does ... we need to duplicate a few chunks of nbdev +def _test_nb(fn, flags=None): + "Execute tests in notebook in `fn` with `flags`" + os.environ["IN_TEST"] = '1' + if flags is None: flags = [] + try: + nb = read_nb(fn) + nb = before_test(nb) # <- THIS is the only change to nbdev code + for f in get_all_flags(nb['cells']): + if f not in flags: return + ep = NoExportPreprocessor(flags, timeout=600, kernel_name='python3') + pnb = nbformat.from_dict(nb) + ep.preprocess(pnb) + finally: os.environ.pop("IN_TEST") + +def _test_one(fname, flags=None, verbose=True): + print(f"testing: {fname}") + start = time.time() + try: + _test_nb(fname, flags=flags) + return True,time.time()-start + except Exception as e: + if "Kernel died before replying to kernel_info" in str(e): + time.sleep(random.random()) + _test_one(fname, flags=flags) + if verbose: print(f'Error in {fname}:\n{e}') + return False,time.time()-start + +def nbdev_test_nbs(fname=None,flags=None,n_workers=None,verbose=True,timing=False): + """ + fname:Param("A notebook name or glob to convert", str)=None, + flags:Param("Space separated list of flags", str)=None, + n_workers:Param("Number of workers to use", int)=None, + verbose:Param("Print errors along the way", bool)=True, + timing:Param("Timing each notebook to see the ones are slow", bool)=False + """ + "Test in parallel the notebooks matching `fname`, passing along `flags`" + if flags is not None: flags = flags.split(' ') + if fname is None: + files = [f for f in Path(get_config().nbs_path).glob('*.ipynb') if not f.name.startswith('_')] + else: files = glob.glob(fname) + files = [Path(f).absolute() for f in sorted(files)] + if len(files)==1 and n_workers is None: n_workers=0 + # make sure we are inside the notebook folder of the project + os.chdir(get_config().nbs_path) + results = parallel(_test_one, files, flags=flags, verbose=verbose, n_workers=n_workers) + passed,times = [r[0] for r in results],[r[1] for r in results] + if all(passed): print("All tests are passing!") + else: + msg = "The following notebooks failed:\n" + raise Exception(msg + '\n'.join([f.name for p,f in zip(passed,files) if not p])) + if timing: + for i,t in sorted(enumerate(times), key=lambda o:o[1], reverse=True): + print(f"Notebook {files[i].name} took {int(t)} seconds") + +def test_run(): + # now we can "nbdev_test_nbs" and have our "before test" callback called + # nbdev_test_nbs('00_core.ipynb') # Use this line to test a single notebook + nbdev_test_nbs(flags='distributed', n_workers=1)