Merge pull request #541 from davidbrochart/nbclient

Async cell execution

.github/workflows/build.yml

@@ -15,7 +15,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout
-      uses: actions/checkout@v1
+      uses: actions/checkout@v2
+    - name: install dependencies
+      run: |
+        sudo apt update
+        sudo apt-get install -y --no-install-recommends libgbm-dev  # see https://github.com/puppeteer/puppeteer/issues/5661
     - name: Install node
       uses: actions/setup-node@v1
       with:

.github/workflows/main.yml

@@ -18,10 +18,10 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest]
-        python-version: [3.5, 3.6, 3.7]
+        python-version: [3.6, 3.7, 3.8]
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
 
     - name: Add conda to $PATH
       run: echo ::add-path::$CONDA/condabin
@@ -34,25 +34,25 @@ jobs:
     - name: Update conda
       run: |
         conda update -y -n base conda setuptools
-    
+
     - name: Init conda
       run: |
         conda init bash
         conda info -a
 
     - name: Create the conda environment
-      run: conda create -q -y -n voila-tests -c conda-forge python=$PYTHON_VERSION pip jupyterlab_pygments==0.1.0 nbconvert=5.5 pytest-cov nodejs flake8 ipywidgets matplotlib xeus-cling
+      run: conda create -q -y -n voila-tests -c conda-forge python=$PYTHON_VERSION pip jupyterlab_pygments==0.1.0 pytest-cov nodejs flake8 ipywidgets matplotlib xeus-cling
       env:
         PYTHON_VERSION: ${{ matrix.python-version }}
 
-    - name: Install dependencies 
+    - name: Install dependencies
       run: |
         source "$CONDA/etc/profile.d/conda.sh"
         conda activate voila-tests
         whereis python
         python --version
-        
-        python -m pip install ".[test]"
+
+        python -m pip install --ignore-installed ".[test]"
         cd tests/test_template; pip install .; cd ../../;
 
     - name: Flake8

.travis.yml

@@ -4,9 +4,9 @@ os:
  - osx
 env:
  matrix:
-  - PYTHON_VERSION=3.5
   - PYTHON_VERSION=3.6
   - PYTHON_VERSION=3.7
+  - PYTHON_VERSION=3.8
 before_install:
   - if [[ $TRAVIS_OS_NAME == osx ]]; then ulimit -n 2048; fi
   - if [[ $TRAVIS_OS_NAME == linux ]]; then sudo apt-get update; fi
@@ -18,10 +18,10 @@ before_install:
   - conda config --set always_yes yes --set changeps1 no
   - conda update -q conda
   - conda info -a
-  - conda create -q -n test-environment -c conda-forge python=$PYTHON_VERSION jupyterlab_pygments==0.1.0 nbconvert=5.5 pytest-cov nodejs flake8 ipywidgets matplotlib xeus-cling
+  - conda create -q -n test-environment -c conda-forge python=$PYTHON_VERSION jupyterlab_pygments==0.1.0 pytest-cov nodejs flake8 ipywidgets matplotlib xeus-cling
   - source activate test-environment
 install:
-  - pip install ".[test]"
+  - pip install --ignore-installed ".[test]"
   - cd tests/test_template; pip install .; cd ../../;
 script:
   - VOILA_TEST_DEBUG=1 VOILA_TEST_XEUS_CLING=1 py.test tests/ --async-test-timeout=240

setup.py

@@ -376,17 +376,19 @@ def get_data_files():
     },
     'install_requires': [
         'async_generator',
-        'jupyter_server>=0.1.0,<0.2.0',
-        'nbconvert>=5.5.0,<6',
+        'jupyter_server>=0.3.0',
+        'jupyter_client>=6.1.3',
+        'nbclient>=0.2.0',
+        'nbconvert==6.0.0a1',
         'jupyterlab_pygments>=0.1.0,<0.2',
         'pygments>=2.4.1,<3'  # Explicitly requiring pygments which is a second-order dependency.
                               # An older versions is generally installed already and is otherwise not updated by pip.
     ],
     'extras_require': {
         'test': [
             'mock',
-            'pytest<4',
-            'pytest-tornado',
+            'pytest',
+            'pytest-tornasync',
             'matplotlib',
             'ipywidgets'
         ]

tests/app/config_paths_test.py

@@ -26,9 +26,8 @@ def test_config_contents_manager(voila_app):
     assert voila_app.contents_manager.use_atomic_writing is False
 
 
-@pytest.mark.gen_test
-def test_template(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_template(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     assert 'test_template.css' in response.body.decode('utf-8')
     assert 'Hi Voila' in response.body.decode('utf-8')

tests/app/cwd_subdir_test.py

@@ -4,16 +4,15 @@
 
 @pytest.fixture
 def cwd_subdir_notebook_url(base_url):
-    return base_url + "/voila/render/subdir/cwd_subdir.ipynb"
+    return base_url + "voila/render/subdir/cwd_subdir.ipynb"
 
 
 @pytest.fixture
 def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
 
 
-@pytest.mark.gen_test
-def test_hello_world(http_client, cwd_subdir_notebook_url):
-    response = yield http_client.fetch(cwd_subdir_notebook_url)
+async def test_hello_world(http_server_client, cwd_subdir_notebook_url):
+    response = await http_server_client.fetch(cwd_subdir_notebook_url)
     html_text = response.body.decode('utf-8')
     assert 'check for the cwd' in html_text

tests/app/cwd_test.py

@@ -9,8 +9,7 @@ def voila_notebook(notebook_directory):
     return os.path.join(notebook_directory, 'cwd.ipynb')
 
 
-@pytest.mark.gen_test
-def test_template_cwd(http_client, base_url, notebook_directory):
-    response = yield http_client.fetch(base_url)
+async def test_template_cwd(http_server_client, base_url, notebook_directory):
+    response = await http_server_client.fetch(base_url)
     html_text = response.body.decode('utf-8')
     assert 'check for the cwd' in html_text

tests/app/execute_cpp_test.py

@@ -6,7 +6,7 @@
 
 @pytest.fixture
 def cpp_file_url(base_url):
-    return base_url + "/voila/render/print.xcpp"
+    return base_url + "voila/render/print.xcpp"
 
 
 @pytest.fixture
@@ -20,8 +20,7 @@ def voila_args(notebook_directory, voila_args_extra):
 
 
 @pytest.mark.skipif(not TEST_XEUS_CLING, reason='opt in to avoid having to install xeus-cling')
-@pytest.mark.gen_test
-def test_non_existing_kernel(http_client, cpp_file_url):
-    response = yield http_client.fetch(cpp_file_url)
+async def test_non_existing_kernel(http_server_client, cpp_file_url):
+    response = await http_server_client.fetch(cpp_file_url)
     assert response.code == 200
     assert 'Hello voila, from c++' in response.body.decode('utf-8')

tests/app/execute_test.py

@@ -1,41 +1,35 @@
 # test basics of voila running a notebook
-import pytest
-
 import tornado.web
-import tornado.gen
 
 import re
 import json
+import asyncio
 
 try:
     from unittest import mock
 except ImportError:
     import mock
 
 
-@pytest.mark.gen_test
-def test_hello_world(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_hello_world(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     html_text = response.body.decode('utf-8')
     assert 'Hi Voila' in html_text
     assert 'print(' not in html_text, 'by default the source code should be stripped'
     assert 'test_template.css' not in html_text, "test_template should not be the default"
 
 
-@pytest.mark.gen_test
-def test_no_execute_allowed(voila_app, app, http_client, base_url):
+async def test_no_execute_allowed(voila_app, app, http_server_client, base_url, http_server_port):
     assert voila_app.app is app
-    response = (yield http_client.fetch(base_url)).body.decode('utf-8')
+    response = (await http_server_client.fetch(base_url)).body.decode('utf-8')
     pattern = r"""kernelId": ["']([0-9a-zA-Z-]+)["']"""
     groups = re.findall(pattern, response)
     kernel_id = groups[0]
     print(kernel_id, base_url)
     session_id = '445edd75-c6f5-45d2-8b58-5fe8f84a7123'
-    url = '{base_url}/api/kernels/{kernel_id}/channels?session_id={session_id}'.format(
-        kernel_id=kernel_id, base_url=base_url, session_id=session_id
-    ).replace('http://', 'ws://')
-    conn = yield tornado.websocket.websocket_connect(url)
+    url = f'ws://localhost:{http_server_port[1]}{base_url}api/kernels/{kernel_id}/channels?session_id={session_id}'
+    conn = await tornado.websocket.websocket_connect(url)
 
     msg = {
         "header": {
@@ -59,8 +53,8 @@ def test_no_execute_allowed(voila_app, app, http_client, base_url):
         "channel": "shell",
     }
     with mock.patch.object(voila_app.log, 'warning') as mock_warning:
-        yield conn.write_message(json.dumps(msg))
+        await conn.write_message(json.dumps(msg))
         # make sure the warning method is called
         while not mock_warning.called:
-            yield tornado.gen.sleep(0.1)
+            await asyncio.sleep(0.1)
     mock_warning.assert_called_with('Received message of type "execute_request", which is not allowed. Ignoring.')

tests/app/image_inlining_test.py

@@ -11,9 +11,8 @@ def voila_notebook(notebook_directory):
     return os.path.join(notebook_directory, 'images.ipynb')
 
 
-@pytest.mark.gen_test
-def test_image_inlining(http_client, base_url, notebook_directory):
-    response = yield http_client.fetch(base_url)
+async def test_image_inlining(http_server_client, base_url, notebook_directory):
+    response = await http_server_client.fetch(base_url)
     html_text = response.body.decode('utf-8')
 
     assert 'data:image/svg+xml;base64,' in html_text

tests/app/many_iopub_messages_test.py

@@ -26,8 +26,7 @@ def _close():
     return client
 
 
-@pytest.mark.gen_test
-def test_template_cwd(http_client, base_url, notebook_directory):
-    response = yield http_client.fetch(base_url)
+async def test_template_cwd(http_server_client, base_url, notebook_directory):
+    response = await http_server_client.fetch(base_url)
     html_text = response.body.decode('utf-8')
     assert 'you should see me' in html_text

tests/app/nbextensions_test.py

@@ -22,9 +22,8 @@ def voila_config_file_paths_arg():
     return '--VoilaTest.config_file_paths=[%r]' % path
 
 
-@pytest.mark.gen_test
-def test_lists_extension(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_lists_extension(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     html_text = response.body.decode('utf-8')
     assert 'Hi Voila' in html_text

tests/app/no_kernelspec_test.py

@@ -3,16 +3,15 @@
 
 @pytest.fixture
 def non_existing_kernel_notebook(base_url):
-    return base_url + "/voila/render/no_kernelspec.ipynb"
+    return base_url + "voila/render/no_kernelspec.ipynb"
 
 
 @pytest.fixture
 def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory] + voila_args_extra
 
 
-@pytest.mark.gen_test
-def test_non_existing_kernel(http_client, non_existing_kernel_notebook):
-    response = yield http_client.fetch(non_existing_kernel_notebook)
+async def test_non_existing_kernel(http_server_client, non_existing_kernel_notebook):
+    response = await http_server_client.fetch(non_existing_kernel_notebook)
     assert response.code == 200
     assert 'Executing without a kernelspec' in response.body.decode('utf-8')

tests/app/no_strip_sources_test.py

@@ -11,9 +11,8 @@ def voila_args_extra():
     return ['--VoilaConfiguration.strip_sources=False']
 
 
-@pytest.mark.gen_test
-def test_no_strip_sources(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_no_strip_sources(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     html_text = response.body.decode('utf-8')
     assert 'Hi Voila' in html_text

tests/app/non_existing_kernel_test.py

@@ -3,16 +3,15 @@
 
 @pytest.fixture
 def non_existing_kernel_notebook(base_url):
-    return base_url + "/voila/render/non_existing_kernel.ipynb"
+    return base_url + "voila/render/non_existing_kernel.ipynb"
 
 
 @pytest.fixture
 def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory] + voila_args_extra
 
 
-@pytest.mark.gen_test
-def test_non_existing_kernel(http_client, non_existing_kernel_notebook):
-    response = yield http_client.fetch(non_existing_kernel_notebook)
+async def test_non_existing_kernel(http_server_client, non_existing_kernel_notebook):
+    response = await http_server_client.fetch(non_existing_kernel_notebook)
     assert response.code == 200
     assert 'non-existing kernel' in response.body.decode('utf-8')

tests/app/notebooks_test.py

@@ -7,8 +7,7 @@ def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
 
 
-@pytest.mark.gen_test
-def test_other_comms(http_client, base_url):
-    response = yield http_client.fetch(base_url + '/voila/render/other_comms.ipynb')
+async def test_other_comms(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url + 'voila/render/other_comms.ipynb')
     html_text = response.body.decode('utf-8')
     assert 'This notebook executed' in html_text

tests/app/serve_directory_test.py

@@ -10,10 +10,9 @@ def voila_args(notebook_directory, voila_args_extra):
     return ['--VoilaTest.root_dir=%r' % notebook_directory, '--VoilaTest.log_level=DEBUG'] + voila_args_extra
 
 
-@pytest.mark.gen_test
-def test_print(http_client, print_notebook_url):
+async def test_print(http_server_client, print_notebook_url):
     print(print_notebook_url)
-    response = yield http_client.fetch(print_notebook_url)
+    response = await http_server_client.fetch(print_notebook_url)
     assert response.code == 200
     assert 'Hi Voila' in response.body.decode('utf-8')
 
@@ -23,18 +22,16 @@ def voila_args_extra():
     return ['--VoilaConfiguration.extension_language_mapping={".py": "python"}']
 
 
-@pytest.mark.gen_test
-def test_print_py(http_client, print_notebook_url):
+async def test_print_py(http_server_client, print_notebook_url):
     print(print_notebook_url)
-    response = yield http_client.fetch(print_notebook_url.replace('ipynb', 'py'))
+    response = await http_server_client.fetch(print_notebook_url.replace('ipynb', 'py'))
     assert response.code == 200
     assert 'Hi Voila' in response.body.decode('utf-8')
 
 
 @pytest.mark.skipif(not TEST_XEUS_CLING, reason='opt in to avoid having to install xeus-cling')
-@pytest.mark.gen_test
-def test_print_julia_notebook(http_client, print_notebook_url):
+async def test_print_julia_notebook(http_server_client, print_notebook_url):
     print(print_notebook_url)
-    response = yield http_client.fetch(print_notebook_url.replace('.ipynb', '_cpp.ipynb'))
+    response = await http_server_client.fetch(print_notebook_url.replace('.ipynb', '_cpp.ipynb'))
     assert response.code == 200
     assert 'Hi Voila, from c++' in response.body.decode('utf-8')

tests/app/template_arg_test.py

@@ -7,9 +7,8 @@ def voila_args_extra():
     return ['--template=test_template']
 
 
-@pytest.mark.gen_test
-def test_template(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_template(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     assert 'test_template.css' in response.body.decode('utf-8')
     assert 'Hi Voila' in response.body.decode('utf-8')

tests/app/template_cli_test.py

@@ -13,8 +13,7 @@ def voila_args_extra():
     return ['--template=None', '--Voila.nbconvert_template_paths=[%r, %r]' % (path_test_template, path_default)]
 
 
-@pytest.mark.gen_test
-def test_template_test(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_template_test(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     assert 'test_template.css' in response.body.decode('utf-8')

tests/app/template_config_file_test.py

@@ -13,9 +13,8 @@ def voila_args_extra():
     return ['--template=test_template', '--Voila.nbconvert_template_paths=[%r, %r]' % (path_test_template, path_default)]
 
 
-@pytest.mark.gen_test
-def test_template_test(http_client, base_url):
-    response = yield http_client.fetch(base_url)
+async def test_template_test(http_server_client, base_url):
+    response = await http_server_client.fetch(base_url)
     assert response.code == 200
     assert 'test_template.css' in response.body.decode('utf-8')
     assert 'Hi Voila' in response.body.decode('utf-8')