diff --git a/.github/workflows/kubernetes_test.yaml b/.github/workflows/kubernetes_test.yaml index eaaa2fe8c..44e88dfd8 100644 --- a/.github/workflows/kubernetes_test.yaml +++ b/.github/workflows/kubernetes_test.yaml @@ -69,6 +69,7 @@ jobs: run: | conda install --quiet --yes -c anaconda pip pip install .[dev] + playwright install - name: Download and Install Kubectl run: | mkdir -p bin @@ -149,6 +150,18 @@ jobs: with: working-directory: tests_e2e + - name: Playwright Tests + env: + KEYCLOAK_USERNAME: ${{ env.CYPRESS_EXAMPLE_USER_NAME }} + KEYCLOAK_PASSWORD: ${{ env.CYPRESS_EXAMPLE_USER_PASSWORD }} + NEBARI_FULL_URL: https://github-actions.nebari.dev/ + working-directory: tests_e2e/playwright + run: | + # create environment file + envsubst < .env.tpl > .env + # run playwright pytest tests in headed mode with the chromium browser + xvfb-run pytest --browser chromium + - name: Save Cypress screenshots and videos if: always() uses: actions/upload-artifact@v3 @@ -157,6 +170,7 @@ jobs: path: | ./tests_e2e/cypress/screenshots/ ./tests_e2e/cypress/videos/ + ./tests_e2e/playwright/videos/ - name: Deployment Pytests run: | diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4f6ceaf56..bd30590be 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,7 +6,7 @@ on: - ".github/workflows/test.yaml" - "tests/**" - "tests_deployment/**" - - "tests_e2e/**" + - "tests_e2e/cypress/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -20,7 +20,7 @@ on: - ".github/workflows/test.yaml" - "tests/**" - "tests_deployment/**" - - "tests_e2e/**" + - "tests_e2e/cypress/**" - "scripts/**" - "src/**" - "pyproject.toml" @@ -56,4 +56,4 @@ jobs: - name: Test Nebari run: | pytest --version - pytest --ignore=tests_deployment + pytest --ignore=tests_deployment --ignore=tests_e2e/playwright diff --git a/.gitignore b/.gitignore index 52e30ed77..d7c6ef5a6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +# docs +.nox +_build +.env + # setuptools scm src/_nebari/_version.py @@ -7,6 +12,9 @@ terraform.tfstate terraform.tfstate.backup .terraform.tfstate.lock.info +# tests +videos/ + # python __pycache__ build/ @@ -41,5 +49,6 @@ nebari-config.yaml .vscode/ .pytest_cache - +.ipynb_checkpoints +.DS_Store /.ruff_cache diff --git a/pyproject.toml b/pyproject.toml index 4c9cf08ac..3c9b11309 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,6 +76,7 @@ dev = [ "black==22.3.0", "dask-gateway", "diagrams", + "python-dotenv", "escapism", "flake8==3.8.4", "importlib-metadata<5.0", @@ -84,6 +85,7 @@ dev = [ "pre-commit", "pytest", "pytest-timeout", + "pytest-playwright", "grayskull", "build", "jinja2", diff --git a/tests_e2e/playwright/.env.tpl b/tests_e2e/playwright/.env.tpl new file mode 100644 index 000000000..399eff80c --- /dev/null +++ b/tests_e2e/playwright/.env.tpl @@ -0,0 +1,3 @@ +KEYCLOAK_USERNAME="USERNAME_OR_GOOGLE_EMAIL" +KEYCLOAK_PASSWORD="PASSWORD" +NEBARI_FULL_URL="https://nebari.quansight.dev/" diff --git a/tests_e2e/playwright/README.md b/tests_e2e/playwright/README.md new file mode 100644 index 000000000..99a285f3a --- /dev/null +++ b/tests_e2e/playwright/README.md @@ -0,0 +1,199 @@ +# Nebari integration testing with Playwright + + +## How does it work? + +Playwright manages interactions with any website. We are using it to interact +with a deployed Nebari instance and test the various integrations that are +included. + +For our test suite, we utilize Playwright's synchronous API. The first task +is to launch the web browser you'd like to test in. Options in our test suite +are `chromium`, `webkit`, and `firefox`. Playwright uses browser contexts to +achieve test isolation. The context can either be created by default or +manually (for the purposes of generating multiple contexts per test in the case +of admin vs user testing). Next the page on the browser is created. For all +tests this starts as a blank page, then during the test, we navigate to a given +url. This is all achieved in the `setup` method of the `Navigator` class. + +## Setup + +Install Nebari with the development requirements (which include Playwright) + +`pip install -e ".[dev]"` + +Then install playwright itself (required). + +`playwright install` + +> If you see the warning `BEWARE: your OS is not officially supported by Playwright; downloading fallback build., it is not critical.` Playwright will likely still work microsoft/playwright#15124 + +### Create environment file + +Create a copy of the `.env` template file + +```bash +cd tests_e2e/playwright +cp .env.tpl .env +``` + +Fill in the newly created `.env` file with the following values: + +* KEYCLOAK_USERNAME: Nebari username for username/password login OR Google email address or Google sign in +* KEYCLOAK_PASSWORD: Password associated with USERNAME +* NEBARI_FULL_URL: full url path including scheme to Nebari instance, e.g. "https://nebari.quansight.dev/" + +This user can be created with the following command (or you can use an existing non-root user): + +``` +nebari keycloak adduser --user --config +``` + +## Running the Playwright tests + +The playwright tests are run inside of pytest using + +```python +pytest tests_e2e/playwright/test_playwright.py +``` + +Videos of the test playback will be available in `$PWD/videos/`. +To see what is happening while the test is run, pass the `--headed` option to `pytest`. +You can also add the `--slowmo=$MILLI_SECONDS` option to add a delay before each action +by Playwright and thus slowing down the process. + +Another option is to run playwright methods outside of pytest. Both +`navigator.py` and `run_notebook.py` can be run as scripts. For example, + +```python + import os + + import dotenv + # load environment variables from .env file + dotenv.load_dotenv() + # instantiate the navigator class + nav = Navigator( + nebari_url="https://nebari.quansight.dev/", + username=os.environ["KEYCLOAK_USERNAME"], + password=os.environ["KEYCLOAK_PASSWORD"], + auth="password", + instance_name="small-instance", + headless=False, + slow_mo=100, + ) + # go through login sequence (defined by `auth` method in Navigator class) + nav.login() + # start the nebari server (defined by `instance_type` in Navigator class) + nav.start_server() + # reset the jupyterlab workspace to ensure we're starting with only the + # Launcher screen open, and we're in the root directory. + nav.reset_workspace() + # instantiate our test application + test_app = RunNotebook(navigator=nav) + # Write the sample notebook on the nebari instance + notebook_filepath_in_repo = ( + "tests_e2e/playwright/test_data/test_notebook_output.ipynb" + ) + notebook_filepath_on_nebari = "test_notebook_output.ipynb" + with open(notebook_filepath_in_repo, "r") as notebook: + test_app.nav.write_file( + filepath=notebook_filepath_on_nebari, content=notebook.read() + ) + # run a sample notebook + test_app.run_notebook( + path="nebari/tests_e2e/playwright/test_data/test_notebook_output.ipynb", + expected_output_text="success: 6", + conda_env="conda-env-default-py", + ) + # close out playwright and its associated browser handles + nav.teardown() +``` + +## Writing Playwright tests + +In general most of the testing happens through `locators` which is Playwright's +way of connecting a python object to the HTML element on the page. +The Playwright API has several mechanisms for getting a locator for an item on +the page (`get_by_role`, `get_by_text`, `get_by_label`, `get_by_placeholder`, +etc). + +```python +button = self.page.get_by_role("button", name="Sign in with Keycloak") +``` + +Once you have a handle on a locator, you can interact with it in different ways, +depending on the type of object. For example, clicking +a button: + +```python +button.click() +``` + +Occasionally you'll need to wait for things to load on the screen. We can +either wait for the page to finish loading: + +```python +self.page.wait_for_load_state("networkidle") +``` + +or we can wait for something specific to happen with the locator itself: + +```python +button.wait_for(timeout=3000, state="attached") +``` + +Note that waiting for the page to finish loading may be deceptive inside of +Jupyterlab since things may need to load _inside_ the page, not necessarily +causing network traffic - or causing several bursts network traffic, which +would incorrectly pass the `wait_for_load_state` after the first burst. + +Playwright has a built-in auto-wait feature which waits for a timeout period +for some actionable items. See https://playwright.dev/docs/actionability . + +### Workflow for creating new tests + +An example of running a new run notebook test might look like this: + +```python + import os + + import dotenv + # load environment variables from .env file + dotenv.load_dotenv() + # instantiate the navigator class + nav = Navigator( + nebari_url="https://nebari.quansight.dev/", + username=os.environ["KEYCLOAK_USERNAME"], + password=os.environ["KEYCLOAK_PASSWORD"], + auth="password", + instance_name="small-instance", + headless=False, + slow_mo=100, + ) + # go through login sequence (defined by `auth` method in Navigator class) + nav.login() + # start the nebari server (defined by `instance_type` in Navigator class) + nav.start_server() + # reset the jupyterlab workspace to ensure we're starting with only the + # Launcher screen open, and we're in the root directory. + nav.reset_workspace() + # instantiate our test application + test_app = RunNotebook(navigator=nav) + # Write the sample notebook on the nebari instance + notebook_filepath_in_repo = ( + "tests_e2e/playwright/test_data/test_notebook_output.ipynb" + ) + notebook_filepath_on_nebari = "test_notebook_output.ipynb" + with open(notebook_filepath_in_repo, "r") as notebook: + test_app.nav.write_file( + filepath=notebook_filepath_on_nebari, content=notebook.read() + ) + # run a sample notebook + test_app.run_notebook( + path="nebari/tests_e2e/playwright/test_data/test_notebook_output.ipynb", + expected_output_text="success: 6", + conda_env="conda-env-default-py", + ) + # close out playwright and its associated browser handles + nav.teardown() +``` diff --git a/tests_e2e/playwright/conftest.py b/tests_e2e/playwright/conftest.py new file mode 100644 index 000000000..bb629917f --- /dev/null +++ b/tests_e2e/playwright/conftest.py @@ -0,0 +1,59 @@ +import logging +import os +from pathlib import Path + +import dotenv +import pytest +from navigator import Navigator + +logger = logging.getLogger() + + +@pytest.fixture(scope="session") +def _navigator_session(browser_name, pytestconfig): + """Set up a navigator instance, login with username/password, start + a server. Teardown when session is complete. + Do not use this for individual tests, use `navigator` fixture + for tests.""" + dotenv.load_dotenv() + # try/except added here in attempt to reach teardown after error in + # order to close the browser context which will save the video so I debug + # the error. + try: + nav = Navigator( + nebari_url=os.environ["NEBARI_FULL_URL"], + username=os.environ["KEYCLOAK_USERNAME"], + password=os.environ["KEYCLOAK_PASSWORD"], + headless=not pytestconfig.getoption("--headed"), + slow_mo=pytestconfig.getoption("--slowmo"), + browser=browser_name, + auth="password", + instance_name="small-instance", # small-instance included by default + video_dir="videos/", + ) + except Exception as e: + logger.debug(e) + raise + + try: + nav.login_password() + nav.start_server() + yield nav + except Exception as e: + logger.debug(e) + raise + finally: + nav.teardown() + + +@pytest.fixture(scope="function") +def navigator(_navigator_session): + """High level navigator instance with a reset workspace.""" + _navigator_session.reset_workspace() + yield _navigator_session + + +@pytest.fixture(scope="session") +def test_data_root(): + here = Path(__file__).parent + return here / "test_data" diff --git a/tests_e2e/playwright/navigator.py b/tests_e2e/playwright/navigator.py new file mode 100644 index 000000000..d60ffdde6 --- /dev/null +++ b/tests_e2e/playwright/navigator.py @@ -0,0 +1,429 @@ +import contextlib +import datetime as dt +import logging +import os +import re +import time +import urllib + +import dotenv +from playwright.sync_api import expect, sync_playwright + +logger = logging.getLogger() + + +class Navigator: + """Base class for Nebari Playwright testing. This provides setup and + teardown methods that all tests will need and some other generally useful + methods such as clearing the workspace. Specific tests such has "Run a + notebook" are included in separate classes which use an instance of + this class. + + The Navigator class and the associated test classes are design to be able + to run either standalone, or inside of pytest. This makes it easy to + develop new tests, but also have them fully prepared to be + included as part of the test suite. + + Parameters + ---------- + nebari_url: str + Nebari URL to access for testing, e.g. "https://{nebari_url} + username: str + Login username for Nebari. For Google login, this will be email address. + password: str + Login password for Nebari. For Google login, this will be the Google + password. + auth: str + Authentication type of this Nebari instance. Options are "google" and + "password". + headless: bool + (Optional) Run the tests in headless mode (without visuals). Defaults + to False. + slow_mo: int + (Optional) Additional milliseconds to add to each Playwright command, + creating the effect of running the tests in slow motion so they are + easier for humans to follow. Defaults to 0. + browser: str + (Optional) Browser on which to run tests. Options are "chromium", + "webkit", and "firefox". Defaults to "chromium". + instance_name: str + (Optional) Server instance type on which to run tests. Options are + based on the configuration of the Nebari instance. Defaults to + "small-instance". Note that special characters (such as parenthesis) + will need to be converted to dashes. Check the HTML element to get the + exact structure. + video_dir: None or str + (Optional) Directory in which to save videos. If None, no video will + be saved. Defaults to None. + """ + + def __init__( + self, + nebari_url, + username, + password, + auth, + headless=False, + slow_mo=0, + browser="chromium", + instance_name="small-instance", + video_dir=None, + ): + self.nebari_url = nebari_url + self.username = username + self.password = password + self.auth = auth + self.initialized = False + self.headless = headless + self.slow_mo = slow_mo + self.browser = browser + self.instance_name = instance_name + self.video_dir = video_dir + + self.setup( + browser=self.browser, + headless=self.headless, + slow_mo=self.slow_mo, + ) + self.wait_for_server_spinup = 300_000 # 5 * 60 * 1_000 # 5 minutes in ms + + @property + def initialize(self): + """Ensure that the Navigator is setup and ready for testing.""" + if not self.initialized: + self.setup( + browser=self.browser, + headless=self.headless, + slow_mo=self.slow_mo, + ) + + def setup(self, browser, headless, slow_mo): + """Initial setup for running playwright. Starts playwright, creates + the browser object, a new browser context, and a new page object. + + Parameters + ---------- + browser: str + Browser on which to run tests. Options are "chromium", + "webkit", and "firefox". + headless: bool + Run the tests in headless mode (without visuals) if True + slow_mo: int + Additional milliseconds to add to each Playwright command, + creating the effect of running the tests in slow motion so they are + easier for humans to follow. + """ + logger.debug(">>> Setting up browser for Playwright") + + self.playwright = sync_playwright().start() + try: + self.browser = getattr(self.playwright, browser).launch( + headless=headless, slow_mo=slow_mo + ) + except AttributeError: + raise RuntimeError(f"{browser} browser is not recognized.") from None + self.context = self.browser.new_context( + ignore_https_errors=True, + record_video_dir=self.video_dir, + ) + self.page = self.context.new_page() + self.initialized = True + + def teardown(self) -> None: + """Shut down and close playwright. This is important to ensure that + no leftover processes are left running in the background.""" + self.context.close() + self.browser.close() # Make sure to close, so that videos are saved. + self.playwright.stop() + logger.debug(">>> Teardown complete.") + + def login(self) -> None: + """Login to nebari deployment using the auth method on the class.""" + try: + return { + "google": self.login_google, + "password": self.login_password, + }[self.auth]() + except KeyError: + raise ValueError(f"Auth type of {self.auth} is invalid.") from None + + def login_google(self) -> None: + """Go to a nebari deployment, login via Google""" + logger.debug(">>> Sign in via Google and start the server") + self.page.goto(self.nebari_url) + expect(self.page).to_have_url(re.compile(f"{self.nebari_url}*")) + + self.page.get_by_role("button", name="Sign in with Keycloak").click() + self.page.get_by_role("link", name="Google").click() + self.page.get_by_role("textbox", name="Email or phone").fill(self.username) + self.page.get_by_role("button", name="Next").click() + self.page.get_by_role("textbox", name="Enter your password").fill(self.password) + + self.page.wait_for_load_state("networkidle") + self.page.get_by_role("button", name="Next").click() + + # let the page load + self.page.wait_for_load_state("networkidle") + + def login_password(self) -> None: + """Go to a nebari deployment, login via Username/Password, and start + a new server. + """ + logger.debug(">>> Sign in via Username/Password") + self.page.goto(self.nebari_url) + expect(self.page).to_have_url(re.compile(f"{self.nebari_url}*")) + + self.page.get_by_role("button", name="Sign in with Keycloak").click() + self.page.get_by_label("Username").fill(self.username) + self.page.get_by_label("Password").click() + self.page.get_by_label("Password").fill(self.password) + self.page.get_by_role("button", name="Sign In").click() + + # let the page load + self.page.wait_for_load_state("networkidle") + + def start_server(self) -> None: + """Start a nebari server. There are several different web interfaces + possible in this process depending on if you already have a server + running or not. In order for this to work, wait for the page to load, + we look for html elements that exist when no server is running, if + they aren't visible, we check for an existing server start option. + """ + # wait for the page to load + logout_button = self.page.get_by_text("Logout", exact=True) + logout_button.wait_for(state="attached") + + # if server is not yet running + start_locator = self.page.get_by_role("button", name="Start My Server") + if start_locator.is_visible(): + start_locator.click() + + # select instance type (this will fail if this instance type is not + # available) + self.page.locator(f"#profile-item-{self.instance_name}").click() + self.page.get_by_role("button", name="Start").click() + + else: + # if the server is already running + start_locator = self.page.get_by_role( + "button", + name="My Server", + exact=True, + ) + start_locator.click() + + # wait for server spinup + self.page.wait_for_url( + urllib.parse.urljoin(self.nebari_url, f"user/{self.username}/*"), + wait_until="networkidle", + ) + + # the jupyter page loads independent of network activity so here + # we wait for the File menu to be available on the page, a proxy for + # the jupyterlab page being loaded. + file_locator = self.page.get_by_text("File", exact=True) + file_locator.wait_for( + timeout=self.wait_for_server_spinup, + state="attached", + ) + + logger.debug(">>> Sign in complete.") + + def _check_for_kernel_popup(self): + """Is the kernel popup currently open? + + Returns + ------- + True if the kernel popup is open. + """ + self.page.wait_for_load_state("networkidle") + visible = self.page.get_by_text("Select Kernel", exact=True).is_visible() + + return visible + + def reset_workspace(self): + """Reset the Jupyterlab workspace. + + * Closes all Tabs & handle possible popups for saving changes, + * make sure any kernel popups are dealt with + * reset file browser is reset to root + * Finally, ensure that the Launcher screen is showing + """ + logger.debug(">>> Reset JupyterLab workspace") + + # server is already running and there is no popup + popup = self._check_for_kernel_popup() + + # server is on running and there is a popup + if popup: + self._set_environment_via_popup(kernel=None) + + # go to Kernel menu + kernel_menuitem = self.page.get_by_text("Kernel", exact=True) + kernel_menuitem.click() + # shut down multiple running kernels + with contextlib.suppress(Exception): + shut_down_all = self.page.get_by_text( + "Shut Down All Kernels...", exact=True + ) + shut_down_all.wait_for(timeout=300, state="attached") + shut_down_all.click() + + # shut down kernel if only one notebook is running + kernel_menuitem.click() + with contextlib.suppress(Exception): + shut_down_current = self.page.get_by_text("Shut Down Kernel", exact=True) + shut_down_current.wait_for(timeout=300, state="attached") + shut_down_current.click() + + # go back to root folder + self.page.get_by_title(f"/home/{self.username}", exact=True).locator( + "path" + ).click() + + # go to File menu + self.page.get_by_text("File", exact=True).click() + # close all tabs + self.page.get_by_role("menuitem", name="Close All Tabs", exact=True).click() + + # there may be a popup to save your work, don't save + if self.page.get_by_text("Save your work", exact=True).is_visible(): + self.page.get_by_role("button", name="Discard", exact=True).click() + + # wait to ensure that the Launcher is showing + self.page.get_by_text("VS Code [↗]", exact=True).wait_for( + timeout=3000, state="attached" + ) + + def _set_environment_via_popup(self, kernel=None): + """Set the environment kernel on a jupyter notebook via the popup + dialog box. If kernel is `None`, `No Kernel` is selected and the + popup is dismissed. + + Attributes + ---------- + kernel: str or None + (Optional) name of conda environment to set. Defaults to None. + + """ + if kernel is None: + # close dialog (deal with the two formats of this dialog) + try: + self.page.get_by_text("Cancel", exact=True).click() + except Exception: + self.page.locator("div").filter(has_text="No KernelSelect").get_by_role( + "button", name="No Kernel" + ).wait_for(timeout=300, state="attached") + else: + # set the environment + # failure here indicates that the environment doesn't exist either + # because of incorrect naming syntax or because the env is still + # being built + self.page.get_by_role("combobox").nth(1).select_option( + f'{{"name":"{kernel}"}}' + ) + # click Select to close popup (deal with the two formats of this dialog) + try: + self.page.get_by_role("button", name="Select", exact=True).click() + except Exception: + self.page.locator("div").filter(has_text="No KernelSelect").get_by_role( + "button", name="Select" + ).click() + + def set_environment(self, kernel): + """Set environment of a jupyter notebook. + + IMPORTANT: The focus MUST be on the notebook on which you want to set + the environment. + + Conda environments may still be being built shortly after deployment. + + Parameters + ---------- + kernel: str + Name of kernel to set. + + Returns + ------- + None + """ + + popup = self._check_for_kernel_popup() + # if there is not a kernel popup, make it appear + if not popup: + self.page.get_by_text("Kernel", exact=True).click() + self.page.get_by_role("menuitem", name="Change Kernel…").get_by_text( + "Change Kernel…" + ).click() + + self._set_environment_via_popup(kernel) + + # wait for the jupyter UI to catch up before moving forward + # extract conda env name + conda_env_label = re.search("conda-env-(.*)-py", kernel).group(1) + # see if the jupyter notebook label for the conda env is visible + kernel_label_loc = self.page.get_by_role("button", name=conda_env_label) + if not kernel_label_loc.is_visible(): + kernel_label_loc.wait_for(state="attached") + + def open_terminal(self): + """Open Terminal in the Nebari Jupyter Lab""" + self.page.get_by_text("File", exact=True).click() + self.page.get_by_text("New", exact=True).click() + self.page.get_by_role("menuitem", name="Terminal").get_by_text( + "Terminal" + ).click() + + def run_terminal_command(self, command): + """Run a command on the terminal in the Nebari Jupyter Lab + + Parameters + ---------- + command: str + command to run in the terminal + """ + self.page.get_by_role("textbox", name="Terminal input").fill(command) + self.page.get_by_role("textbox", name="Terminal input").press("Enter") + + def write_file(self, filepath, content): + """Write a file to Nebari instance filesystem + + The terminal is a blackbox for the browser. We can't access any of the + displayed text, therefore we have no way of knowing if the commands + are done executing. For this reason, there is an unavoidable sleep + here that prevents playwright from moving on to ensure that the focus + remains on the Terminal until we are done issuing our commands. + + Parameters + ---------- + filepath: str + path to write the file on the nebari file system + content: str + text to write to that file. + """ + start = dt.datetime.now() + logger.debug(f"Writing notebook to {filepath}") + self.open_terminal() + self.run_terminal_command(f"cat <{filepath}") + self.run_terminal_command(content) + self.run_terminal_command("EOF") + self.run_terminal_command(f"ls {filepath}") + logger.debug(f"time to complete {dt.datetime.now() - start}") + time.sleep(2) + + +if __name__ == "__main__": + dotenv.load_dotenv() + nav = Navigator( + nebari_url="https://nebari.quansight.dev/", + username=os.environ["KEYCLOAK_USERNAME"], + password=os.environ["KEYCLOAK_PASSWORD"], + auth="password", + instance_name="small-instance", + headless=False, + slow_mo=100, + ) + nav.login() + nav.start_server() + nav.reset_workspace() + nav.teardown() diff --git a/tests_e2e/playwright/run_notebook.py b/tests_e2e/playwright/run_notebook.py new file mode 100644 index 000000000..351f266de --- /dev/null +++ b/tests_e2e/playwright/run_notebook.py @@ -0,0 +1,115 @@ +import contextlib +import logging +import os +from pathlib import Path + +import dotenv +from navigator import Navigator + +logger = logging.getLogger() + + +class RunNotebook: + def __init__(self, navigator: Navigator): + self.nav = navigator + self.nav.initialize + + def run_notebook( + self, path, expected_output_text, conda_env, runtime=30000, retry=2 + ): + """Run jupyter notebook and check for expected output text anywhere on + the page. + + Note: This will look for and exact match of expected_output_text + _anywhere_ on the page so be sure that your text is unique. + + Conda environments may still be being built shortly after deployment. + + conda_env: str + Name of conda environment. Python conda environments have the + structure "conda-env-nebari-git-nebari-git-dashboard-py" where + the actual name of the environment is "dashboard". + """ + logger.debug(f">>> Running notebook: {path}") + filename = Path(path).name + + # navigate to specific notebook + file_locator = self.nav.page.get_by_text("File", exact=True) + + file_locator.wait_for( + timeout=self.nav.wait_for_server_spinup, + state="attached", + ) + file_locator.click() + self.nav.page.get_by_role("menuitem", name="Open from Path…").get_by_text( + "Open from Path…" + ).click() + self.nav.page.get_by_placeholder("/path/relative/to/jlab/root").fill(path) + self.nav.page.get_by_role("button", name="Open", exact=True).click() + # give the page a second to open, otherwise the options in the kernel + # menu will be disabled. + self.nav.page.wait_for_load_state("networkidle") + if self.nav.page.get_by_text( + "Could not find path:", + exact=False, + ).is_visible(): + logger.debug("Path to notebook is invalid") + raise RuntimeError("Path to notebook is invalid") + # make sure the focus is on the dashboard tab we want to run + self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() + self.nav.set_environment(kernel=conda_env) + + # make sure that this notebook is one currently selected + self.nav.page.get_by_role("tab", name=filename).get_by_text(filename).click() + + for i in range(retry): + self._restart_run_all() + + output_locator = self.nav.page.get_by_text(expected_output_text, exact=True) + with contextlib.suppress(Exception): + if output_locator.is_visible(): + break + + def _restart_run_all(self): + # restart run all cells + self.nav.page.get_by_text("Kernel", exact=True).click() + self.nav.page.get_by_role( + "menuitem", name="Restart Kernel and Run All Cells…" + ).get_by_text("Restart Kernel and Run All Cells…").click() + + # Restart dialog appears most, but not all of the time (e.g. set + # No Kernel, then Restart Run All) + restart_dialog_button = self.nav.page.get_by_role( + "button", name="Restart", exact=True + ) + if restart_dialog_button.is_visible(): + restart_dialog_button.click() + + +if __name__ == "__main__": + dotenv.load_dotenv() + nav = Navigator( + nebari_url="https://nebari.quansight.dev/", + username=os.environ["KEYCLOAK_USERNAME"], + password=os.environ["KEYCLOAK_PASSWORD"], + auth="password", + instance_name="small-instance", + headless=False, + slow_mo=100, + ) + nav.login() + nav.start_server() + nav.reset_workspace() + test_app = RunNotebook(navigator=nav) + notebook_filepath_in_repo = "test_data/test_notebook_output.ipynb" + notebook_filepath_on_nebari = "test_notebook_output.ipynb" + with open(notebook_filepath_in_repo, "r") as notebook: + test_app.nav.write_file( + filepath=notebook_filepath_on_nebari, content=notebook.read() + ) + test_app.run_notebook( + path="nebari/tests_e2e/playwright/test_data/test_notebook_output.ipynb", + expected_output_text="success: 6", + conda_env="conda-env-default-py", + ) + nav.teardown() diff --git a/tests_e2e/playwright/test_data/test_notebook_output.ipynb b/tests_e2e/playwright/test_data/test_notebook_output.ipynb new file mode 100644 index 000000000..47768a92b --- /dev/null +++ b/tests_e2e/playwright/test_data/test_notebook_output.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "2bc13e3d-e873-47c7-87cb-f32616223799", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "import math" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96086b64-43c1-40d2-bef8-39ec98e49379", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "foo = 3" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "564c0da2-e9cf-4a93-a782-caad2eafba1b", + "metadata": { + "tags": [] + }, + "outputs": [], + "source": [ + "print(f'success: { foo + foo}')" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests_e2e/playwright/test_playwright.py b/tests_e2e/playwright/test_playwright.py new file mode 100644 index 000000000..ff636904b --- /dev/null +++ b/tests_e2e/playwright/test_playwright.py @@ -0,0 +1,14 @@ +from run_notebook import RunNotebook + + +def test_notebook(navigator, test_data_root): + test_app = RunNotebook(navigator=navigator) + notebook_name = "test_notebook_output.ipynb" + with open(test_data_root / notebook_name, "r") as notebook: + test_app.nav.write_file(filepath=notebook_name, content=notebook.read()) + test_app.run_notebook( + path=notebook_name, + expected_output_text="success: 6", + conda_env="conda-env-default-py", + runtime=60000, + )