Fix CI, run lint, reduce end-to-end tests flakiness (#417)

* add macos tests

* add locally-generated snapshots

* check if filebrowser is already open before opening it

* add actual snapshoy from CI

* Update Playwright Snapshots

* use notebook toolbar, mask bttns on the right

* correct selector

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* run lint

* Update Playwright Snapshots

* timestampLocator -> timestamp

* wait until fonts are ready before taking a snapshot

* add maxDiffPixels

* Revert "wait until fonts are ready before taking a snapshot"

This reverts commit 9554559.

* pin jupyterlab version for e2e tests and snapshots

* remove macos tests

* rename relevant job to end-to-end test

* remove darwin/macos snapshots

* specify jupyterlab version correctly

* run lint

* add jlpm install to Lint the extension step

* remove ubuntu postfix from test report

* Make e2e tests a separate file

* add build steps

* pin version of jupyterlab in e2e test file

* introduce modifyListResponse, remove maxDiffPixels

* modifyListResponse -> setJobList

* set expected URL

* always set create_time to 1

* update list view snapshot

* run check option for lint

* run lint locally

---------

Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/build.yml

@@ -27,7 +27,8 @@ jobs:
       - name: Lint the extension
         run: |
           set -eux
-          jlpm
+          jlpm install
+          jlpm run lint:check
 
       - name: Test the extension
         run: |
@@ -87,58 +88,3 @@ jobs:
           jupyter labextension list
           jupyter labextension list 2>&1 | grep -ie "@jupyterlab/scheduler.*OK"
           python -m jupyterlab.browser_check
-
-  integration-tests:
-    name: Integration tests
-    needs: build
-    runs-on: ubuntu-latest
-
-    env:
-      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
-
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Base Setup
-        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
-
-      - name: Download extension package
-        uses: actions/download-artifact@v2
-        with:
-          name: extension-artifacts
-
-      - name: Install the extension
-        run: |
-          set -eux
-          python -m pip install "jupyterlab~=4.0" jupyter_scheduler*.whl
-
-      - name: Install dependencies
-        working-directory: ui-tests
-        env:
-          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
-        run: jlpm install
-
-      - name: Set up browser cache
-        uses: actions/cache@v2
-        with:
-          path: |
-            ${{ github.workspace }}/pw-browsers
-          key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }}
-
-      - name: Install browser
-        working-directory: ui-tests
-        run: jlpm install-chromium
-
-      - name: Execute integration tests
-        working-directory: ui-tests
-        run: jlpm test
-
-      - name: Upload Playwright Test report
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: jupyter_scheduler-playwright-tests
-          path: |
-            ui-tests/test-results
-            ui-tests/playwright-report

.github/workflows/e2e-tests.yml

@@ -0,0 +1,71 @@
+name: E2E Tests
+
+# suppress warning raised by https://github.com/jupyter/jupyter_core/pull/292
+env:
+  JUPYTER_PLATFORM_DIRS: '1'
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: '*'
+
+jobs:
+  e2e-tests:
+    name: Linux
+    runs-on: ubuntu-latest
+
+    env:
+      PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/pw-browsers
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Base Setup
+        uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1
+
+      - name: Install extension dependencies
+        run: python -m pip install -U jupyterlab==4.0.3
+
+      - name: Build the extension
+        run: |
+          set -eux
+          python -m pip install .
+
+          jupyter server extension list
+          jupyter server extension list 2>&1 | grep -ie "jupyter_scheduler.*OK"
+
+          jupyter labextension list
+          jupyter labextension list 2>&1 | grep -ie "@jupyterlab/scheduler.*OK"
+          python -m jupyterlab.browser_check
+
+      - name: Install ui-tests dependencies
+        working-directory: ui-tests
+        env:
+          PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: 1
+        run: jlpm install
+
+      - name: Set up browser cache
+        uses: actions/cache@v2
+        with:
+          path: |
+            ${{ github.workspace }}/pw-browsers
+          key: ${{ runner.os }}-${{ hashFiles('ui-tests/yarn.lock') }}
+
+      - name: Install browser
+        working-directory: ui-tests
+        run: jlpm install-chromium
+
+      - name: Execute integration tests
+        working-directory: ui-tests
+        run: jlpm test
+
+      - name: Upload Playwright Test report
+        if: always()
+        uses: actions/upload-artifact@v2
+        with:
+          name: jupyter_scheduler-playwright-tests-linux
+          path: |
+            ui-tests/test-results
+            ui-tests/playwright-report

.github/workflows/update-integration-tests.yml

@@ -38,7 +38,7 @@ jobs:
           python_version: '3.11'
 
       - name: Install dependencies
-        run: python -m pip install -U jupyterlab~=4.0
+        run: python -m pip install -U jupyterlab==4.0.3
 
       - name: Install extension
         run: |

ui-tests/helpers/SchedulerHelper.ts

@@ -1,11 +1,15 @@
 import { expect, IJupyterLabPageFixture } from '@jupyterlab/galata';
 import type { Locator, TestInfo } from '@playwright/test';
 
-export enum SELECTORS {
+enum SELECTORS {
   // tbutton = toolbar button
   CREATE_JOB_TBUTTON = 'button.jp-ToolbarButtonComponent[data-command="scheduling:create-from-notebook"][title="Create a notebook job"]',
   LAUNCHER_CARD = 'div.jp-LauncherCard[title="Notebook Jobs"]',
-  LIST_VIEW_TIMES = 'td.MuiTableCell-body:has-text(" AM"), td.MuiTableCell-body:has-text(" PM")'
+  LIST_VIEW_TIMES = 'td.MuiTableCell-body:has-text(" AM"), td.MuiTableCell-body:has-text(" PM")',
+  NOTEBOOK_TOOLBAR = '.jp-NotebookPanel-toolbar[aria-label="notebook actions"]',
+  ENABLE_DEBUGGER_TBUTTON = '.jp-DebuggerBugButton',
+  KERNEL_NAME_TBUTTON = '.jp-KernelName',
+  EXECUTION_INDICATOR_TBUTTON = '.jp-Notebook-ExecutionIndicator'
 }
 
 type SnapshotOptions = {
@@ -50,16 +54,47 @@ export class SchedulerHelper {
   ) {}
 
   /**
-   * JupyterLab launcher "Notebook Jobs" card selector
+   * JupyterLab launcher "Notebook Jobs" card locator
    */
   get launcherCard() {
     return this.page.locator(SELECTORS.LAUNCHER_CARD);
   }
 
+  /**
+   *  Locates notebook toolbar
+   */
+  get notebookToolbar() {
+    return this.page.locator(SELECTORS.NOTEBOOK_TOOLBAR);
+  }
+
+  /**
+   *  Locates "Create a notebook job" button in notebook toolbar
+   */
   get createJobTbutton() {
     return this.page.locator(SELECTORS.CREATE_JOB_TBUTTON);
   }
 
+  /**
+   *  Locates "Enable debugger" icon in notebook toolbar
+   */
+  get enableDebuggerTbutton() {
+    return this.page.locator(SELECTORS.ENABLE_DEBUGGER_TBUTTON);
+  }
+
+  /**
+   *  Locates kernel name button in notebook toolbar
+   */
+  get kernelNameTbutton() {
+    return this.page.locator(SELECTORS.KERNEL_NAME_TBUTTON);
+  }
+
+  /**
+   *  Locates execution indicator icon in notebook toolbar
+   */
+  get executionIndicatorTbutton() {
+    return this.page.locator(SELECTORS.EXECUTION_INDICATOR_TBUTTON);
+  }
+
   /**
    * Locates the previously created notebook's listing in the filebrowser.
    */
@@ -79,7 +114,7 @@ export class SchedulerHelper {
    * Locates the column of timestamps in the list view. Used to mask this column
    * during snapshot tests.
    */
-  get timestampLocator() {
+  get timestamp() {
     return this.page.locator(SELECTORS.LIST_VIEW_TIMES);
   }
 
@@ -181,6 +216,21 @@ export class SchedulerHelper {
     expect(await target.screenshot(screenshotArgs)).toMatchSnapshot(filename);
   }
 
+  async standardizeListCreateTime() {
+    await this.page.route('**/scheduler/*', async (route, req) => {
+      if (req.url().includes('max_items')) {
+        const res = await route.fetch();
+        const json = await res.json();
+        json.jobs[0].create_time = 1;
+        route.fulfill({
+          status: res.status(),
+          headers: res.headers(),
+          body: JSON.stringify(json)
+        });
+      }
+    });
+  }
+
   protected async _waitForCreateJobLoaded() {
     await this.page.waitForSelector('text=Loading …', { state: 'hidden' });
   }

ui-tests/tests/jupyter_scheduler.spec.ts

@@ -3,7 +3,7 @@ import { SchedulerHelper } from '../helpers/SchedulerHelper';
 
 enum FILENAMES {
   LAUNCHER = 'launcher.png',
-  NOTEBOOK = 'notebook-view.png',
+  NOTEBOOK_TOOLBAR = 'notebook-toolbar.png',
   FILEBROWSER_MENU = 'filebrowser-menu.png',
   // TODO: resolve this inconsistency in our frontend code. One entry point
   // includes the file extension in the job name, the other does not.
@@ -34,7 +34,14 @@ test.describe('Jupyter Scheduler', () => {
   test('shows notebook toolbar button', async () => {
     await scheduler.createNotebook();
     await expect(scheduler.createJobTbutton).toBeVisible();
-    await scheduler.assertSnapshot(FILENAMES.NOTEBOOK);
+    await scheduler.assertSnapshot(FILENAMES.NOTEBOOK_TOOLBAR, {
+      locator: scheduler.notebookToolbar,
+      mask: [
+        scheduler.enableDebuggerTbutton,
+        scheduler.kernelNameTbutton,
+        scheduler.executionIndicatorTbutton
+      ]
+    });
   });
 
   test('opens create job view from notebook toolbar', async ({ page }) => {
@@ -62,14 +69,11 @@ test.describe('Jupyter Scheduler', () => {
     await scheduler.assertSnapshot(FILENAMES.CREATE_VIEW_FROM_FILEBROWSER);
   });
 
-  test('shows newly created job in job list view', async () => {
+  test('shows newly created job in job list view', async ({ page }) => {
     await scheduler.createNotebook();
     await scheduler.createJobFromFilebrowser();
-
-    const timeStamp = scheduler.timestampLocator;
-    await scheduler.assertSnapshot(FILENAMES.LIST_VIEW, {
-      mask: [timeStamp]
-    });
+    await scheduler.standardizeListCreateTime();
+    await scheduler.assertSnapshot(FILENAMES.LIST_VIEW);
   });
 
   test.afterEach(async () => {