From 2ffdbe614a2f041d161038645ed6f0e5eb36409e Mon Sep 17 00:00:00 2001 From: Alexey Snigir Date: Fri, 13 Dec 2024 15:47:09 +0100 Subject: [PATCH] tests refactor --- .../action.yml | 6 +- .../test-e2e-component-library-vizro-core.yml | 1 + .github/workflows/test-e2e-vizro-ai-ui.yml | 66 ++++++++++++++ .github/workflows/test-vizro-ai-ui.yml | 87 ------------------- vizro-ai/hatch.toml | 5 +- vizro-ai/pyproject.toml | 2 + .../tests/{vizro_ai_ui => e2e}/conftest.py | 11 ++- .../{vizro_ai_ui => e2e}/test_vizro_ai_ui.py | 14 +-- vizro-ai/tests/helpers/__init__.py | 0 vizro-ai/tests/{ => tests_utils}/__init__.py | 0 .../e2e_asserts.py} | 2 +- .../e2e_constants.py} | 2 + .../common.py => tests_utils/e2e_waiters.py} | 7 +- .../genre_popularity_by_country.csv | 0 .../wait-for-it.sh | 0 vizro-ai/tests/vizro_ai_ui/__init__.py | 0 16 files changed, 92 insertions(+), 111 deletions(-) create mode 100644 .github/workflows/test-e2e-vizro-ai-ui.yml delete mode 100644 .github/workflows/test-vizro-ai-ui.yml rename vizro-ai/tests/{vizro_ai_ui => e2e}/conftest.py (82%) rename vizro-ai/tests/{vizro_ai_ui => e2e}/test_vizro_ai_ui.py (80%) delete mode 100644 vizro-ai/tests/helpers/__init__.py rename vizro-ai/tests/{ => tests_utils}/__init__.py (100%) rename vizro-ai/tests/{helpers/checkers.py => tests_utils/e2e_asserts.py} (96%) rename vizro-ai/tests/{helpers/constants.py => tests_utils/e2e_constants.py} (97%) rename vizro-ai/tests/{helpers/common.py => tests_utils/e2e_waiters.py} (74%) rename vizro-ai/tests/{vizro_ai_ui => tests_utils}/genre_popularity_by_country.csv (100%) rename vizro-ai/tests/{vizro_ai_ui => tests_utils}/wait-for-it.sh (100%) delete mode 100644 vizro-ai/tests/vizro_ai_ui/__init__.py diff --git a/.github/actions/failed-artifacts-and-slack-notifications/action.yml b/.github/actions/failed-artifacts-and-slack-notifications/action.yml index a0b1ce6de..716a8d686 100644 --- a/.github/actions/failed-artifacts-and-slack-notifications/action.yml +++ b/.github/actions/failed-artifacts-and-slack-notifications/action.yml @@ -7,8 +7,8 @@ runs: - name: Copy failed screenshots shell: bash run: | - mkdir /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/ - cd /home/runner/work/vizro/vizro/vizro-core/ + mkdir ${{ env.PROJECT_PATH }}failed_screenshots/ + cd ${{ env.PROJECT_PATH }} cp *.png failed_screenshots - name: Archive production artifacts @@ -16,7 +16,7 @@ runs: with: name: Failed screenshots path: | - /home/runner/work/vizro/vizro/vizro-core/failed_screenshots/*.png + ${{ env.PROJECT_PATH }}failed_screenshots/*.png - name: Send custom JSON data to Slack id: slack diff --git a/.github/workflows/test-e2e-component-library-vizro-core.yml b/.github/workflows/test-e2e-component-library-vizro-core.yml index 85c08a0e6..bc162f0f3 100644 --- a/.github/workflows/test-e2e-component-library-vizro-core.yml +++ b/.github/workflows/test-e2e-component-library-vizro-core.yml @@ -43,3 +43,4 @@ jobs: env: TESTS_NAME: Vizro e2e component library tests SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PROJECT_PATH: /home/runner/work/vizro/vizro/vizro-core/ diff --git a/.github/workflows/test-e2e-vizro-ai-ui.yml b/.github/workflows/test-e2e-vizro-ai-ui.yml new file mode 100644 index 000000000..581d38383 --- /dev/null +++ b/.github/workflows/test-e2e-vizro-ai-ui.yml @@ -0,0 +1,66 @@ +name: e2e tests for VizroAI UI + +defaults: + run: + working-directory: vizro-ai + +on: + push: + branches: [main] + pull_request: + branches: + - main + paths: + - "vizro-ai/**" + - "!vizro-ai/docs/**" + +env: + PYTHONUNBUFFERED: 1 + FORCE_COLOR: 1 + PYTHON_VERSION: "3.12" + +jobs: + test-e2e-vizro-ai-ui-fork: + if: ${{ github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Passed fork step + run: echo "Success!" + + test-e2e-vizro-ai-ui: + if: ${{ ! github.event.pull_request.head.repo.fork }} + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + + - name: Install Hatch + run: pip install hatch + + - name: Show dependency tree + run: hatch run pip tree + + - name: Run e2e VizroAI UI tests + run: | + hatch run vizro-ai-ui + tests/test_utils/wait-for-it.sh 127.0.0.1:8050 -t 30 + hatch run test-e2e-vizro-ai-ui + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} + + - name: Create artifacts and slack notifications + if: failure() + uses: ./.github/actions/failed-artifacts-and-slack-notifications + env: + TESTS_NAME: e2e VizroAI UI tests + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + PROJECT_PATH: /home/runner/work/vizro/vizro/vizro-ai/ diff --git a/.github/workflows/test-vizro-ai-ui.yml b/.github/workflows/test-vizro-ai-ui.yml deleted file mode 100644 index fb8999ab8..000000000 --- a/.github/workflows/test-vizro-ai-ui.yml +++ /dev/null @@ -1,87 +0,0 @@ -name: Integration tests for VizroAI UI - -defaults: - run: - working-directory: vizro-ai - -on: - push: - branches: [main] - pull_request: - branches: - - qa/component_library_tests - - main - paths: - - "vizro-ai/**" - - "!vizro-ai/docs/**" - -env: - PYTHONUNBUFFERED: 1 - FORCE_COLOR: 1 - -jobs: - test-vizro-ai-ui-fork: - if: ${{ github.event.pull_request.head.repo.fork }} - name: test-vizro-ai-ui - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Passed fork step - run: echo "Success!" - - test-vizro-ai-ui: - if: ${{ ! github.event.pull_request.head.repo.fork }} - name: test-vizro-ai-ui - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python 3.12 - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Install Hatch - run: pip install hatch - - - name: Show dependency tree - run: hatch run tests:pip tree - - - name: Run vizroAI UI tests - run: | - hatch run tests:vizro-ai-ui - tests/vizro_ai_ui/wait-for-it.sh 127.0.0.1:8050 -t 30 - hatch run tests:test-vizro-ai-ui - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }} - - - name: Copy failed screenshots - if: failure() - run: | - mkdir /home/runner/work/vizro/vizro/vizro-ai/failed_screenshots/ - cp tests*.png failed_screenshots - - - name: Archive screenshot artifacts - uses: actions/upload-artifact@v4 - if: failure() - with: - name: Failed screenshots - path: | - /home/runner/work/vizro/vizro/vizro-ai/failed_screenshots/*.png - - - name: Send custom JSON data to Slack - id: slack - uses: slackapi/slack-github-action@v1.26.0 - if: failure() - with: - payload: | - { - "text": "VizroAI UI tests build result: ${{ job.status }}\nBranch: ${{ github.head_ref }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" - } - env: - SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} - SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK diff --git a/vizro-ai/hatch.toml b/vizro-ai/hatch.toml index a73a7010c..a235665a8 100644 --- a/vizro-ai/hatch.toml +++ b/vizro-ai/hatch.toml @@ -56,7 +56,7 @@ test-unit-coverage = [ "- coverage combine", "coverage report" ] -test-vizro-ai-ui = "pytest -vs --reruns 1 tests/vizro_ai_ui --headless {args}" +test-e2e-vizro-ai-ui = "pytest -vs --reruns 1 tests/e2e/test_vizro_ai_ui.py --headless {args}" vizro-ai-ui = "python examples/dashboard_ui/app.py &" [envs.docs] @@ -83,8 +83,5 @@ serve = "mkdocs serve --open" extra-dependencies = ["pydantic==1.10.16"] python = "3.9" -[envs.tests] -python = "3.12" - [version] path = "src/vizro_ai/__init__.py" diff --git a/vizro-ai/pyproject.toml b/vizro-ai/pyproject.toml index 791d2ab72..487126df5 100644 --- a/vizro-ai/pyproject.toml +++ b/vizro-ai/pyproject.toml @@ -68,6 +68,8 @@ filterwarnings = [ # Ignore warning for Pydantic v1 API and Python 3.13: "ignore:Failing to pass a value to the 'type_params' parameter of 'typing.ForwardRef._evaluate' is deprecated:DeprecationWarning" ] +norecursedirs = ["tests/tests_utils"] +pythonpath = ["tests/tests_utils"] [tool.ruff] extend = "../pyproject.toml" diff --git a/vizro-ai/tests/vizro_ai_ui/conftest.py b/vizro-ai/tests/e2e/conftest.py similarity index 82% rename from vizro-ai/tests/vizro_ai_ui/conftest.py rename to vizro-ai/tests/e2e/conftest.py index db4e8ebe5..dc5c09cf5 100644 --- a/vizro-ai/tests/vizro_ai_ui/conftest.py +++ b/vizro-ai/tests/e2e/conftest.py @@ -3,7 +3,7 @@ import pytest from selenium import webdriver from selenium.webdriver.chrome.options import Options -from tests.helpers.checkers import browser_console_warnings_checker +from e2e_asserts import browser_console_warnings_checker @pytest.fixture() @@ -41,11 +41,10 @@ def test_failed_check(request): yield if request.node.rep_setup.failed: return "setting up a test failed!", request.node.nodeid - elif request.node.rep_setup.passed: - if request.node.rep_call.failed: - driver = request.node.funcargs["chromedriver"] - take_screenshot(driver, request.node.nodeid) - return "executing test failed", request.node.nodeid + elif request.node.rep_setup.passed and request.node.rep_call.failed: + driver = request.node.funcargs["chromedriver"] + take_screenshot(driver, request.node.nodeid) + return "executing test failed", request.node.nodeid def take_screenshot(driver, nodeid): diff --git a/vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py b/vizro-ai/tests/e2e/test_vizro_ai_ui.py similarity index 80% rename from vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py rename to vizro-ai/tests/e2e/test_vizro_ai_ui.py index d5d55423b..7bf3d2428 100644 --- a/vizro-ai/tests/vizro_ai_ui/test_vizro_ai_ui.py +++ b/vizro-ai/tests/e2e/test_vizro_ai_ui.py @@ -2,7 +2,7 @@ import pytest from selenium.common import InvalidSelectorException, TimeoutException -from tests.helpers.common import ( +from e2e_waiters import ( wait_for, webdriver_click_waiter, webdriver_waiter, @@ -11,8 +11,8 @@ @pytest.mark.filterwarnings("ignore:HTTPResponse.getheader():DeprecationWarning") -@pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning") -@pytest.mark.filterwarnings("ignore:unclosed file:ResourceWarning") +# @pytest.mark.filterwarnings("ignore::pytest.PytestUnhandledThreadExceptionWarning") +# @pytest.mark.filterwarnings("ignore:unclosed file:ResourceWarning") @pytest.mark.parametrize( "chromedriver", [({"port": 8050})], @@ -30,11 +30,11 @@ def test_chart_ui(chromedriver): # upload file file_input = webdriver_waiter_css(chromedriver, 'input[type="file"]') - file_input.send_keys(os.path.abspath("tests/vizro_ai_ui/genre_popularity_by_country.csv")) - webdriver_click_waiter(chromedriver, '//*[@id="data-upload-id"]') + file_input.send_keys(os.path.abspath("tests/tests_utils/genre_popularity_by_country.csv")) + webdriver_click_waiter(chromedriver, '//*[@id="data-upload"]') # enter prompt - prompt = webdriver_waiter(chromedriver, '//*[@id="text-area-id"]') + prompt = webdriver_waiter(chromedriver, '//*[@id="text-area"]') prompt.send_keys("Create bar graph by genre") # choose gpt version @@ -43,7 +43,7 @@ def test_chart_ui(chromedriver): webdriver_click_waiter(chromedriver, '//*/div[text()="gpt-4o-mini"]') # click run VizroAI - webdriver_click_waiter(chromedriver, '//*[@id="trigger-button-id"]') + webdriver_click_waiter(chromedriver, '//*[@id="trigger-button"]') # check result def _text_waiter(): diff --git a/vizro-ai/tests/helpers/__init__.py b/vizro-ai/tests/helpers/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/vizro-ai/tests/__init__.py b/vizro-ai/tests/tests_utils/__init__.py similarity index 100% rename from vizro-ai/tests/__init__.py rename to vizro-ai/tests/tests_utils/__init__.py diff --git a/vizro-ai/tests/helpers/checkers.py b/vizro-ai/tests/tests_utils/e2e_asserts.py similarity index 96% rename from vizro-ai/tests/helpers/checkers.py rename to vizro-ai/tests/tests_utils/e2e_asserts.py index d019f5a6d..59decd628 100644 --- a/vizro-ai/tests/helpers/checkers.py +++ b/vizro-ai/tests/tests_utils/e2e_asserts.py @@ -1,5 +1,5 @@ from hamcrest import any_of, assert_that, contains_string -from tests.helpers.constants import ( +from e2e_constants import ( INVALID_PROP_ERROR, REACT_NOT_RECOGNIZE_ERROR, REACT_RENDERING_ERROR, diff --git a/vizro-ai/tests/helpers/constants.py b/vizro-ai/tests/tests_utils/e2e_constants.py similarity index 97% rename from vizro-ai/tests/helpers/constants.py rename to vizro-ai/tests/tests_utils/e2e_constants.py index 857dfe171..211535976 100644 --- a/vizro-ai/tests/helpers/constants.py +++ b/vizro-ai/tests/tests_utils/e2e_constants.py @@ -7,3 +7,5 @@ WILLRECEIVEPROPS_RENAMED_WARNING = "componentWillReceiveProps has been renamed" READPIXELS_WARNING = "GPU stall due to ReadPixels" WEBGL_WARNING = "WebGL" # https://issues.chromium.org/issues/40277080 + +TIMEOUT = 30 diff --git a/vizro-ai/tests/helpers/common.py b/vizro-ai/tests/tests_utils/e2e_waiters.py similarity index 74% rename from vizro-ai/tests/helpers/common.py rename to vizro-ai/tests/tests_utils/e2e_waiters.py index 31f464f99..d8ceda468 100644 --- a/vizro-ai/tests/helpers/common.py +++ b/vizro-ai/tests/tests_utils/e2e_waiters.py @@ -6,6 +6,7 @@ from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions from selenium.webdriver.support.wait import WebDriverWait +from e2e_constants import TIMEOUT def wait_for(condition_function, *args): @@ -20,20 +21,20 @@ def wait_for(condition_function, *args): def webdriver_click_waiter(browserdriver, xpath): - WebDriverWait(browserdriver, 10, ignored_exceptions=StaleElementReferenceException).until( + WebDriverWait(browserdriver, TIMEOUT, ignored_exceptions=StaleElementReferenceException).until( expected_conditions.element_to_be_clickable((By.XPATH, xpath)) ).click() def webdriver_waiter(browserdriver, xpath): - elem = WebDriverWait(browserdriver, 10, ignored_exceptions=StaleElementReferenceException).until( + elem = WebDriverWait(browserdriver, TIMEOUT, ignored_exceptions=StaleElementReferenceException).until( expected_conditions.presence_of_element_located((By.XPATH, xpath)) ) return elem def webdriver_waiter_css(browserdriver, xpath): - elem = WebDriverWait(browserdriver, 30, ignored_exceptions=StaleElementReferenceException).until( + elem = WebDriverWait(browserdriver, TIMEOUT, ignored_exceptions=StaleElementReferenceException).until( expected_conditions.presence_of_element_located((By.CSS_SELECTOR, xpath)) ) return elem diff --git a/vizro-ai/tests/vizro_ai_ui/genre_popularity_by_country.csv b/vizro-ai/tests/tests_utils/genre_popularity_by_country.csv similarity index 100% rename from vizro-ai/tests/vizro_ai_ui/genre_popularity_by_country.csv rename to vizro-ai/tests/tests_utils/genre_popularity_by_country.csv diff --git a/vizro-ai/tests/vizro_ai_ui/wait-for-it.sh b/vizro-ai/tests/tests_utils/wait-for-it.sh similarity index 100% rename from vizro-ai/tests/vizro_ai_ui/wait-for-it.sh rename to vizro-ai/tests/tests_utils/wait-for-it.sh diff --git a/vizro-ai/tests/vizro_ai_ui/__init__.py b/vizro-ai/tests/vizro_ai_ui/__init__.py deleted file mode 100644 index e69de29bb..000000000