From daf8f2ae64f9907c9e27114660659a5f5d1cb6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matthias=20Ri=C3=9Fe?= <9308656+matrss@users.noreply.github.com> Date: Thu, 21 Mar 2024 14:25:38 +0100 Subject: [PATCH] Test on macOS 13 and 14 (#2249) * Add a workflow to test on macOS 13 and 14 runners The macOS 13 runners use Intel processors while the macOS 14 runners use Apple's ARM processors. * Disable some problematic tests on macOS * Fix a potential race condition * Increase pytest timeout The test_tutorial_dir seems to be able to run into a 30 second timeout under normal operation on the macOS 14 runners, so this increases the timeout a bit to accommodate. --- .github/workflows/testing-all-oses.yml | 53 ++++++++++++++++++++++++++ pytest.ini | 2 +- tests/_test_msui/test_mscolab.py | 10 ++++- tests/_test_mswms/test_wms.py | 17 ++++++++- 4 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/testing-all-oses.yml diff --git a/.github/workflows/testing-all-oses.yml b/.github/workflows/testing-all-oses.yml new file mode 100644 index 000000000..2083af10a --- /dev/null +++ b/.github/workflows/testing-all-oses.yml @@ -0,0 +1,53 @@ +name: Test MSS + +on: + push: + branches: + - develop + - stable + - 'GSOC**' + pull_request: + branches: + - develop + - stable + - 'GSOC**' + +jobs: + test: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: ["macos-13", "macos-14", "ubuntu-latest"] + order: ["normal", "reverse"] + steps: + - uses: actions/checkout@v4 + - name: Build requirements.txt file + run: | + sed -n '/^requirements:/,/^test:/p' localbuild/meta.yaml | + sed -e "s/.*- //" | + sed -e "s/menuinst.*//" | + sed -e "s/.*://" > requirements.tmp.txt + cat requirements.d/development.txt >> requirements.tmp.txt + sed -e '/^$/d' -e '/^#.*$/d' requirements.tmp.txt > requirements.txt + rm requirements.tmp.txt + cat requirements.txt + - name: Get current year and calendar week + id: year-and-week + run: echo "year-and-week=$(date +%Y-%V)" >> "$GITHUB_OUTPUT" + - uses: mamba-org/setup-micromamba@v1 + with: + environment-file: requirements.txt + environment-name: ci + cache-environment: true + # Set the cache key in a way that the cache is invalidated every week on monday + cache-environment-key: environment-${{ steps.year-and-week.outputs.year-and-week }} + - name: Run tests + timeout-minutes: 20 + # The ignored files can somehow cause the test suite to timeout. + # I have no idea yet on why this happens and how to fix it. + # Even a module level skip is not enough, they need to be completely ignored. + # TODO: fix those tests and drop the ignores + run: micromamba run -n ci env QT_QPA_PLATFORM=offscreen pytest -v -n logical --durations=20 --cov=mslib + --ignore=tests/_test_msui/test_sideview.py --ignore=tests/_test_msui/test_topview.py --ignore=tests/_test_msui/test_wms_control.py + ${{ (matrix.order == 'normal' && ' ') || (matrix.order == 'reverse' && '--reverse') }} tests diff --git a/pytest.ini b/pytest.ini index 33d560e1e..3f0e6d3f9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -6,7 +6,7 @@ log_file = pytest.log log_file_level = DEBUG log_file_format = %(asctime)s %(levelname)s %(message)s log_file_date_format = %Y-%m-%d %H:%M:%S -timeout = 30 +timeout = 60 filterwarnings = # These namespaces are declared in a way not conformant with PEP420. Not much we can do about that here, we should keep an eye on when this is fixed in our dependencies though. ignore:Deprecated call to `pkg_resources.declare_namespace\('(xstatic|xstatic\.pkg|mpl_toolkits|mpl_toolkits\.basemap_data|sphinxcontrib|zope|fs|fs\.opener)'\)`\.:DeprecationWarning diff --git a/tests/_test_msui/test_mscolab.py b/tests/_test_msui/test_mscolab.py index 49a9f0992..acf80dc92 100644 --- a/tests/_test_msui/test_mscolab.py +++ b/tests/_test_msui/test_mscolab.py @@ -25,6 +25,7 @@ limitations under the License. """ import os +import sys import fs import fs.errors import fs.opener.errors @@ -350,6 +351,10 @@ def test_handle_export(self, mockbox, qtbot): for i in range(wp_count): assert exported_waypoints.waypoint_data(i).lat == self.window.mscolab.waypoints_model.waypoint_data(i).lat + @pytest.mark.skipif( + sys.platform == "darwin", + reason="This test is flaky on macOS because of some cleanup error in temporary files.", + ) @pytest.mark.parametrize("name", [("example.ftml", "actionImportFlightTrackFTML", 5), ("example.csv", "actionImportFlightTrackCSV", 5), ("example.txt", "actionImportFlightTrackTXT", 5), @@ -403,7 +408,10 @@ def test_browse_add_operation(self, mockopen, qtbot): self.window.mscolab.add_proj_dialog.buttonBox.Ok) with mock.patch("PyQt5.QtWidgets.QMessageBox.information") as m: QtTest.QTest.mouseClick(okWidget, QtCore.Qt.LeftButton) - m.assert_called_once() + + def assert_(): + m.assert_called_once() + qtbot.wait_until(assert_) def assert_(): assert self.window.listOperationsMSC.model().rowCount() == 1 diff --git a/tests/_test_mswms/test_wms.py b/tests/_test_mswms/test_wms.py index 937c900ad..ccce3301b 100644 --- a/tests/_test_mswms/test_wms.py +++ b/tests/_test_mswms/test_wms.py @@ -25,7 +25,7 @@ See the License for the specific language governing permissions and limitations under the License. """ - +import sys import os from shutil import move @@ -388,6 +388,13 @@ def test_import_error(self): assert mslib.mswms.wms.mswms_settings.__file__ is not None assert mslib.mswms.wms.mswms_auth.__file__ is not None + @pytest.mark.skipif( + sys.platform == "darwin", + reason="""\ +There is a race condition between modifying with ncap2 and asserting that the file changed where the server might not +see the change before the request is made, which leads to a failure of the following assert. +""".strip(), + ) def test_files_changed(self): def do_test(): environ = { @@ -429,6 +436,14 @@ def do_test(): "data_access", new=watch_access): do_test() + @pytest.mark.skipif( + sys.platform == "darwin", + reason="""\ +This test changes global variables (e.g. DOCS_LOCATION) which can affect other tests depending on test order +(e.g. tests/_test_mswms/test_mss_plot_driver.py::Test_VSec::test_VS_gallery_template fails consistently in reverse order +on macOS 14). +""".strip(), + ) def test_gallery(self, tmpdir): tempdir = tmpdir.mkdir("static") docsdir = tmpdir.mkdir("docs")