Merge #4455

4455: Improve instructions for db-file fixtures r=jenshnielsen a=fblanchetNaN

<!--

Thanks for submitting a pull request against QCoDeS.

To help us effectively merge your pr please consider the following check list.

- [ ] Make sure that the pull request contains a short description of the changes made.
- [ ] If you are submitting a new feature please document it. This can be in the form of inline
      docstrings, an example notebook or restructured text files.
- [ ] Please include automatic tests for the changes made when possible.

Unless your change is a small or trivial fix please add a small changelog entry:

- [ ] Create a file in the docs\changes\newsfragments folder with a short description of the change.

This file should be in the format number.categoryofcontribution. Here the number should either be the number
of the pull request. To get the number of the pull request one must
first the pull request and then subsequently update the number. The category of contribution should be
one of ``breaking``, ``new``, ``improved``, ``new_driver`` ``improved_driver``, ``underthehood``.

If this fixes a known bug reported against QCoDeS:

- [ ] Please include a string in the following form ``closes #xxx`` where ``xxx``` is the number of the bug fixed.

Please have a look at [the contributing guide](https://qcodes.github.io/Qcodes/community/contributing.html)
for more information.

If you are in doubt about any of this please ask and we will be happy to help.

-->

Improve `pytest.skip` messages when db-file are missing to point out at submodule qcodes_db_fixtures.

Explicitly recommend cloning with submodules, include the `git clone` command and options in the documentation.

Fix related error 404 in CONTRIBUTING.rst.

See QCoDeS/qcodes_generate_test_db#8

Co-authored-by: Jens H. Nielsen <[email protected]>
Co-authored-by: Florian Blanchet <[email protected]>

CONTRIBUTING.rst

@@ -75,8 +75,8 @@ Development
 Setup
 ~~~~~
 
--  Clone and register the package for development as described in the
-   `README <README.md#installation>`__
+-  Clone and register the package for development as described
+   `here <http://qcodes.github.io/Qcodes/start/index.html#installation>`__
 -  Run tests
 -  Ready to hack
 

docs/start/index.rst

@@ -76,12 +76,11 @@ Finally we install QCoDeS into this environment.
 
 Installing QCoDeS from GitHub
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Clone the QCoDeS repository from GitHub from https://github.com/QCoDeS/Qcodes
+Clone the QCoDeS repository and submodules from GitHub from https://github.com/QCoDeS/Qcodes
 
 .. code:: bash
 
-    conda create qcodesdev python=3.9
-    conda activate qcodesdev
+    git clone --recurse-submodules https://github.com/QCoDeS/Qcodes <path-to-repository>
 
 Finally install QCoDeS add the repository via
 

qcodes/tests/common.py

@@ -7,6 +7,8 @@
 from time import sleep
 from typing import TYPE_CHECKING, Any, Callable, Dict, Hashable, Optional, Tuple, Type
 
+import pytest
+
 import qcodes
 from qcodes.configuration import Config, DotDict
 from qcodes.metadatable import Metadatable
@@ -132,6 +134,15 @@ def error_caused_by(excinfo: 'ExceptionInfo[Any]', cause: str) -> bool:
         return False
 
 
+def skip_if_no_fixtures(dbname):
+    if not os.path.exists(dbname):
+        pytest.skip(
+            "No db-file fixtures found. "
+            "Make sure that your git clone of qcodes has submodules "
+            "This can be done by executing: `git submodule update --init`"
+        )
+
+
 class DumyPar(Metadatable):
 
     """Docstring for DumyPar. """

qcodes/tests/dataset/test_database_creation_and_upgrading.py

@@ -41,7 +41,7 @@
 )
 from qcodes.dataset.sqlite.queries import get_run_description, update_GUIDs
 from qcodes.dataset.sqlite.query_helpers import is_column_in_table, one
-from qcodes.tests.common import error_caused_by
+from qcodes.tests.common import error_caused_by, skip_if_no_fixtures
 from qcodes.tests.dataset.conftest import temporarily_copied_DB
 
 fixturepath = os.sep.join(qcodes.tests.dataset.__file__.split(os.sep)[:-1])
@@ -129,10 +129,7 @@ def test_perform_actual_upgrade_0_to_1():
 
     dbname_old = os.path.join(v0fixpath, 'empty.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=0) as conn:
 
@@ -158,9 +155,7 @@ def test_perform_actual_upgrade_1_to_2():
 
     dbname_old = os.path.join(v1fixpath, 'empty.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=1) as conn:
 
@@ -188,10 +183,7 @@ def test_perform_actual_upgrade_2_to_3_empty():
 
     dbname_old = os.path.join(v2fixpath, 'empty.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn:
 
@@ -218,10 +210,7 @@ def test_perform_actual_upgrade_2_to_3_empty_runs():
 
     dbname_old = os.path.join(v2fixpath, 'empty_runs.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn:
 
@@ -234,10 +223,7 @@ def test_perform_actual_upgrade_2_to_3_some_runs():
 
     dbname_old = os.path.join(v2fixpath, 'some_runs.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the"
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=2) as conn:
 
@@ -328,10 +314,7 @@ def test_perform_upgrade_v2_v3_to_v4_fixes():
 
     dbname_old = os.path.join(v3fixpath, 'some_runs_upgraded_2.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the"
-                    " https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn:
 
@@ -474,10 +457,7 @@ def test_perform_upgrade_v3_to_v4():
 
     dbname_old = os.path.join(v3fixpath, 'some_runs_upgraded_2.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn:
 
@@ -642,10 +622,7 @@ def test_perform_actual_upgrade_4_to_5(db_file):
     db_file += '.db'
     dbname_old = os.path.join(v4fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=4) as conn:
         # firstly, assert the situation with 'snapshot' column of 'runs' table
@@ -667,10 +644,7 @@ def test_perform_actual_upgrade_5_to_6():
     db_file = 'empty.db'
     dbname_old = os.path.join(fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=5) as conn:
         perform_db_upgrade_5_to_6(conn)
@@ -704,10 +678,7 @@ def test_perform_upgrade_6_7():
     db_file = 'empty.db'
     dbname_old = os.path.join(fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=6) as conn:
         perform_db_upgrade_6_to_7(conn)
@@ -721,10 +692,7 @@ def test_perform_actual_upgrade_6_to_7():
     db_file = 'some_runs.db'
     dbname_old = os.path.join(fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=6) as conn:
         assert isinstance(conn, ConnectionPlus)
@@ -780,10 +748,7 @@ def test_perform_actual_upgrade_6_to_newest_add_new_data():
     db_file = 'some_runs.db'
     dbname_old = os.path.join(fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=6) as conn:
         assert isinstance(conn, ConnectionPlus)
@@ -871,10 +836,7 @@ def test_perform_actual_upgrade_7_to_8(db_file):
     db_file += '.db'
     dbname_old = os.path.join(v7fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=7) as conn:
 
@@ -901,17 +863,14 @@ def test_latest_available_version():
     assert _latest_available_version() == 9
 
 
-@pytest.mark.parametrize('version', VERSIONS)
+@pytest.mark.parametrize("version", VERSIONS[:-1])
 def test_getting_db_version(version):
 
     fixpath = os.path.join(fixturepath, 'db_files', f'version{version}')
 
     dbname = os.path.join(fixpath, 'empty.db')
 
-    if not os.path.exists(dbname):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname)
 
     (db_v, new_v) = get_db_version_and_newest_available_version(dbname)
 
@@ -928,10 +887,7 @@ def test_perform_actual_upgrade_8_to_9(db_file):
     db_file += '.db'
     dbname_old = os.path.join(v8fixpath, db_file)
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the "
-                    "https://github.com/QCoDeS/qcodes_generate_test_db/ repo")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=8) as conn:
 

qcodes/tests/dataset/test_database_extract_runs.py

@@ -33,7 +33,7 @@
 from qcodes.dataset.sqlite.connection import path_to_dbfile
 from qcodes.dataset.sqlite.database import get_db_version_and_newest_available_version
 from qcodes.dataset.sqlite.queries import get_experiments
-from qcodes.tests.common import error_caused_by
+from qcodes.tests.common import error_caused_by, skip_if_no_fixtures
 from qcodes.tests.instrument_mocks import DummyInstrument
 
 
@@ -687,9 +687,7 @@ def test_old_versions_not_touched(two_empty_temp_db_connections,
     fixturepath = os.path.join(fixturepath,
                                'fixtures', 'db_files', 'version2',
                                'some_runs.db')
-    if not os.path.exists(fixturepath):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(fixturepath)
 
     # First test that we cannot use an old version as source
 

qcodes/tests/dataset/test_fix_functions.py

@@ -3,16 +3,19 @@
 
 import pytest
 
-from qcodes.dataset.database_fix_functions import (
-    fix_version_4a_run_description_bug, fix_wrong_run_descriptions)
+import qcodes.dataset.descriptions.versioning.serialization as serial
+import qcodes.dataset.descriptions.versioning.v0 as v0
 import qcodes.tests.dataset
-from qcodes.dataset.sqlite.db_upgrades import get_user_version
-from qcodes.dataset.sqlite.queries import get_run_description
+from qcodes.dataset.database_fix_functions import (
+    fix_version_4a_run_description_bug,
+    fix_wrong_run_descriptions,
+)
 from qcodes.dataset.descriptions.param_spec import ParamSpec
-import qcodes.dataset.descriptions.versioning.v0 as v0
-import qcodes.dataset.descriptions.versioning.serialization as serial
-from qcodes.dataset.descriptions.versioning.converters import old_to_new
 from qcodes.dataset.descriptions.rundescriber import RunDescriber
+from qcodes.dataset.descriptions.versioning.converters import old_to_new
+from qcodes.dataset.sqlite.db_upgrades import get_user_version
+from qcodes.dataset.sqlite.queries import get_run_description
+from qcodes.tests.common import skip_if_no_fixtures
 from qcodes.tests.dataset.conftest import temporarily_copied_DB
 
 fixturepath = os.sep.join(qcodes.tests.dataset.__file__.split(os.sep)[:-1])
@@ -24,9 +27,7 @@ def test_version_4a_bugfix():
 
     dbname_old = os.path.join(v4fixpath, 'some_runs.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=4) as conn:
 
@@ -54,10 +55,7 @@ def test_version_4a_bugfix_raises():
     v3fixpath = os.path.join(fixturepath, 'db_files', 'version3')
     dbname_old = os.path.join(v3fixpath, 'some_runs_without_run_description.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip(
-            "No db-file fixtures found. You can generate test db-files"
-            " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=3) as conn:
         with pytest.raises(RuntimeError):
@@ -69,10 +67,7 @@ def test_fix_wrong_run_descriptions():
 
     dbname_old = os.path.join(v3fixpath, 'some_runs_without_run_description.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip(
-            "No db-file fixtures found. You can generate test db-files"
-            " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(dbname_old)
 
     def make_ps(n):
         ps = ParamSpec(f'p{n}', label=f'Parameter {n}',
@@ -112,9 +107,7 @@ def test_fix_wrong_run_descriptions_raises():
 
     dbname_old = os.path.join(v4fixpath, 'some_runs.db')
 
-    if not os.path.exists(dbname_old):
-        pytest.skip("No db-file fixtures found. You can generate test db-files"
-                    " using the scripts in the legacy_DB_generation folder")
+    skip_if_no_fixtures(dbname_old)
 
     with temporarily_copied_DB(dbname_old, debug=False, version=4) as conn:
         with pytest.raises(RuntimeError):