Merge pull request #149 from NWChemEx/update_testing_docs

Update Testing Docs

docs/requirements.txt

@@ -2,3 +2,4 @@ docutils<=0.16
 sphinx
 sphinx_rtd_theme
 sphinxcontrib.bibtex
+sphinx_tabs

docs/source/conf.py

@@ -36,7 +36,7 @@
 extensions = [
     'sphinx.ext.autodoc', 'sphinx.ext.doctest', 'sphinx.ext.intersphinx',
     'sphinx.ext.todo', 'sphinx.ext.coverage', 'sphinx.ext.mathjax',
-    'sphinxcontrib.bibtex'
+    'sphinxcontrib.bibtex', 'sphinx_tabs.tabs'
 ]
 dir_path = os.path.dirname(os.path.realpath(__file__))
 doc_path = os.path.dirname(dir_path)
@@ -137,7 +137,9 @@
 # -- Options for intersphinx extension ---------------------------------------
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'python': ('https://docs.python.org/3', None)}
+intersphinx_mapping = {
+    # 'python': ('https://docs.python.org/3', None)
+}
 
 # -- Options for todo extension ----------------------------------------------
 

docs/source/testing/integration.rst

@@ -18,4 +18,135 @@
 Writing Integration Tests for NWChemEx
 ######################################
 
-TODO: Write me!!!!
+NWChemEx is a modular ecosystem designed with separation of concerns as a key
+design point. An example of this separation can be found with the 
+SCF, integrals, ChemCache libraries. These components of NWX are linked by 
+SimDE and are intended to be used together, but are not explicitly required for 
+the development of the other (see :numref:`fig_deps_chart`). The unit tests for 
+these libraries are intended to ensure basic functionality and correctness, 
+which can usually be accomplished with simple test data that allow the unit 
+tests to run quickly. 
+
+.. _fig_deps_chart:
+
+.. figure:: assets/DependencyChart.png
+    :align: center
+    :scale: 50 %
+
+    A simplified diagram of the NWChemEx dependency structure. Note that plugins
+    depending on SimDE are parallel to one another, and are integrated together
+    within NWChemEx. Arrows point from a dependency to the dependent library.
+
+
+With that said, the initial development and testing of the SCF becomes very 
+awkward when one is unable to easily acquire real integrals for real molecular 
+systems. Additionally, changes to the integrals code could have deleterious
+effects on the SCF code, which we would like to detect before merging. For these
+(and other) reasons, it can be useful to implement integration tests to ensure
+the continued interoperability of the isolated components of the NWX stack.
+Because the tests are built on top of the plugins, it is simple to include
+NWChemEx itself as a dependency of the test (see :numref:`fig_integration_chart`).
+This way, changes at the plugin level can be screened to guarantee that they
+don't break interoperability with the others.
+
+.. _fig_integration_chart:
+
+.. figure:: assets/DependencyChartExtended.png
+    :align: center
+    :scale: 50 %
+
+    A diagram illustrating the relationship between the integration tests, the
+    library they test, and the top-level NWChemEx library. Arrows point from a 
+    dependency to the dependent library.
+
+
+CMake for Integration Testing
+=============================
+
+The following code-block provides an example for how one can add the option for
+an integration test to a project that uses the NWX ecosystem.
+
+.. code-block:: CMake
+
+    # Include option
+    cmaize_option_list(
+        BUILD_INTEGRATION_TESTS ON "Build the integration tests?"
+    )
+
+    # How to build the integration tests
+    if("${BUILD_INTEGRATION_TESTS}")
+        include(nwx_pybind11)
+        # Set relevant test directories
+        set(CXX_INCLUDE_DIR /path/to/cxx/includes)
+        set(CXX_TEST_DIR /path/to/cxx/integration/tests)
+        set(PYTHON_TEST_DIR /path/to/python/integration/tests)
+
+        # Build NWChemEx for the test
+        cmaize_find_or_build_dependency(
+            nwchemex
+            URL github.com/NWChemEx/NWChemEx
+            VERSION master
+            BUILD_TARGET nwchemex
+            FIND_TARGET nwx::nwchemex
+            CMAKE_ARGS BUILD_TESTING=OFF
+                       BUILD_PYBIND11_PYBINDINGS=ON
+        )
+
+        # Add integration tests
+        nwx_pybind11_tests(
+            py_test_integration_scf
+            "${PYTHON_TEST_DIR}/test_main.py"
+            SUBMODULES parallelzone pluginplay chemist simde chemcache friendzone nwchemex
+        )
+    endif()
+
+Integration Tests
+=================
+
+Building on the description provided in :ref:`writing_unit_tests`, integration
+tests are written in the same manner. Below is an example of how to use NWChemEx
+in our new integration test to acquire input values and submodules that may be
+needed by a module in our project.
+
+.. code-block:: python
+
+    import unittest
+    import nwchemex
+    import scf
+    from pluginplay import ModuleManager
+    from simde import AOEnergy
+    from simde import MoleculeFromString
+    from simde import MolecularBasisSet
+
+    class TestIntegration(unittest.TestCase):
+
+        def test_scf_module(self):
+            # Module we want to test
+            key = "SCF Module"
+
+            # Property Types from SimDE
+            molecule_pt = MoleculeFromString()
+            basis_set_pt = MolecularBasisSet()
+            energy_pt = AOEnergy()
+
+            # Can use NWChemEx modules to get inputs
+            mol = self.mm.run_as(molecule_pt, "NWX Molecules", "water")
+            bs = self.mm.run_as(basis_set_pt, "sto-3g", mol)
+
+            # set NWChemEx modules as needed submodules
+            submod_key = "A submodule of my SCF module"
+            integral_key = "Some integral needed to run SCF"
+            mm.change_submod(key, submod_key, integral_key)
+
+            # Test our module
+            egy = self.mm.run_as(energy_pt, key, mol, bs)
+            self.assertAlmostEqual(egy, 3.14159265359, places=6)
+
+        def test_another_module(self):
+            # Add more tests where appropriate
+            pass
+
+        def setUp(self):
+            self.mm = ModuleManager()
+            nwchemex.load_modules(mm) # Also loads out SCF modules
+

docs/source/testing/unit.rst

@@ -18,4 +18,148 @@
 Writing Unit Tests for NWChemEx
 ###############################
 
-TODO: Write me!!!!
+Within the first party NWChemEx libraries, we aim for extensive unit testing to 
+ensure functionality and correctness. All classes, functions, and modules added
+to any of the first party libraries will be expected to have corresponding unit
+tests. Testing of functions (as well as Plugin modules) should minimally ensure 
+that all return routes and errors are checked. Tests for classes should do the 
+same for all member functions, while additionally testing that the state of all 
+instances is consistent at construction and after modifications. Generally, the
+unit tests should be able to run quickly, and use simplified data with the 
+minimum level of complexity need to ensure completeness in the testing.
+
+The C++ unit tests use the `Catch2 framework <https://github.com/catchorg/Catch2>`_,
+while python tests use the `unittest framework <https://docs.python.org/3/library/unittest.html>`_.
+Assume the following class and related comparison function are intended to be 
+added to one of the first party libraries:
+
+.. tabs::
+
+    .. tab:: C++
+
+        .. code-block:: c++
+            :linenos:
+
+            #include <stdexcept>
+
+            class ToBeTested {
+            private:
+                using value_type = int;
+                value_type my_value_;
+
+            public:
+                ToBeTested(value_type a_value = 0) : my_value_(a_value) {}
+
+                value_type check_my_value() { return my_value_; }
+
+                void change_my_value(value_type new_value) {
+                    if(new_value == 13) throw std::runtime_error("Unlucky Number");
+                    my_value_ = new_value;
+                }
+
+                bool operator==(const ToBeTested& rhs) const noexcept {
+                    return my_value_ == rhs.my_value_;
+                }
+
+            }; // ToBeTested
+
+            inline bool operator!=(const ToBeTested& lhs, const ToBeTested& rhs) {
+                return !(lhs == rhs);
+            }
+
+    .. tab:: Python
+
+        .. code-block:: python
+            :linenos:
+
+            class ToBeTested():
+
+            def __init__(self, a_value = 0): 
+                self.__my_value = a_value
+
+            def check_my_value(self):
+                return self.__my_value
+
+            def change_my_value(self, new_value):
+                if new_value == 13: 
+                    raise RuntimeError("Unlucky Number")
+                self.__my_value = new_value
+
+            def __eq__(self, other):
+                if not isinstance(other, ToBeTested):
+                    return NotImplemented
+                return self.__my_value == other.__my_value
+
+An example unit test for the above looks like:
+
+.. tabs::
+
+    .. tab:: C++
+
+        .. code-block:: c++
+            :linenos:
+
+            #include "to_be_tested.hpp"
+            #include <catch2/catch.hpp>
+
+            TEST_CASE("ToBeTested") {
+                auto defaulted  = ToBeTested();
+                auto with_value = ToBeTested(3);
+
+                SECTION("Comparisons") {
+                    SECTION("operator==") {
+                        REQUIRE(defaulted == ToBeTested());
+                        REQUIRE(with_value == ToBeTested(3));
+                        REQUIRE_FALSE(defaulted == with_value);
+                    }
+                    SECTION("operator!=") { 
+                        REQUIRE(defaulted != with_value); 
+                    }
+                }
+
+                SECTION("check_my_value") {
+                    REQUIRE(defaulted.check_my_value() == 0);
+                    REQUIRE(with_value.check_my_value() == 3);
+                }
+
+                SECTION("change_my_value") {
+                    SECTION("Not Unlucky") {
+                        defaulted.change_my_value(7);
+                        REQUIRE(defaulted.check_my_value() == 7);
+                    }
+                    SECTION("Unlucky") {
+                        REQUIRE_THROWS_AS(defaulted.change_my_value(13),
+                                          std::runtime_error);
+                    }
+                }
+            }
+
+    .. tab:: Python
+
+        .. code-block:: python
+            :linenos:
+
+            from to_be_tested import ToBeTested
+            import unittest
+
+            class TestNewClass(unittest.TestCase):
+                def setUp(self):
+                    self.defaulted = ToBeTested()
+                    self.with_value = ToBeTested(3)
+
+                def test_equality(self):
+                    self.assertEqual(self.defaulted, ToBeTested())
+                    self.assertEqual(self.with_value, ToBeTested(3))
+                    self.assertNotEqual(self.defaulted, self.with_value)
+
+                def test_check_my_value(self):
+                    self.assertEqual(self.defaulted.check_my_value(), 0)
+                    self.assertEqual(self.with_value.check_my_value(), 3)
+
+                def test_change_my_value(self):
+                    self.defaulted.change_my_value(7)
+                    self.assertEqual(self.defaulted.check_my_value(), 7)
+
+                    with self.assertRaises(RuntimeError) as context:
+                        self.defaulted.change_my_value(13)
+                    self.assertTrue("Unlucky Number" in str(context.exception))