Django Testing Additions

.github/workflows/testing_and_validation.yml

@@ -101,5 +101,5 @@ jobs:
           cp -a ssl/local/*.pem ssl/scheduler/.
 
       - name: Run Tests
-        run: ./scripts/run_tests.sh --venv dmod_venv -v -srv
+        run: ./scripts/run_tests.sh --venv dmod_venv -v -d -srv
         timeout-minutes: 1

.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 .idea/
 /docker_host_volumes/
 /venv/
+/*_venv/
 /venv_*/
 .env
 docker/main/base/ssh/

python/gui/maas_experiment/application_values.py

@@ -262,19 +262,6 @@ def get_full_localtimezone():
     STATIC_RESOURCE_DIRECTORY / "rest_forwarding.json"
 )
 
-REQUIRED_ENVIRONMENT_VARIABLES: typing.List[typing.Dict[str, str]] = [
-    {
-        "name": "MAAS_ENDPOINT_HOST",
-        "purpose": "The default host address for MaaS"
-    },
-    {
-        "name": "MAAS_ENDPOINT_PORT",
-        "purpose": "The port for the default MaaS endpoint"
-    }
-]
-"""A list of all environments that are required to get the system to function"""
-
-
 def load_maas_endpoints() -> typing.Dict[str, typing.Dict[str, str]]:
     endpoints: typing.Dict[str, typing.Dict[str, str]] = dict()
 

python/gui/maas_experiment/settings.py

@@ -144,27 +144,3 @@
     BASE_DIRECTORY / "static"
 ]
 
-
-def ensure_required_environment_variables():
-    missing_variables = [
-        variable_name
-        for variable_name in REQUIRED_ENVIRONMENT_VARIABLES
-        if variable_name['name'] not in os.environ.keys()
-    ]
-
-    if missing_variables:
-        missing_keys = [
-            variable['name']
-            for variable in missing_variables
-        ]
-
-        error("The following required environment variables are missing:")
-
-        for missing_variable in missing_variables:
-            error(f"{missing_variable['name']}: {missing_variable['purpose']}")
-
-        raise ValueError(f"The following environment variables have not been set: [{', '.join(missing_keys)}]")
-
-
-ensure_required_environment_variables()
-

python/lib/metrics/setup.py

@@ -20,6 +20,6 @@
     author_email='',
     url='',
     license='',
-    install_requires=[],
+    install_requires=['scikit-learn', 'pandas'],
     packages=find_namespace_packages(exclude=['dmod.test', 'schemas', 'ssl', 'src'])
 )

python/services/evaluationservice/dmod/evaluationservice/_version.py

@@ -1 +1 @@
-__version__ = '0.6.1'
+__version__ = '0.6.2'

python/services/evaluationservice/setup.py

@@ -17,6 +17,15 @@
     author_email='',
     url='',
     license='',
-    install_requires=['redis', 'dmod-evaluations', 'channels', 'channels-redis'],
+    install_requires=[
+        'redis',
+        'dmod-evaluations',
+        'channels',
+        'channels-redis',
+        'django-rq',
+        'Django~=4.2',
+        'djangorestframework',
+        'geopandas'
+    ],
     packages=find_namespace_packages(exclude=['dmod.test', 'deprecated', 'conf', 'schemas', 'ssl', 'src'])
 )

requirements.txt

@@ -1,3 +1,4 @@
+wheel
 pytz
 pyyaml
 docker
@@ -9,7 +10,6 @@ Faker
 flake8
 autopep8
 setuptools
-wheel
 Deprecated
 cryptography
 flask

scripts/code_style.sh

@@ -87,24 +87,6 @@ run_autofix()
         "${1:-.}"
 }
 
-# Make sure we end up in the same starting directory, and deactivate venv if it was activated
-cleanup_before_exit()
-{
-    # Make sure we don't accidentally run this more than once
-    CLEANUP_DONE=$((${CLEANUP_DONE:=0}+1))
-    if [ ${CLEANUP_DONE} -gt 1 ]; then
-        >&2 echo "Warning: cleanup function being run for ${CLEANUP_DONE} time"
-    fi
-    # Go back to shell starting dir
-    cd "${STARTING_DIR:?}"
-
-    # If the flag is set that a virtual environment was activated, then deactivate it
-    if [ ${VENV_WAS_ACTIVATED:-1} -eq 0 ]; then
-        >&2 echo "Deactiving active virtual env at ${VIRTUAL_ENV}"
-        deactivate
-    fi
-}
-
 # Options args loop
 while [[ ${#} -gt 0 ]]; do
     case "${1}" in

scripts/dist_package.sh

@@ -48,24 +48,6 @@ Options:
     echo "${_O}" 2>&1
 }
 
-# Make sure we end up in the same starting directory, and deactivate venv if it was activated
-cleanup_before_exit()
-{
-    # Make sure we don't accidentally run this more than once
-    CLEANUP_DONE=$((${CLEANUP_DONE:=0}+1))
-    if [ ${CLEANUP_DONE} -gt 1 ]; then
-        >&2 echo "Warning: cleanup function being run for ${CLEANUP_DONE} time"
-    fi
-    # Go back to shell starting dir
-    cd "${STARTING_DIR:?}"
-
-    # If the flag is set that a virtual environment was activated, then deactivate it
-    if [ ${VENV_WAS_ACTIVATED:-1} -eq 0 ]; then
-        >&2 echo "Deactiving active virtual env at ${VIRTUAL_ENV}"
-        deactivate
-    fi
-}
-
 while [ ${#} -gt 0 ]; do
     case "${1}" in
         -h|--help|-help)

scripts/run_tests.sh

@@ -20,6 +20,8 @@ fi
 # Import bash-only shared functions used for python-dev-related scripts
 . ${SHARED_FUNCS_DIR}/py_dev_bash_func.sh
 
+DJANGO_TEST_SCRIPT_PATH="${SCRIPT_PARENT_DIR}/test_django.py"
+
 SUPPORTED_PACKAGES=()
 
 PACKAGE_TESTING_SCRIPT=${SCRIPT_PARENT_DIR}/test_package.sh
@@ -50,6 +52,10 @@ Options:
         Include service packages in what is supported and tested,
         which are ignored by default
 
+    --django | -d
+        Include Django services in what is supported and tested,
+        which are ignored by default
+
     --venv <dir>
         Set the directory of the virtual environment to use.
         By default, the following directories will be checked,
@@ -116,6 +122,10 @@ list_packages()
     for p in "${SUPPORTED_PACKAGES[@]}"; do
         echo "    ${p}"
     done
+    if [ -z "${TEST_DJANGO_SERVICES:-}" ]; then
+        echo "Supported Django Services:"
+        ${DJANGO_TEST_SCRIPT_PATH} --list
+    fi
 }
 
 print_test_result()
@@ -178,6 +188,10 @@ while [ ${#} -gt 0 ]; do
             [ -n "${DO_SERVICE_PACKAGES:-}" ] && usage && exit 1
             DO_SERVICE_PACKAGES='true'
             ;;
+        --django|-d)
+            [ -n "${TEST_DJANGO_SERVICES:-}" ] && usage && exit 1;
+            TEST_DJANGO_SERVICES='true';
+            ;;
         --venv)
             [ -n "${VENV_DIR:-}" ] && usage && exit 1
             VENV_DIR="$(py_dev_validate_venv_dir "${2}")"
@@ -261,6 +275,39 @@ for (( i=0; i<${len}; i++)); do
     fi
 done
 
+
+if [ -n "${TEST_DJANGO_SERVICES:-}" ]; then
+    if [ -n "${DO_QUIET:-}" ]; then
+        py_dev_activate_venv
+        echo "Running django tests in quiet mode"
+        ${DJANGO_TEST_SCRIPT_PATH} --quiet
+        DJANGO_ERROR_COUNT=$?
+    elif [ -z "${SET_VERBOSE:-}" ]; then
+        py_dev_activate_venv
+        ${DJANGO_TEST_SCRIPT_PATH}
+        DJANGO_ERROR_COUNT=$?
+    else
+        echo "-----------------------------------"
+        echo ""
+        echo "Django Tests:"
+        echo ""
+        py_dev_activate_venv
+        VERBOSE_DJANGO_OUTPUT=$("${DJANGO_TEST_SCRIPT_PATH}" --verbose)
+        DJANGO_ERROR_COUNT=$?
+        echo "$VERBOSE_DJANGO_OUTPUT"
+
+        # Extract the lines containing the summary to print later
+        # Knowing that there will only be 2 values is a stopgap
+        DJANGO_SUMMARY=$(echo "$VERBOSE_DJANGO_OUTPUT" | tail -n 2)
+        echo ""
+        echo ""
+        echo ""
+    fi
+    TOTAL_RESULT=$((TOTAL_RESULT+$DJANGO_ERROR_COUNT))
+    py_dev_deactivate_venv
+fi
+
+
 # If we got verbose output, print summary at the end, along with something extra to have stand out
 if [ -n "${SET_VERBOSE:-}" ]; then
     echo "**************************************************"
@@ -270,6 +317,10 @@ if [ -n "${SET_VERBOSE:-}" ]; then
         _P_RES=${PACKAGE_RESULT[${j}]}
         print_test_result ${SUPPORTED_PACKAGES[${j}]} ${_P_RES}
     done
+
+    if [ -n "${DJANGO_SUMMARY:-}" ]; then
+        echo "$DJANGO_SUMMARY"
+    fi
 fi
 
 echo ""

scripts/secrets/setup_git_secrets.sh

@@ -113,7 +113,7 @@ OPTIONS:
     echo "${_INFO_OUT}" | less
 }
 
-cleanup_before_exit()
+cleanup_repo()
 {
     # Only do this deletion if both the flag is set and we have established at some point it is safe to do so
     if [ ${SAFE_REMOVE_REPO:-1} -eq 0 ] && [ -n "${DO_REMOVE_REPO:-}" ];  then
@@ -292,7 +292,7 @@ while [ ${#} -gt 0 ]; do
 done
 
 # Trap to make sure we "clean up" script activity before exiting
-trap cleanup_before_exit 0 1 2 3 6 15
+trap cleanup_repo 0 1 2 3 6 15
 
 # Make sure a repo directory was set
 if [ -z "${REPO_DIR:-}" ]; then

scripts/shared/py_dev_func.sh

@@ -122,4 +122,32 @@ py_dev_activate_venv()
     else
         VENV_WAS_ACTIVATED=1
     fi
+}
+
+# If appropriate, deactivate the virtual env set within VENV_DIR, and note whether this is done in VENV_WAS_ACTIVATED.
+py_dev_deactivate_venv()
+{
+    # If the flag is set that a virtual environment was activated, then deactivate it
+    if [ ${VENV_WAS_ACTIVATED:-1} -eq 0 ]; then
+        >&2 echo "Deactiving active virtual env at ${VIRTUAL_ENV}"
+        >&2 echo ""
+        deactivate
+        VENV_WAS_ACTIVATED=1
+    fi
+}
+
+
+# Make sure we end up in the same starting directory, and deactivate venv if it was activated
+cleanup_before_exit()
+{
+    # Make sure we don't accidentally run this more than once
+    CLEANUP_DONE=$((${CLEANUP_DONE:=0}+1))
+    if [ ${CLEANUP_DONE} -gt 1 ]; then
+        >&2 echo "Warning: cleanup function being run for ${CLEANUP_DONE} time"
+    fi
+
+    # Go back to shell starting dir
+    cd "${STARTING_DIR:?}"
+
+    py_dev_deactivate_venv
 }