Add Custom Settings to Apps Endpoint of Portal API

docs/conf.py

@@ -11,15 +11,17 @@
 #
 # All configuration values have a default; values that are commented out
 # serve to show the default.
-from setuptools_scm import get_version
 import os
-from pathlib import Path
-import subprocess
 import sys
+import subprocess
+from dataclasses import asdict
+from pathlib import Path
 from unittest import mock
 
 import django
 from django.conf import settings
+from setuptools_scm import get_version
+from sphinxawesome_theme import ThemeOptions, LinkIcon
 
 # Mock Dependencies
 # NOTE: No obvious way to automatically anticipate all the sub modules without
@@ -78,7 +80,7 @@
     "sqlalchemy.orm",
     "tethys_apps.harvester",
     "tethys_apps.models",  # Mocked to prevent issues with loading apps during docs build.
-    "tethys_compute.utilities",  # Mocked to prevent issues with DictionaryField and List Field during docs build.
+    "tethys_apps.admin",  # Mocked to prevent issues with loading models during docs build.
     "yaml",
 ]
 
@@ -97,6 +99,9 @@ def __getattr__(cls, name):
 print("{}".format(", ".join(MOCK_MODULES)))
 sys.modules.update((mod_name, MockModule()) for mod_name in MOCK_MODULES)
 
+# patcher = mock.patch("tethys_apps.models.register_custom_group")
+# patcher.start()
+
 # Fixes django settings module problem
 sys.path.insert(0, os.path.abspath(".."))
 
@@ -243,27 +248,35 @@ def __getattr__(cls, name):
 
 html_title = f"{project} Documentation"
 html_short_title = "Tethys Docs"
-html_logo = "images/features/tethys-logo-75.png"
+# html_logo = "images/features/tethys-logo-75.png"
 html_favicon = "images/default_favicon.ico"
 html_static_path = ["_static"]
 html_css_files = [
     "css/tethys.css",
 ]
 
 html_theme = "sphinxawesome_theme"
-html_theme_options = {
-    "extra_header_links": {
+theme_options = ThemeOptions(
+    main_nav_links={
         "Tutorials": "tutorials",
         "SDK": "tethys_sdk",
         "CLI": "tethys_cli",
         "Tethys Portal": "tethys_portal",
         "Migrate Apps": "whats_new/app_migration",
-        "GitHub": "https://github.com/tethysplatform/tethys",
     },
-    "show_breadcrumbs": False,
-    "show_prev_next": True,
-    "show_scrolltop": True,
-}
+    extra_header_link_icons={
+        "GitHub": LinkIcon(
+            icon='<svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-github" viewBox="0 0 16 16"><path d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27s1.36.09 2 .27c1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58-8-8-8"/></svg>',
+            link="https://github.com/tethysplatform/tethys",
+        ),
+    },
+    logo_dark="images/features/tethys-logo-75.png",  # todo: svg logo
+    logo_light="images/features/tethys_logo_inverse.png",  # todo: svg logo
+    show_breadcrumbs=False,
+    show_prev_next=True,
+    show_scrolltop=True,
+)
+html_theme_options = asdict(theme_options)
 
 html_collapsible_definitions = True
 

docs/docs_environment.yml

@@ -13,7 +13,7 @@ dependencies:
   - pip
   - tethys_dataset_services >=2.0.0
   - django =3.2.*
-  - sphinx <7.2
+  - sphinx
   - sphinx-argparse
   - make
   - setuptools_scm

docs/installation/database_configuration.rst

@@ -0,0 +1,49 @@
+.. _database_configuration:
+
+**********************
+Database Configuration
+**********************
+
+**Last Updated:** September 2023
+
+Tethys Platform supports several options for setting up a DB server: local, docker, or remote. By default Tethys uses a local SQLite database. When using the default settings, setting up a database can be done with one simple command:
+
+.. code-block:: bash
+
+    tethys db configure
+
+.. note::
+
+    The tethys db command (:ref:`tethys_db_cmd`) will create a local database file in the location specified by the ``NAME`` setting in the ``DATABASES`` section of the :file:`portal_config.yml` file (by default ``tethys_platform.sqlite``). If the value of ``NAME`` is a relative path then the database file will be created relative to directory specified by the ``TETHYS_HOME`` environment variable. By default ``TETHYS_HOME`` is at `~/.tethys`.
+
+For information on additional database settings refer to the :ref:`database_settings` setting.
+
+Using PostreSQL
+===============
+
+.. important::
+
+    This feature requires that the PostgreSQL database and the ``psycopg2`` library be installed. Starting with Tethys 5.0 or if you are using ``micro-tethys-platform``, you will need to install ``postgresql`` and ``psycopg2`` using conda as follows:
+
+    .. code-block:: bash
+
+        # conda: conda-forge channel strongly recommended
+        conda install -c conda-forge postgresql psycopg2
+
+While Tethys Platform uses the SQLite database by default for ease of use in development environments, it has excellent support for PostgreSQL (which is recommended for production). To use a local PostgreSQL database (for development) you will need to change the ``ENGINE`` setting in the ``DATABASES`` section of the :file:`portal_config.yml` file to ``django.db.backends.postgresql``, and you will need to add the ``DIR`` setting in the same section. This can easily be done using the :ref:`tethys_settings_cmd`:
+
+.. code-block:: bash
+
+    tethys settings --set DATABASES.default.ENGINE django.db.backends.postgresql --set DATABASES.default.DIR psql
+
+.. note::
+
+    The tethys db command (:ref:`tethys_db_cmd`) will create a local database server in the directory specified by the ``DIR`` setting in the ``DATABASES`` section of the :file:`portal_config.yml` file. If the value of ``DIR`` is a relative path then the database server will be created relative to directory specified by the ``TETHYS_HOME`` environment variable. By default ``TETHYS_HOME`` is at `~/.tethys`.
+
+    As an alternative to creating a local database server you can also configure a Docker DB server (see :ref:`using_docker`). A local database server is only recommended for development environments. For production environments please refer to :ref:`production_installation`.
+
+
+Using Other Databases
+=====================
+
+While many of the convenience tools that Tethys provides only support SQLite and PostgreSQL, Tethys can be configured with other Database engines. See `<https://docs.djangoproject.com/en/3.2/ref/databases/>`_ for more information.

docs/whats_new.rst

@@ -17,14 +17,14 @@ Python 3.12
 * Verified Tethys Platform works using Python 3.12
 
 Conda Forge Package
-------------------------
+-------------------
 
 * Tethys Platform is now fully packaged on conda-forge!
 
 See: :ref:`development_installation`
 
 Optional Dependencies and Micro Tethys
-------------
+--------------------------------------
 
 * Made many of the dependencies of ``tethys-platform`` optional and released new ``micro-tethys-platform`` conda package on the ``tethysplatform`` channel with minimal dependencies
 * Updated docs to reflect what features are now optional and what dependencies are needed to support those features

tests/apps/tethysapp-test_app/tethysapp/test_app/app.py

@@ -39,12 +39,14 @@ def custom_settings(self):
                 name="default_name",
                 type=CustomSetting.TYPE_STRING,
                 description="Default model name.",
+                include_in_api=True,
                 required=True,
             ),
             CustomSetting(
                 name="max_count",
                 type=CustomSetting.TYPE_INTEGER,
                 description="Maximum allowed count in a method.",
+                include_in_api=False,
                 required=False,
             ),
             CustomSetting(
@@ -57,6 +59,7 @@ def custom_settings(self):
                 name="enable_feature",
                 type=CustomSetting.TYPE_BOOLEAN,
                 description="Enable this feature when True.",
+                include_in_api=True,
                 required=False,
             ),
             JSONCustomSetting(
@@ -72,18 +75,21 @@ def custom_settings(self):
             JSONCustomSetting(
                 name="JSON_setting_default_value_required",
                 description="This is JSON setting with a default value",
+                include_in_api=True,
                 required=True,
                 default={"Test": "JSON test String"},
             ),
             JSONCustomSetting(
                 name="JSON_setting_default_value",
                 description="This is JSON setting with a default value",
+                include_in_api=False,
                 required=False,
                 default={"Test": "JSON test String"},
             ),
             SecretCustomSetting(
                 name="Secret_Test_required",
                 description="This is SECRET setting with required True",
+                include_in_api=True,
                 required=True,
             ),
             SecretCustomSetting(

tests/unit_tests/test_tethys_apps/test_admin.py

@@ -97,7 +97,14 @@ def test_has_add_permission(self):
 
     def test_CustomSettingInline(self):
         expected_readonly_fields = ("name", "description", "type", "required")
-        expected_fields = ("name", "description", "type", "value", "required")
+        expected_fields = (
+            "name",
+            "description",
+            "type",
+            "value",
+            "include_in_api",
+            "required",
+        )
         expected_model = CustomSetting
 
         ret = CustomSettingInline(mock.MagicMock(), mock.MagicMock())
@@ -108,7 +115,7 @@ def test_CustomSettingInline(self):
 
     def test_SecretCustomSettingInline(self):
         expected_readonly_fields = ("name", "description", "required")
-        expected_fields = ("name", "description", "value", "required")
+        expected_fields = ("name", "description", "value", "include_in_api", "required")
         expected_model = SecretCustomSetting
 
         ret = SecretCustomSettingInline(mock.MagicMock(), mock.MagicMock())
@@ -119,7 +126,7 @@ def test_SecretCustomSettingInline(self):
 
     def test_JSONCustomSettingInline(self):
         expected_readonly_fields = ("name", "description", "required")
-        expected_fields = ("name", "description", "value", "required")
+        expected_fields = ("name", "description", "value", "include_in_api", "required")
         expected_model = JSONCustomSetting
 
         ret = JSONCustomSettingInline(mock.MagicMock(), mock.MagicMock())

tests/unit_tests/test_tethys_apps/test_models/test_CustomSetting.py

@@ -501,3 +501,54 @@ def test_get_value_json_with_no_required_and_empty_value(self):
         # Check value is a required setting returns None
 
         self.assertEqual(custom_setting.get_value(), None)
+
+    def test_include_in_api(self):
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="default_name"
+        )
+        self.assertTrue(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="max_count"
+        )
+        self.assertFalse(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="change_factor"
+        )
+        self.assertFalse(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="enable_feature"
+        )
+        self.assertTrue(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="JSON_setting_not_default_value_required"
+        )
+        self.assertFalse(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="JSON_setting_not_default_value"
+        )
+        self.assertFalse(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="JSON_setting_default_value_required"
+        )
+        self.assertTrue(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="JSON_setting_default_value"
+        )
+        self.assertFalse(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="Secret_Test_required"
+        )
+        self.assertTrue(custom_setting.include_in_api)
+
+        custom_setting = self.test_app.settings_set.select_subclasses().get(
+            name="Secret_Test2_without_required"
+        )
+        self.assertFalse(custom_setting.include_in_api)

tests/unit_tests/test_tethys_portal/test_utilities.py

@@ -1,3 +1,5 @@
+import datetime
+import uuid
 import unittest
 from unittest import mock
 
@@ -84,3 +86,21 @@ def test_utilities_no_user_exist_next(self, mock_redirect, mock_authenticate):
 
         # mock redirect after logged in using next parameter or default to user profile
         mock_redirect.assert_called_once_with("foo")
+
+    def test_json_serializer_datetime(self):
+        d = datetime.datetime(2020, 1, 1)
+        ret = utilities.json_serializer(d)
+        self.assertEqual("2020-01-01T00:00:00", ret)
+
+    def test_json_serializer_uuid(self):
+        u = uuid.uuid4()
+        ret = utilities.json_serializer(u)
+        self.assertEqual(str(u), ret)
+
+    def test_json_serializer_other(self):
+        with self.assertRaises(TypeError) as cm:
+            utilities.json_serializer(1)
+
+        self.assertEqual(
+            'Object of type "int" is not JSON serializable', str(cm.exception)
+        )

tests/unit_tests/test_tethys_portal/test_views/test_api.py

@@ -156,6 +156,56 @@ def test_get_app_valid_id_with_prefix(self):
             r"^/test/prefix/admin/tethys_apps/tethysapp/[0-9]+/change/$",
         )
 
+    @override_settings(STATIC_URL="/static")
+    @override_settings(PREFIX_URL="/")
+    @override_settings(LOGIN_URL="/accounts/login/")
+    def test_get_app_authenticated(self):
+        self.client.force_login(self.user)
+        self.reload_urlconf()
+
+        """Test get_app API endpoint with valid app id."""
+        response = self.client.get(reverse("api:get_app", kwargs={"app": "test-app"}))
+        self.assertEqual(response.status_code, 200)
+        self.assertIsInstance(response, JsonResponse)
+        json = response.json()
+        self.assertIn("title", json)
+        self.assertIn("description", json)
+        self.assertIn("tags", json)
+        self.assertIn("package", json)
+        self.assertIn("urlNamespace", json)
+        self.assertIn("color", json)
+        self.assertIn("icon", json)
+        self.assertIn("exitUrl", json)
+        self.assertIn("rootUrl", json)
+        self.assertIn("settingsUrl", json)
+        self.assertEqual("Test App", json["title"])
+        self.assertEqual(
+            "Place a brief description of your app here.", json["description"]
+        )
+        self.assertEqual("", json["tags"])
+        self.assertEqual("test_app", json["package"])
+        self.assertEqual("test_app", json["urlNamespace"])
+        self.assertEqual("#2c3e50", json["color"])
+        self.assertEqual("/static/test_app/images/icon.gif", json["icon"])
+        self.assertEqual("/apps/", json["exitUrl"])
+        self.assertEqual("/apps/test-app/", json["rootUrl"])
+        self.assertRegex(
+            json["settingsUrl"],
+            r"^/admin/tethys_apps/tethysapp/[0-9]+/change/$",
+        )
+        self.assertDictEqual(
+            {
+                "JSON_setting_default_value_required": {
+                    "type": "JSON",
+                    "value": {"Test": "JSON test String"},
+                },
+                "Secret_Test_required": {"type": "SECRET", "value": None},
+                "default_name": {"type": "STRING", "value": None},
+                "enable_feature": {"type": "BOOLEAN", "value": None},
+            },
+            json["customSettings"],
+        )
+
     def test_get_app_invalid_id(self):
         """Test get_app API endpoint with invalid app id."""
         response = self.client.get(reverse("api:get_app", kwargs={"app": "foo-bar"}))