Pylon-level tunables; pylon settings in tunables

pylon/core/tools/config.py

@@ -2,7 +2,7 @@
 # coding=utf-8
 # pylint: disable=I0011,E0401
 
-#   Copyright 2020 getcarrier.io
+#   Copyright 2025 getcarrier.io
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -51,27 +51,132 @@ def config_substitution(obj, secrets):
 def vault_secrets(settings):
     """ Get secrets from HashiCorp Vault """
     if "vault" not in settings:
-        return dict()
+        return {}
+    #
     config = settings["vault"]
+    #
     client = hvac.Client(
         url=config["url"],
         verify=config.get("ssl_verify", False),
-        namespace=config.get("namespace", None)
+        namespace=config.get("namespace", None),
     )
+    #
     if "auth_token" in config:
         client.token = config["auth_token"]
+    #
     if "auth_username" in config:
         client.auth.userpass.login(
             config.get("auth_username"), config.get("auth_password", "")
         )
+    #
     if "auth_role_id" in config:
         client.auth.approle.login(
             config.get("auth_role_id"), config.get("auth_secret_id", "")
         )
+    #
     if not client.is_authenticated():
         logging.error("Vault authentication failed")
-        return dict()
-    return client.secrets.kv.v2.read_secret_version(
-        path=config.get("secrets_path", "secrets"),
-        mount_point=config.get("secrets_mount_point", "kv")
-    ).get("data", dict()).get("data", dict())
+        return {}
+    #
+    secrets_path = config.get("secrets_path", "secrets")
+    secrets_mount_point = config.get("secrets_mount_point", "kv")
+    #
+    secrets_kv_version = config.get("secrets_kv_version", 2)
+    secrets_version = config.get("secrets_version", None)
+    #
+    try:
+        if secrets_kv_version == 1:
+            result = client.secrets.kv.v1.read_secret(
+                path=secrets_path,
+                mount_point=secrets_mount_point,
+            ).get("data", {})
+        elif secrets_kv_version == 2:
+            result = client.secrets.kv.v2.read_secret_version(
+                path=secrets_path,
+                version=secrets_version,
+                mount_point=secrets_mount_point,
+                raise_on_deleted_version=True,
+            ).get("data", {}).get("data", {})
+        else:
+            logging.error("Unknown Vault KV version: %s", secrets_kv_version)
+            result = {}
+    except:  # pylint: disable=W0702
+        logging.exception("Failed to read Vault secrets")
+        result = {}
+    #
+    return result
+
+
+def tunable_exists(tunable):
+    """ Tunables: check """
+    from tools import context  # pylint: disable=C0415,E0401
+    from pylon.framework.db.models.tunable_value import TunableValue  # pylint: disable=C0415
+    #
+    with context.pylon_db.make_session() as db_session:
+        tunable_obj = db_session.query(TunableValue).get(tunable)
+        #
+        if tunable_obj is None:
+            return False
+        #
+        return True
+
+
+def tunable_get(tunable, default=None):
+    """ Tunables: get """
+    from tools import context  # pylint: disable=C0415,E0401
+    from pylon.framework.db.models.tunable_value import TunableValue  # pylint: disable=C0415
+    #
+    with context.pylon_db.make_session() as db_session:
+        tunable_obj = db_session.query(TunableValue).get(tunable)
+        #
+        if tunable_obj is None:
+            return default
+        #
+        return tunable_obj.value
+
+
+def tunable_set(tunable, value):
+    """ Tunables: set """
+    from tools import context  # pylint: disable=C0415,E0401
+    from pylon.framework.db.models.tunable_value import TunableValue  # pylint: disable=C0415
+    #
+    with context.pylon_db.make_session() as db_session:
+        tunable_obj = db_session.query(TunableValue).get(tunable)
+        #
+        if tunable_obj is None:
+            tunable_obj = TunableValue(
+                tunable=tunable,
+                value=value,
+            )
+            #
+            db_session.add(tunable_obj)
+        else:
+            tunable_obj.value = value
+        #
+        try:
+            db_session.commit()
+        except:  # pylint: disable=W0702
+            db_session.rollback()
+            raise
+        #
+        return None
+
+
+def tunable_delete(tunable):
+    """ Tunables: delete """
+    from tools import context  # pylint: disable=C0415,E0401
+    from pylon.framework.db.models.tunable_value import TunableValue  # pylint: disable=C0415
+    #
+    with context.pylon_db.make_session() as db_session:
+        tunable_obj = db_session.query(TunableValue).get(tunable)
+        #
+        if tunable_obj is not None:
+            db_session.delete(tunable_obj)
+            #
+            try:
+                db_session.commit()
+            except:  # pylint: disable=W0702
+                db_session.rollback()
+                raise
+        #
+        return None

pylon/core/tools/db_support.py

@@ -41,26 +41,9 @@
 #
 
 
-def init(context):
-    """ Init DB support """
-    if context.before_reloader:
-        log.info(
-            "Running in development mode before reloader is started. Skipping DB support init"
-        )
-        return
-    #
-    log.info("Initializing DB support")
-    #
-    # App DB
-    #
-    context.db = Context()
-    context.db.config = context.settings.get("db", {})
-    #
-    context.db.url = get_db_url(context.db)
-    context.db.engine = make_engine(context.db)
-    #
-    context.db.schema_mapper = lambda schema: schema
-    context.db.make_session = make_session_fn(context.db)
+def basic_init(context):
+    """ Init basic DB support """
+    log.info("Initializing basic DB support")
     #
     # Pylon DB
     #
@@ -101,6 +84,40 @@ def init(context):
         context.pylon_db.metadata.create_all(bind=context.pylon_db.engine)
     except:  # pylint: disable=W0702
         log.exception("Failed to create Pylon DB entities")
+
+
+def basic_deinit(context):
+    """ De-init basic DB support """
+    log.info("De-initializing basic DB support")
+    #
+    # Pylon DB
+    #
+    try:
+        context.pylon_db.engine.dispose()
+    except:  # pylint: disable=W0702
+        pass
+
+
+def init(context):
+    """ Init DB support """
+    if context.before_reloader:
+        log.info(
+            "Running in development mode before reloader is started. Skipping DB support init"
+        )
+        return
+    #
+    log.info("Initializing DB support")
+    #
+    # App DB
+    #
+    context.db = Context()
+    context.db.config = context.settings.get("db", {})
+    #
+    context.db.url = get_db_url(context.db)
+    context.db.engine = make_engine(context.db)
+    #
+    context.db.schema_mapper = lambda schema: schema
+    context.db.make_session = make_session_fn(context.db)
     #
     # App hooks
     #
@@ -118,19 +135,14 @@ def deinit(context):
     #
     log.info("De-initializing DB support")
     #
-    # Pylon DB
-    #
-    try:
-        context.pylon_db.engine.dispose()
-    except:  # pylint: disable=W0702
-        pass
-    #
     # App DB
     #
     try:
         context.db.engine.dispose()
     except:  # pylint: disable=W0702
         pass
+    #
+    basic_deinit(context)
 
 
 #

pylon/core/tools/seed.py

@@ -2,7 +2,7 @@
 # coding=utf-8
 # pylint: disable=I0011
 
-#   Copyright 2021 getcarrier.io
+#   Copyright 2025 getcarrier.io
 #
 #   Licensed under the Apache License, Version 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
@@ -49,6 +49,11 @@ def load_settings():
     if not settings_data:
         return None
     #
+    return parse_settings(settings_data)
+
+
+def parse_settings(settings_data):
+    """ Parse settings from data """
     try:
         settings = yaml.load(os.path.expandvars(settings_data), Loader=yaml.SafeLoader)
         settings = config.config_substitution(settings, config.vault_secrets(settings))

pylon/framework/db/models/tunable_value.py

@@ -0,0 +1,29 @@
+#!/usr/bin/python3
+# coding=utf-8
+
+#   Copyright 2025 getcarrier.io
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+""" Tunable value DB model """
+
+from sqlalchemy import Column, Text, LargeBinary  # pylint: disable=E0401
+
+from tools import context  # pylint: disable=E0401
+
+
+class TunableValue(context.pylon_db.Base):  # pylint: disable=C0111,R0903
+    __tablename__ = "tunable_value"
+
+    tunable = Column(Text, primary_key=True)
+    value = Column(LargeBinary, unique=False, default=b"")

pylon/framework/toolkit.py

@@ -23,13 +23,17 @@
 from pylon.core.tools.module.this import This
 
 
-def init(context):
-    """ Make tools holder and pre-populate toolkit """
+def basic_init(context):
+    """ Make tools holder """
     # Make tools holder
     if "tools" not in sys.modules:
         sys.modules["tools"] = types.ModuleType("tools")
         sys.modules["tools"].__path__ = []
     # Register context as a tool
     setattr(sys.modules["tools"], "context", context)
+
+
+def init(context):
+    """ Pre-populate toolkit """
     # Register module helpers as a tool
     setattr(sys.modules["tools"], "this", This(context))

pylon/main.py

@@ -59,6 +59,7 @@
 from pylon.core.tools import log
 from pylon.core.tools import log_support
 from pylon.core.tools import db_support
+from pylon.core.tools import config
 from pylon.core.tools import module
 from pylon.core.tools import event
 from pylon.core.tools import seed
@@ -71,6 +72,7 @@
 from pylon.core.tools import traefik
 from pylon.core.tools import exposure
 
+from pylon.core.tools.dict import recursive_merge
 from pylon.core.tools.signal import signal_sigterm
 from pylon.core.tools.signal import kill_remaining_processes
 from pylon.core.tools.signal import ZombieReaper
@@ -99,6 +101,24 @@ def main():  # pylint: disable=R0912,R0914,R0915
     if not context.settings:
         log.error("Settings are empty or invalid. Exiting")
         os._exit(1)  # pylint: disable=W0212
+    # Basic init
+    toolkit.basic_init(context)
+    db_support.basic_init(context)
+    # Tunable pylon settings
+    tunable_settings_data = config.tunable_get("pylon_settings", None)
+    if tunable_settings_data is not None:
+        log.info("Loading and parsing tunable settings")
+        tunable_settings = seed.parse_settings(tunable_settings_data)
+        if tunable_settings:
+            tunable_settings_mode = tunable_settings.get("pylon", {}).get(
+                "tunable_settings_mode", "override"
+            )
+            if tunable_settings_mode == "merge":
+                context.settings = recursive_merge(context.settings, tunable_settings)
+            elif tunable_settings_mode == "update":
+                context.settings.update(tunable_settings)
+            else:
+                context.settings = tunable_settings
     # Save reloader status
     context.reloader_used = context.settings.get("server", {}).get(
         "use_reloader",
@@ -108,6 +128,9 @@ def main():  # pylint: disable=R0912,R0914,R0915
             context.debug and \
             context.reloader_used and \
             os.environ.get("WERKZEUG_RUN_MAIN", "false").lower() != "true"
+    # Basic de-init in case reloader is used
+    if context.before_reloader:
+        db_support.basic_deinit(context)
     # Save global node name
     context.node_name = context.settings.get("server", {}).get("name", socket.gethostname())
     # Generate pylon ID