From f616b5b280db9586139cdd1c847c97de5e0a0f28 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 13:40:42 -0300 Subject: [PATCH 1/9] feat: added KytosNAppSetupException exception --- kytos/core/exceptions.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/kytos/core/exceptions.py b/kytos/core/exceptions.py index 93ceceda0..c4f5dad76 100644 --- a/kytos/core/exceptions.py +++ b/kytos/core/exceptions.py @@ -101,6 +101,17 @@ def __str__(self): return self.message +class KytosNAppSetupException(KytosNAppException): + """KytosNAppSetupException. """ + + def __init__(self, message="KytosNAppSetupException") -> None: + """KytosNAppSetupException.""" + super().__init__(message=message) + + def __str__(self): + return f"KytosNAppSetupException: {self.message}" + + class KytosNAppMissingInitArgument(KytosNAppException): """Exception thrown when NApp have a missing init argument.""" From f8f1096eb3336ff0951755b9232354c5ce96ed82 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 13:42:22 -0300 Subject: [PATCH 2/9] feat: added retries for bootstrap_index parametrized maxTimeMS with env var MONGO_IDX_TIMEOUTMS --- kytos/core/db.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kytos/core/db.py b/kytos/core/db.py index ae309f546..89d9aaf26 100644 --- a/kytos/core/db.py +++ b/kytos/core/db.py @@ -11,8 +11,11 @@ import pymongo.helpers from pymongo import MongoClient from pymongo.errors import AutoReconnect, OperationFailure +from tenacity import (retry, retry_if_exception_type, stop_after_attempt, + wait_random) from kytos.core.exceptions import KytosDBInitException +from kytos.core.retry import before_sleep LOG = logging.getLogger(__name__) @@ -88,6 +91,18 @@ class Mongo: db_name = os.environ.get("MONGO_DBNAME") or "napps" @classmethod + @retry( + stop=stop_after_attempt( + int(os.environ.get("MONGO_AUTO_RETRY_STOP_AFTER_ATTEMPT", 3)) + ), + wait=wait_random( + min=int(os.environ.get("MONGO_AUTO_RETRY_WAIT_RANDOM_MIN", 0.1)), + max=int(os.environ.get("MONGO_AUTO_RETRY_WAIT_RANDOM_MAX", 1)), + ), + before_sleep=before_sleep, + retry=retry_if_exception_type((OperationFailure, AutoReconnect)), + reraise=True + ) def bootstrap_index( cls, collection: str, @@ -99,6 +114,9 @@ def bootstrap_index( indexes = set() if "background" not in kwargs: kwargs["background"] = True + if "maxTimeMS" not in kwargs: + timeout_ms = int(os.environ.get("MONGO_IDX_TIMEOUTMS") or 30000) + kwargs["maxTimeMS"] = timeout_ms for value in db[collection].index_information().values(): if "key" in value and isinstance(value["key"], list): From 39c8e46006898aaeddaa3da305ad9525bd19508a Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 13:43:27 -0300 Subject: [PATCH 3/9] feat: handled broad exceptions on controller.start --- kytos/core/controller.py | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/kytos/core/controller.py b/kytos/core/controller.py index bf43d5bcd..85f930c4f 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -41,7 +41,8 @@ from kytos.core.db import db_conn_wait from kytos.core.dead_letter import DeadLetter from kytos.core.events import KytosEvent -from kytos.core.exceptions import KytosAPMInitException, KytosDBInitException +from kytos.core.exceptions import (KytosAPMInitException, KytosDBInitException, + KytosNAppSetupException) from kytos.core.helpers import executors, now from kytos.core.interface import Interface from kytos.core.logs import LogManager @@ -249,14 +250,20 @@ def toggle_debug(self, name=None): def start(self, restart=False): """Create pidfile and call start_controller method.""" self.enable_logs() - if self.options.database: - self.db_conn_or_core_shutdown() - self.start_auth() - if self.options.apm: - self.init_apm_or_core_shutdown() - if not restart: - self.create_pidfile() - self.start_controller() + # pylint: disable=broad-except + try: + if self.options.database: + self.db_conn_or_core_shutdown() + self.start_auth() + if self.options.apm: + self.init_apm_or_core_shutdown() + if not restart: + self.create_pidfile() + self.start_controller() + except Exception as exc: + message = f"Kytos couldn't start because of {str(exc)}" + self.log.exception(message, exc_info=True) + sys.exit(message) def create_pidfile(self): """Create a pidfile.""" @@ -877,10 +884,11 @@ def load_napp(self, username, napp_name): try: napp = napp_module.Main(controller=self) - except: # noqa pylint: disable=bare-except + except Exception as exc: # noqa pylint: disable=bare-except self.log.critical("NApp initialization failed: %s/%s", username, napp_name, exc_info=True) - return + msg = f"NApp {username}/{napp_name} exception {str(exc)} " + raise KytosNAppSetupException(msg) self.napps[(username, napp_name)] = napp @@ -916,6 +924,8 @@ def load_napps(self): except FileNotFoundError as exception: self.log.error("Could not load NApp %s: %s", napp.id, exception) + msg = f"NApp {napp.id} exception {str(exception)}" + raise KytosNAppSetupException(msg) def unload_napp(self, username, napp_name): """Unload a specific NApp. From 0ed1a320c20197c2be113028b3c53980297d9630 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 13:44:02 -0300 Subject: [PATCH 4/9] tests: augmented create_index unit test --- tests/unit/test_core/test_db.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/unit/test_core/test_db.py b/tests/unit/test_core/test_db.py index 3c96c5c6d..cf4aa667f 100644 --- a/tests/unit/test_core/test_db.py +++ b/tests/unit/test_core/test_db.py @@ -92,13 +92,16 @@ def test_boostrap_index(self) -> None: keys = [("interfaces.id", 1)] Mongo().bootstrap_index(coll, keys) assert db[coll].create_index.call_count == 1 - db[coll].create_index.assert_called_with(keys, background=True) + db[coll].create_index.assert_called_with(keys, + background=True, + maxTimeMS=30000) keys = [("interfaces.id", 1), ("interfaces.name", 1)] Mongo().bootstrap_index(coll, keys) assert db[coll].create_index.call_count == 2 db[coll].create_index.assert_called_with(keys, - background=True) + background=True, + maxTimeMS=30000) @staticmethod @patch("kytos.core.db.LOG") From eb65a699805a9117004f6fe9aef7d326fce2a250 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 13:44:32 -0300 Subject: [PATCH 5/9] tests: augmented controller unit tests accordingly --- tests/unit/test_core/test_controller.py | 27 ++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_core/test_controller.py b/tests/unit/test_core/test_controller.py index 14c1379d2..2a990c7c0 100644 --- a/tests/unit/test_core/test_controller.py +++ b/tests/unit/test_core/test_controller.py @@ -14,6 +14,7 @@ from kytos.core.buffers import KytosBuffers from kytos.core.config import KytosConfig from kytos.core.events import KytosEvent +from kytos.core.exceptions import KytosNAppSetupException from kytos.core.logs import LogManager @@ -181,6 +182,29 @@ def test_start(self, *args): mock_db_conn_or_shutdown.assert_not_called() mock_init_apm_or_shutdown.assert_not_called() + @patch('kytos.core.controller.sys') + @patch('kytos.core.controller.Controller.init_apm_or_core_shutdown') + @patch('kytos.core.controller.Controller.db_conn_or_core_shutdown') + @patch('kytos.core.controller.Controller.start_controller') + @patch('kytos.core.controller.Controller.create_pidfile') + @patch('kytos.core.controller.Controller.enable_logs') + def test_start_error_broad_exception(self, *args): + """Test start error handling broad exception.""" + (mock_enable_logs, mock_create_pidfile, + mock_start_controller, mock_db_conn_or_shutdown, + mock_init_apm_or_shutdown, mock_sys) = args + mock_start_controller.side_effect = Exception + self.controller.log = MagicMock() + self.controller.start() + + mock_enable_logs.assert_called() + mock_create_pidfile.assert_called() + mock_start_controller.assert_called() + mock_db_conn_or_shutdown.assert_not_called() + mock_init_apm_or_shutdown.assert_not_called() + self.controller.log.exception.assert_called() + mock_sys.exit.assert_called() + @patch('kytos.core.controller.Controller.init_apm_or_core_shutdown') @patch('kytos.core.controller.Controller.db_conn_or_core_shutdown') @patch('kytos.core.controller.Controller.start_controller') @@ -545,7 +569,8 @@ def test_load_napp__error(self, *args): module.Main.side_effect = Exception mock_import_napp.return_value = module - self.controller.load_napp('kytos', 'napp') + with self.assertRaises(KytosNAppSetupException): + self.controller.load_napp('kytos', 'napp') self.assertEqual(self.controller.napps, {}) From 94f6233aa400d5c885bce1e9d37b599d1b6466ea Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 14:38:10 -0300 Subject: [PATCH 6/9] feat: replaced self.log.exception with print trace to be deterministic --- kytos/core/controller.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kytos/core/controller.py b/kytos/core/controller.py index 85f930c4f..5d5506cd9 100644 --- a/kytos/core/controller.py +++ b/kytos/core/controller.py @@ -22,6 +22,7 @@ import re import sys import threading +import traceback from concurrent.futures import ThreadPoolExecutor from importlib import import_module from importlib import reload as reload_module @@ -261,9 +262,10 @@ def start(self, restart=False): self.create_pidfile() self.start_controller() except Exception as exc: - message = f"Kytos couldn't start because of {str(exc)}" - self.log.exception(message, exc_info=True) - sys.exit(message) + exc_fmt = traceback.format_exc(chain=True) + message = f"Kytos couldn't start because of {str(exc)} {exc_fmt}" + print(message) + sys.exit(1) def create_pidfile(self): """Create a pidfile.""" @@ -885,10 +887,8 @@ def load_napp(self, username, napp_name): try: napp = napp_module.Main(controller=self) except Exception as exc: # noqa pylint: disable=bare-except - self.log.critical("NApp initialization failed: %s/%s", - username, napp_name, exc_info=True) msg = f"NApp {username}/{napp_name} exception {str(exc)} " - raise KytosNAppSetupException(msg) + raise KytosNAppSetupException(msg) from exc self.napps[(username, napp_name)] = napp From b3052606276524336c00bfde6ce8484a4a7b7140 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 14:38:24 -0300 Subject: [PATCH 7/9] tests: refactored unit test accordingly --- tests/unit/test_core/test_controller.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/unit/test_core/test_controller.py b/tests/unit/test_core/test_controller.py index 2a990c7c0..39a9c7e14 100644 --- a/tests/unit/test_core/test_controller.py +++ b/tests/unit/test_core/test_controller.py @@ -194,7 +194,6 @@ def test_start_error_broad_exception(self, *args): mock_start_controller, mock_db_conn_or_shutdown, mock_init_apm_or_shutdown, mock_sys) = args mock_start_controller.side_effect = Exception - self.controller.log = MagicMock() self.controller.start() mock_enable_logs.assert_called() @@ -202,7 +201,6 @@ def test_start_error_broad_exception(self, *args): mock_start_controller.assert_called() mock_db_conn_or_shutdown.assert_not_called() mock_init_apm_or_shutdown.assert_not_called() - self.controller.log.exception.assert_called() mock_sys.exit.assert_called() @patch('kytos.core.controller.Controller.init_apm_or_core_shutdown') From 7e9d3c7fe2e8c7b6cce12749bb0213dea8bd6706 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 15:15:27 -0300 Subject: [PATCH 8/9] bump: bumped 2022.3.3 --- docs/conf.py | 4 ++-- kytos/core/metadata.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index 68a22436b..151296ef8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -64,10 +64,10 @@ # built documents. # # The short X.Y version. -version = u'2022.3.2' +version = u'2022.3.3' show_version = False # The full version, including alpha/beta/rc tags. -release = u'2022.3.2' +release = u'2022.3.3' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/kytos/core/metadata.py b/kytos/core/metadata.py index 50e3dc177..830d70e30 100644 --- a/kytos/core/metadata.py +++ b/kytos/core/metadata.py @@ -2,7 +2,7 @@ The present metadata is intended to be used mainly on the setup. """ -__version__ = '2022.3.2' +__version__ = '2022.3.3' __author__ = 'Kytos Team' __author_email__ = 'devel@lists.kytos.io' __license__ = 'MIT' From 8be9f926456cebb833f9edacc6b2752767a3c2e6 Mon Sep 17 00:00:00 2001 From: Vinicius Arcanjo Date: Tue, 26 Sep 2023 15:16:18 -0300 Subject: [PATCH 9/9] Updated CHANGELOG.rst --- CHANGELOG.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 526dc09ec..5799890a4 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -6,6 +6,15 @@ All notable changes to the kytos project will be documented in this file. UNRELEASED - Under development ****************************** +[2022.3.3] 2023-09-26 +********************** + +Changed +======= +- Parametrized default ``maxTimeMS`` when creating an index via ``Mongo.boostrap_index`` via environment variable ``MONGO_IDX_TIMEOUTMS=30000``. The retries parameters reuse the same environment variables ``MONGO_AUTO_RETRY_STOP_AFTER_ATTEMPT=3``, ``MONGO_AUTO_RETRY_WAIT_RANDOM_MIN=0.1``, ``MONGO_AUTO_RETRY_WAIT_RANDOM_MAX=1`` that NApps controllers have been using. +- ``kytosd`` process will exit if a NApp raises an exception during its ``setup()`` execution. + + [2022.3.2] 2023-06-19 **********************