diff --git a/releases/unreleased/reduced-the-number-of-connections-to-sortinghat.yml b/releases/unreleased/reduced-the-number-of-connections-to-sortinghat.yml new file mode 100644 index 00000000..c74df8c9 --- /dev/null +++ b/releases/unreleased/reduced-the-number-of-connections-to-sortinghat.yml @@ -0,0 +1,9 @@ +--- +title: Reduced the number of connections to SortingHat +category: performance +author: Quan Zhou +issue: null +notes: > + Mordred makes a lot of connections to the SortingHat server + which could cause the uWSGI queue to fill up. When the uWSGI + queue is full, Mordred cannot connect to the SortingHat server. diff --git a/sirmordred/sirmordred.py b/sirmordred/sirmordred.py index 32036f7f..5bc8bca7 100755 --- a/sirmordred/sirmordred.py +++ b/sirmordred/sirmordred.py @@ -50,6 +50,7 @@ from sirmordred.task_manager import TasksManager from sirmordred.task_panels import TaskPanels, TaskPanelsMenu from sirmordred.task_projects import TaskProjects +from sortinghat.cli.client import SortingHatClient logger = logging.getLogger(__name__) @@ -198,7 +199,7 @@ def _split_tasks(tasks_cls): repos_backend = self._get_repos_by_backend() for backend in repos_backend: # Start new Threads and add them to the threads list to complete - t = TasksManager(backend_tasks, backend, stopper, self.config, small_delay) + t = TasksManager(backend_tasks, backend, stopper, self.config, self.client, small_delay) threads.append(t) t.start() @@ -206,7 +207,7 @@ def _split_tasks(tasks_cls): if len(global_tasks) > 0: # FIXME timer is applied to all global_tasks, does it make sense? # All tasks are executed in the same thread sequentially - gt = TasksManager(global_tasks, "Global tasks", stopper, self.config, big_delay) + gt = TasksManager(global_tasks, "Global tasks", stopper, self.config, self.client, big_delay) threads.append(gt) gt.start() if big_delay > 0: @@ -248,14 +249,14 @@ def __execute_initial_load(self): if self.conf['phases']['panels']: tasks = [TaskPanels, TaskPanelsMenu] stopper.set() - tm = TasksManager(tasks, "Global tasks", stopper, self.config) + tm = TasksManager(tasks, "Global tasks", stopper, self.config, self.client) tm.start() tm.join() logger.info("Loading projects") tasks = [TaskProjects] stopper.set() - tm = TasksManager(tasks, "Global tasks", stopper, self.config) + tm = TasksManager(tasks, "Global tasks", stopper, self.config, self.client) tm.start() tm.join() logger.info("Projects loaded") @@ -280,7 +281,7 @@ def start(self): # check we have access to the needed ES if not self.check_es_access(): - print('Can not access Elasticsearch service. Exiting sirmordred ...') + print('Can not access ElasticSearch/OpenSearch service. Exiting sirmordred ...') sys.exit(1) # If bestiary is configured check that it is working @@ -289,6 +290,9 @@ def start(self): print('Can not access bestiary service. Exiting sirmordred ...') sys.exit(1) + # Create SortingHat Client + self.__create_sh_client(self.config) + # Initial round: panels and projects loading self.__execute_initial_load() @@ -336,3 +340,28 @@ def start(self): logger.error(var) logger.info("Finished SirMordred engine ...") + + def __create_sh_client(self, config): + self.config = config + self.conf = config.get_conf() + + sortinghat = self.conf.get('sortinghat', None) + self.db_sh = sortinghat['database'] if sortinghat else None + self.db_user = sortinghat['user'] if sortinghat else None + self.db_password = sortinghat['password'] if sortinghat else None + self.db_host = sortinghat['host'] if sortinghat else '127.0.0.1' + self.db_path = sortinghat.get('path', None) if sortinghat else None + self.db_port = sortinghat.get('port', None) if sortinghat else None + self.db_ssl = sortinghat.get('ssl', False) if sortinghat else False + self.db_verify_ssl = sortinghat.get('verify_ssl', True) if sortinghat else True + self.db_tenant = sortinghat.get('tenant', True) if sortinghat else None + self.db_unaffiliate_group = sortinghat['unaffiliated_group'] if sortinghat else None + if sortinghat and not hasattr(self, 'client'): + self.client = SortingHatClient(host=self.db_host, port=self.db_port, + path=self.db_path, ssl=self.db_ssl, + user=self.db_user, password=self.db_password, + verify_ssl=self.db_verify_ssl, + tenant=self.db_tenant) + self.client.connect() + elif not sortinghat: + self.client = None diff --git a/sirmordred/task.py b/sirmordred/task.py index 652dbc58..b82f78e4 100644 --- a/sirmordred/task.py +++ b/sirmordred/task.py @@ -29,7 +29,6 @@ from grimoire_elk.elk import get_ocean_backend from grimoire_elk.utils import get_connector_from_name, get_elastic from grimoire_elk.enriched.utils import grimoire_con -from sortinghat.cli.client import SortingHatClient logger = logging.getLogger(__name__) @@ -42,10 +41,11 @@ class Task(): 'studies', 'node_regex', 'anonymize'] PARAMS_WITH_SPACES = ['blacklist-jobs'] - def __init__(self, config): + def __init__(self, config, sortinghat_client=None): self.backend_section = None self.config = config self.conf = config.get_conf() + self.client = sortinghat_client sortinghat = self.conf.get('sortinghat', None) self.db_sh = sortinghat['database'] if sortinghat else None @@ -58,15 +58,6 @@ def __init__(self, config): self.db_verify_ssl = sortinghat.get('verify_ssl', True) if sortinghat else True self.db_tenant = sortinghat.get('tenant', True) if sortinghat else None self.db_unaffiliate_group = sortinghat['unaffiliated_group'] if sortinghat else None - if sortinghat: - self.client = SortingHatClient(host=self.db_host, port=self.db_port, - path=self.db_path, ssl=self.db_ssl, - user=self.db_user, password=self.db_password, - verify_ssl=self.db_verify_ssl, - tenant=self.db_tenant) - self.client.connect() - else: - self.client = None self.grimoire_con = grimoire_con(conn_retries=12) # 30m retry diff --git a/sirmordred/task_autorefresh.py b/sirmordred/task_autorefresh.py index 3b81d434..7a895935 100644 --- a/sirmordred/task_autorefresh.py +++ b/sirmordred/task_autorefresh.py @@ -43,8 +43,8 @@ class TaskAutorefresh(Task): """Refresh the last modified identities for all the backends.""" - def __init__(self, config): - super().__init__(config) + def __init__(self, config, sortinghat_client): + super().__init__(config, sortinghat_client) self.last_autorefresh_backend = {} diff --git a/sirmordred/task_enrich.py b/sirmordred/task_enrich.py index bb107e43..e600c287 100644 --- a/sirmordred/task_enrich.py +++ b/sirmordred/task_enrich.py @@ -53,8 +53,8 @@ class TaskEnrich(Task): """ Basic class shared by all enriching tasks """ - def __init__(self, config, backend_section=None, allowed_repos=None): - super().__init__(config) + def __init__(self, config, sortinghat_client, backend_section=None, allowed_repos=None): + super().__init__(config, sortinghat_client) self.backend_section = backend_section self.allowed_repos = set(allowed_repos) if allowed_repos else None # This will be options in next iteration diff --git a/sirmordred/task_identities.py b/sirmordred/task_identities.py index a0fe99e3..7436f328 100644 --- a/sirmordred/task_identities.py +++ b/sirmordred/task_identities.py @@ -38,8 +38,8 @@ class TaskIdentitiesMerge(Task): """ Task for processing identities in SortingHat """ - def __init__(self, conf): - super().__init__(conf) + def __init__(self, conf, soringhat_client): + super().__init__(conf, soringhat_client) self.last_autorefresh = datetime.utcnow() # Last autorefresh date def is_backend_task(self): diff --git a/sirmordred/task_manager.py b/sirmordred/task_manager.py index 058a890e..13626597 100644 --- a/sirmordred/task_manager.py +++ b/sirmordred/task_manager.py @@ -50,7 +50,7 @@ class TasksManager(threading.Thread): IDENTITIES_TASKS_ON_LOCK = threading.Lock() IDENTITIES_TASKS_ON = False - def __init__(self, tasks_cls, backend_section, stopper, config, timer=0): + def __init__(self, tasks_cls, backend_section, stopper, config, sortinghat_client, timer=0): """ :tasks_cls : tasks classes to be executed using the backend :backend_section: perceval backend section name @@ -64,6 +64,7 @@ def __init__(self, tasks_cls, backend_section, stopper, config, timer=0): self.stopper = stopper # To stop the thread from parent self.timer = timer self.thread_id = None + self.client = sortinghat_client def add_task(self, task): self.tasks.append(task) @@ -80,7 +81,7 @@ def __set_thread_id(): logger.debug(self.tasks_cls) for tc in self.tasks_cls: # create the real Task from the class - task = tc(self.config) + task = tc(self.config, self.client) task.set_backend_section(self.backend_section) self.tasks.append(task) diff --git a/tests/test_task.py b/tests/test_task.py index 8e1998f6..c68a6cd3 100644 --- a/tests/test_task.py +++ b/tests/test_task.py @@ -33,6 +33,8 @@ from sirmordred.config import Config from sirmordred.task import Task +from sortinghat.cli.client import SortingHatClient + CONF_FILE = 'test.cfg' BACKEND_NAME = 'stackexchange' COLLECTION_URL_STACKEXCHANGE = 'http://127.0.0.1:9200' @@ -47,14 +49,23 @@ def read_file(filename, mode='r'): class TestTask(unittest.TestCase): """Task tests""" + def setUp(self): + self.config = Config(CONF_FILE) + self.conf = self.config.get_conf() + sh = self.conf.get('sortinghat') + self.sortinghat_client = SortingHatClient(host=sh['host'], port=sh.get('port', None), + path=sh.get('path', None), ssl=sh.get('ssl', False), + user=sh['user'], password=sh['password'], + verify_ssl=sh.get('verify_ssl', True), + tenant=sh.get('tenant', True)) + self.sortinghat_client.connect() def test_initialization(self): """Test whether attributes are initializated""" - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) - self.assertEqual(task.config, config) + self.assertEqual(task.config, self.config) self.assertEqual(task.db_sh, task.conf['sortinghat']['database']) self.assertEqual(task.db_user, task.conf['sortinghat']['user']) self.assertEqual(task.db_password, task.conf['sortinghat']['password']) @@ -63,15 +74,13 @@ def test_initialization(self): def test_run(self): """Test whether the Task could be run""" - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) self.assertEqual(task.execute(), None) def test_compose_p2o_params(self): """Test whether p2o params are built correctly for a backend and a repository""" - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) params = task._compose_p2o_params("stackexchange", "https://stackoverflow.com/questions/tagged/example") self.assertDictEqual(params, {'url': "https://stackoverflow.com/questions/tagged/example"}) @@ -92,8 +101,7 @@ def test_compose_p2o_params(self): def test_extract_repo_tags(self): """Test the extraction of tags in repositories""" - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) url, tags = task._extract_repo_tags("git", "https://github.com/zhquan_example/repo --labels=[ENG, SUPP]") self.assertEqual(url, "https://github.com/zhquan_example/repo") self.assertListEqual(tags, ["ENG", "SUPP"]) @@ -113,8 +121,7 @@ def test_compose_perceval_params(self): expected_repo_params = json.loads(read_file('data/task-params-expected')) - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) for backend in expected_repo_params.keys(): repo = expected_repo_params.get(backend)['repo'] @@ -126,8 +133,7 @@ def test_compose_perceval_params(self): def test_get_collection_url(self): """Test whether the collection url could be overwritten in a backend""" - config = Config(CONF_FILE) - task = Task(config) + task = Task(self.config, self.sortinghat_client) task.backend_section = "stackexchange" self.assertEqual(task._get_collection_url(), COLLECTION_URL_STACKEXCHANGE) diff --git a/tests/test_task_autorefresh.py b/tests/test_task_autorefresh.py index 78143d10..6e2d382d 100644 --- a/tests/test_task_autorefresh.py +++ b/tests/test_task_autorefresh.py @@ -39,7 +39,7 @@ from sirmordred.task_enrich import TaskEnrich from sirmordred.task_projects import TaskProjects -from sortinghat.cli.client import SortingHatSchema +from sortinghat.cli.client import SortingHatClient, SortingHatSchema from sgqlc.operation import Operation @@ -131,8 +131,16 @@ def add_domain(client, args): client.execute(op) def setUp(self): - config = Config(CONF_FILE) - task = TaskAutorefresh(config) + self.config = Config(CONF_FILE) + self.conf = self.config.get_conf() + sh = self.conf.get('sortinghat') + self.sortinghat_client = SortingHatClient(host=sh['host'], port=sh.get('port', None), + path=sh.get('path', None), ssl=sh.get('ssl', False), + user=sh['user'], password=sh['password'], + verify_ssl=sh.get('verify_ssl', True), + tenant=sh.get('tenant', True)) + self.sortinghat_client.connect() + task = TaskAutorefresh(self.config, self.sortinghat_client) # Clean database entities = SortingHat.unique_identities(task.client) @@ -162,16 +170,14 @@ def setUp(self): def test_initialization(self): """Test whether attributes are initialized""" - config = Config(CONF_FILE) - task = TaskAutorefresh(config) + task = TaskAutorefresh(self.config, self.sortinghat_client) - self.assertEqual(task.config, config) + self.assertEqual(task.config, self.config) def test_is_backend_task(self): """Test whether the Task is not a backend task""" - config = Config(CONF_FILE) - task = TaskAutorefresh(config) + task = TaskAutorefresh(self.config, self.sortinghat_client) self.assertFalse(task.is_backend_task()) @@ -179,18 +185,16 @@ def test_execute(self): """Test whether the Task could be run""" # Create a raw and enriched indexes - config = Config(CONF_FILE) - - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION - task_collection = TaskRawDataCollection(config, backend_section=backend_section) + task_collection = TaskRawDataCollection(self.config, backend_section=backend_section) task_collection.execute() - task_enrich = TaskEnrich(config, backend_section=backend_section) + task_enrich = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) task_enrich.execute() - task_autorefresh = TaskAutorefresh(config) + task_autorefresh = TaskAutorefresh(self.config, self.sortinghat_client) task_autorefresh.config.set_param('es_enrichment', 'autorefresh', True) # This does nothing because it uses now as from_date: task_autorefresh.execute() @@ -205,7 +209,7 @@ def test_execute(self): self.assertIsNone(task_autorefresh.execute()) # Check that the autorefresh went well - cfg = config.get_conf() + cfg = self.conf es_enrichment = cfg['es_enrichment']['url'] enrich_index = es_enrichment + "/" + cfg[GIT_BACKEND_SECTION]['enriched_index'] diff --git a/tests/test_task_collection.py b/tests/test_task_collection.py index 7517e604..4d9d6fc5 100644 --- a/tests/test_task_collection.py +++ b/tests/test_task_collection.py @@ -111,7 +111,8 @@ def test_execute(self): raw_index = es_collection + "/" + cfg[GIT_BACKEND_SECTION]['raw_index'] r = requests.get(raw_index + "/_search?size=0", verify=False) - raw_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + raw_items = total['value'] if isinstance(total, dict) else total self.assertEqual(raw_items, 3603) def test_execute_no_collection(self): @@ -130,7 +131,8 @@ def test_execute_no_collection(self): raw_index = es_collection + "/" + cfg[GIT_BACKEND_SECTION]['raw_index'] r = requests.get(raw_index + "/_search?size=0", verify=False) - raw_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + raw_items = total['value'] if isinstance(total, dict) else total self.assertEqual(raw_items, 40) def test_execute_filter_no_collection(self): diff --git a/tests/test_task_enrich.py b/tests/test_task_enrich.py index 0fbc60d3..44a3c0b9 100644 --- a/tests/test_task_enrich.py +++ b/tests/test_task_enrich.py @@ -40,7 +40,7 @@ from sirmordred.task_collection import TaskRawDataCollection from sirmordred.task_enrich import TaskEnrich -from sortinghat.cli.client import (SortingHatSchema) +from sortinghat.cli.client import SortingHatClient, SortingHatSchema from sgqlc.operation import Operation @@ -62,6 +62,21 @@ def read_file(filename, mode='r'): class TestTaskEnrich(unittest.TestCase): """Task tests""" + def _setup(self, conf_file): + self.config = Config(conf_file) + self.conf = self.config.get_conf() + sh = self.conf.get('sortinghat', None) + if sh: + self.sortinghat_client = SortingHatClient(host=sh['host'], port=sh.get('port', None), + path=sh.get('path', None), ssl=sh.get('ssl', False), + user=sh['user'], password=sh['password'], + verify_ssl=sh.get('verify_ssl', True), + tenant=sh.get('tenant', True)) + self.sortinghat_client.connect() + else: + + self.sortinghat_client = None + @staticmethod def get_organizations(task): args = { @@ -91,8 +106,8 @@ def delete_organization(task, args): task.client.execute(op) def setUp(self): - config = Config(CONF_FILE) - task = TaskEnrich(config) + self._setup(CONF_FILE) + task = TaskEnrich(self.config, self.sortinghat_client) # Clean database # Remove identities @@ -112,18 +127,18 @@ def setUp(self): def test_initialization(self): """Test whether attributes are initializated""" - config = Config(CONF_FILE) + self._setup(CONF_FILE) backend_section = GIT_BACKEND_SECTION - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) - self.assertEqual(task.config, config) + self.assertEqual(task.config, self.config) self.assertEqual(task.backend_section, backend_section) def test_select_aliases(self): - config = Config(CONF_FILE) - cfg = config.get_conf() + self._setup(CONF_FILE) + cfg = self.conf # We need to load the projects - task = TaskEnrich(config) + task = TaskEnrich(self.config, self.sortinghat_client) expected_aliases = [ 'git', 'git_author', @@ -135,8 +150,9 @@ def test_select_aliases(self): def test_retain_identities(self): """""" - config = Config(CONF_FILE) - cfg = config.get_conf() + + self._setup(CONF_FILE) + cfg = self.conf # Remove old enriched index es_enrichment = cfg['es_enrichment']['url'] @@ -144,15 +160,15 @@ def test_retain_identities(self): requests.delete(enrich_index, verify=False) # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION # Create raw data - task_collection = TaskRawDataCollection(config, backend_section=backend_section) + task_collection = TaskRawDataCollection(self.config, backend_section=backend_section) task_collection.execute() # Create enriched data - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) self.assertEqual(task.execute(), None) entities_before = SortingHat.unique_identities(task.client) @@ -176,25 +192,25 @@ def test_retain_identities(self): self.assertGreater(len(entities_before), len(entities_after)) def test_execute_retain_data(self): - config = Config(CONF_FILE) - cfg = config.get_conf() + self._setup(CONF_FILE) + cfg = self.conf # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION # Create raw data - task_collection = TaskRawDataCollection(config, backend_section=backend_section) + task_collection = TaskRawDataCollection(self.config, backend_section=backend_section) task_collection.execute() # Test enriched data - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) task.execute() es_enrichment = cfg['es_enrichment']['url'] enrich_index = es_enrichment + "/" + cfg[GIT_BACKEND_SECTION]['enriched_index'] r = requests.get(enrich_index + "/_search?size=0", verify=False) - enriched_items_before = r.json()['hits']['total'] - # enriched_items_before = r.json()['hits']['total']['value'] + total = r.json()['hits']['total'] + enriched_items_before = total['value'] if isinstance(total, dict) else total # 1 year retention_time = 525600 @@ -202,26 +218,26 @@ def test_execute_retain_data(self): task.execute() r = requests.get(enrich_index + "/_search?size=0", verify=False) - enriched_items_after = r.json()['hits']['total'] - # enriched_items_after = r.json()['hits']['total']['value'] + total = r.json()['hits']['total'] + enriched_items_after = total['value'] if isinstance(total, dict) else total self.assertGreater(enriched_items_before, enriched_items_after) def test_execute(self): """Test whether the Task could be run""" - config = Config(CONF_FILE) - cfg = config.get_conf() + self._setup(CONF_FILE) + cfg = self.conf # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION # Create raw data - task_collection = TaskRawDataCollection(config, backend_section=backend_section) + task_collection = TaskRawDataCollection(self.config, backend_section=backend_section) task_collection.execute() # Test enriched data - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) self.assertEqual(task.execute(), None) # Check that the enrichment went well @@ -231,27 +247,29 @@ def test_execute(self): enrich_index = es_enrichment + "/" + cfg[GIT_BACKEND_SECTION]['enriched_index'] r = requests.get(raw_index + "/_search?size=0", verify=False) - raw_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + raw_items = total['value'] if isinstance(total, dict) else total r = requests.get(enrich_index + "/_search?size=0", verify=False) - enriched_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + enriched_items = total['value'] if isinstance(total, dict) else total self.assertEqual(raw_items, enriched_items) def test_execute_no_sh(self): """Test whether the Task could be run without SortingHat""" - config = Config(CONF_FILE_NO_SH) - cfg = config.get_conf() + self._setup(CONF_FILE_NO_SH) + cfg = self.conf # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION # Create raw data - task_collection = TaskRawDataCollection(config, backend_section=backend_section) + task_collection = TaskRawDataCollection(self.config, backend_section=backend_section) task_collection.execute() # Test enriched data - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) self.assertEqual(task.execute(), None) # Check that the enrichment went well @@ -261,20 +279,23 @@ def test_execute_no_sh(self): enrich_index = es_enrichment + "/" + cfg[GIT_BACKEND_SECTION]['enriched_index'] r = requests.get(raw_index + "/_search?size=0", verify=False) - raw_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + raw_items = total['value'] if isinstance(total, dict) else total + r = requests.get(enrich_index + "/_search?size=0", verify=False) - enriched_items = r.json()['hits']['total'] + total = r.json()['hits']['total'] + enriched_items = total['value'] if isinstance(total, dict) else total self.assertEqual(raw_items, enriched_items) def test_studies(self): """Test whether the studies configuration works """ - config = Config(CONF_FILE) - cfg = config.get_conf() + self._setup(CONF_FILE) + cfg = self.conf # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() backend_section = GIT_BACKEND_SECTION - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) # Configure no studies cfg.set_param('git', 'studies', None) @@ -311,7 +332,7 @@ def test_execute_from_archive(self): # proj_file -> 'test-projects-archive.json' stored within the conf file conf_file = 'archives-test.cfg' - config = Config(conf_file) + self._setup(conf_file) backend_sections = ['askbot', 'bugzilla', 'bugzillarest', 'confluence', 'discourse', 'dockerhub', 'gerrit', 'github:issue', 'github:pull', @@ -320,13 +341,13 @@ def test_execute_from_archive(self): 'redmine', 'remo', 'rss', 'stackexchange', 'slack', 'telegram', 'twitter'] # We need to load the projects - TaskProjects(config).execute() + TaskProjects(self.config, self.sortinghat_client).execute() for backend_section in backend_sections: - task = TaskRawDataCollection(config, backend_section=backend_section) + task = TaskRawDataCollection(self.config, backend_section=backend_section) task.execute() for backend_section in backend_sections: - task = TaskEnrich(config, backend_section=backend_section) + task = TaskEnrich(self.config, self.sortinghat_client, backend_section=backend_section) self.assertEqual(task.execute(), None) diff --git a/tests/test_task_identities.py b/tests/test_task_identities.py index fd1ac17c..d7c674c0 100644 --- a/tests/test_task_identities.py +++ b/tests/test_task_identities.py @@ -37,7 +37,7 @@ from sirmordred.config import Config from sirmordred.task_identities import TaskIdentitiesMerge -from sortinghat.cli.client import SortingHatSchema +from sortinghat.cli.client import SortingHatClient, SortingHatSchema CONF_FILE = 'test.cfg' @@ -55,6 +55,17 @@ def read_file(filename, mode='r'): class TestTaskIdentitiesMerge(unittest.TestCase): """Task tests""" + def _setup(self, conf_file): + self.config = Config(conf_file) + self.conf = self.config.get_conf() + sh = self.conf.get('sortinghat') + self.sortinghat_client = SortingHatClient(host=sh['host'], port=sh.get('port', None), + path=sh.get('path', None), ssl=sh.get('ssl', False), + user=sh['user'], password=sh['password'], + verify_ssl=sh.get('verify_ssl', True), + tenant=sh.get('tenant', True)) + self.sortinghat_client.connect() + @staticmethod def get_organizations(task): args = { @@ -105,8 +116,8 @@ def add_domain(task, args): task.client.execute(op) def setUp(self): - config = Config(CONF_FILE) - task = TaskIdentitiesMerge(config) + self._setup(CONF_FILE) + task = TaskIdentitiesMerge(self.config, self.sortinghat_client) # Clean database # Remove identities @@ -137,20 +148,20 @@ def setUp(self): def test_initialization(self): """Test whether attributes are initializated""" - config = Config(CONF_FILE) - task = TaskIdentitiesMerge(config) + self._setup(CONF_FILE) + task = TaskIdentitiesMerge(self.config, self.sortinghat_client) - self.assertEqual(task.config, config) + self.assertEqual(task.config, self.config) def test_is_backend_task(self): - config = Config(CONF_FILE) - task = TaskIdentitiesMerge(config) + self._setup(CONF_FILE) + task = TaskIdentitiesMerge(self.config, self.sortinghat_client) self.assertFalse(task.is_backend_task()) def test_execute(self): - config = Config(CONF_FILE) - task = TaskIdentitiesMerge(config) + self._setup(CONF_FILE) + task = TaskIdentitiesMerge(self.config, self.sortinghat_client) self.assertIsNone(task.execute()) args = { diff --git a/tests/test_task_manager.py b/tests/test_task_manager.py index 0f71f33e..c09c7201 100644 --- a/tests/test_task_manager.py +++ b/tests/test_task_manager.py @@ -34,6 +34,8 @@ from sirmordred.task_enrich import TaskEnrich from sirmordred.task_projects import TaskProjects +from sortinghat.cli.client import SortingHatClient + CONF_FILE = 'test.cfg' @@ -42,9 +44,17 @@ class TestTasksManager(unittest.TestCase): def setUp(self): self.config = Config(CONF_FILE) + self.conf = self.config.get_conf() + sh = self.conf.get('sortinghat') + self.sortinghat_client = SortingHatClient(host=sh['host'], port=sh.get('port', None), + path=sh.get('path', None), ssl=sh.get('ssl', False), + user=sh['user'], password=sh['password'], + verify_ssl=sh.get('verify_ssl', True), + tenant=sh.get('tenant', True)) + self.sortinghat_client.connect() mordred = SirMordred(self.config) - task = TaskProjects(self.config) + task = TaskProjects(self.config, self.sortinghat_client) self.assertEqual(task.execute(), None) self.backends = mordred._get_repos_by_backend() @@ -217,11 +227,12 @@ def test_repos_by_backend(self): self.assertEqual(self.backends[backend], ['bitergia']) def test_initialization(self): - """Test whether attributes are initializated""" + """Test whether attributes are initialized""" small_delay = 0 first_backend = self.backends[list(self.backends.keys())[0]] - manager = TasksManager(self.backend_tasks, first_backend, self.stopper, self.config, timer=small_delay) + manager = TasksManager(self.backend_tasks, first_backend, self.stopper, self.config, + self.sortinghat_client, timer=small_delay) self.assertEqual(manager.config, self.config) self.assertEqual(manager.stopper, self.stopper) @@ -235,12 +246,13 @@ def test_add_task(self): small_delay = 0 first_backend = list(self.backends.keys())[0] - manager = TasksManager(self.backend_tasks, first_backend, self.stopper, self.config, timer=small_delay) + manager = TasksManager(self.backend_tasks, first_backend, self.stopper, self.config, + self.sortinghat_client, timer=small_delay) self.assertEqual(manager.tasks, []) for tc in manager.tasks_cls: - task = tc(manager.config) + task = tc(manager.config, manager.client) task.set_backend_section(manager.backend_section) manager.tasks.append(task) @@ -250,7 +262,8 @@ def test_run_on_error(self): """Test whether an exception is thrown if a task fails""" small_delay = 0 - manager = TasksManager(self.backend_tasks, "fake-section", self.stopper, self.config, timer=small_delay) + manager = TasksManager(self.backend_tasks, "fake-section", self.stopper, + self.config, self.sortinghat_client, timer=small_delay) with self.assertRaises(Exception): manager.run()