diff --git a/integrations/gitlab/.port/spec.yaml b/integrations/gitlab/.port/spec.yaml index 67c7d6ed0b..69768ed941 100644 --- a/integrations/gitlab/.port/spec.yaml +++ b/integrations/gitlab/.port/spec.yaml @@ -13,7 +13,7 @@ configurations: - name: tokenMapping required: true type: object - description: "Mapping of Gitlab tokens to Port Ocean tokens. example: {\"THE_GROUP_TOKEN\":[\"getport-labs/**\", \"GROUP/PROJECT PATTERN TO RUN FOR\"]}" + description: "Mapping of Gitlab tokens to the groups scopes to ingest data from into port. Example: {\"THE_GROUP_TOKEN\":[\"getport-labs/**\", \"GROUP/PROJECT PATTERN TO RUN FOR\"]}" sensitive: true - name: appHost required: false @@ -29,3 +29,8 @@ configurations: type: boolean description: If set to true, will use system hook instead of project hooks. default: false + - name: tokenGroupHooksOverrideMapping + required: false + type: object + description: "Mapping of Gitlab tokens to groups in which to create webhooks, if not set, it will create webhooks only on root groups. Example: {\"THE_GROUP_ADMIN_TOKEN\":[\"GROUP1_FULL_PATH\", \"GROUP2_FULL_PATH\"]}" + sensitive: true diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index 4fbbd3e899..f5578f1d6a 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.58 (2024-03-20) +=================== + +### Features + +- Added support for webhooks creation by specified groups through the config (PORT-7140) + + 0.1.57 (2024-03-20) =================== diff --git a/integrations/gitlab/gitlab_integration/bootstrap.py b/integrations/gitlab/gitlab_integration/bootstrap.py deleted file mode 100644 index c5557d57c9..0000000000 --- a/integrations/gitlab/gitlab_integration/bootstrap.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Type, List - -from gitlab import Gitlab - -from gitlab_integration.events.event_handler import EventHandler, SystemEventHandler -from gitlab_integration.events.hooks.base import HookHandler -from gitlab_integration.events.hooks.issues import Issues -from gitlab_integration.events.hooks.jobs import Job -from gitlab_integration.events.hooks.merge_request import MergeRequest -from gitlab_integration.events.hooks.pipelines import Pipelines -from gitlab_integration.events.hooks.push import PushHook -from gitlab_integration.events.hooks.group import GroupHook -from gitlab_integration.gitlab_service import GitlabService - -event_handler = EventHandler() -system_event_handler = SystemEventHandler() - - -def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> None: - handlers = [ - PushHook(gitlab_service), - MergeRequest(gitlab_service), - Job(gitlab_service), - Issues(gitlab_service), - Pipelines(gitlab_service), - GroupHook(gitlab_service), - ] - for handler in handlers: - event_ids = [f"{event_name}:{webhook_id}" for event_name in handler.events] - event_handler.on(event_ids, handler.on_hook) - - -def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None: - handlers: List[Type[HookHandler]] = [ - PushHook, - MergeRequest, - Job, - Issues, - Pipelines, - GroupHook, - ] - for handler in handlers: - system_event_handler.on(handler) - - for gitlab_service in gitlab_clients: - system_event_handler.add_client(gitlab_service) - - -def setup_application( - token_mapping: dict[str, list[str]], - gitlab_host: str, - app_host: str, - use_system_hook: bool, -) -> None: - clients = [] - for token, group_mapping in token_mapping.items(): - gitlab_client = Gitlab(gitlab_host, token) - gitlab_service = GitlabService(gitlab_client, app_host, group_mapping) - clients.append(gitlab_service) - if use_system_hook: - gitlab_service.create_system_hook() - else: - webhook_ids = gitlab_service.create_webhooks() - for webhook_id in webhook_ids: - setup_listeners(gitlab_service, webhook_id) - if use_system_hook: - setup_system_listeners(clients) diff --git a/integrations/gitlab/gitlab_integration/events/setup.py b/integrations/gitlab/gitlab_integration/events/setup.py new file mode 100644 index 0000000000..78b667add5 --- /dev/null +++ b/integrations/gitlab/gitlab_integration/events/setup.py @@ -0,0 +1,186 @@ +from typing import Type, List + +from gitlab import Gitlab + +from gitlab_integration.events.event_handler import EventHandler, SystemEventHandler +from gitlab_integration.events.hooks.base import HookHandler +from gitlab_integration.events.hooks.issues import Issues +from gitlab_integration.events.hooks.jobs import Job +from gitlab_integration.events.hooks.merge_request import MergeRequest +from gitlab_integration.events.hooks.pipelines import Pipelines +from gitlab_integration.events.hooks.push import PushHook +from gitlab_integration.events.hooks.group import GroupHook +from gitlab_integration.gitlab_service import GitlabService +from port_ocean.exceptions.core import OceanAbortException + + +class GitlabTokenNotFoundException(OceanAbortException): + pass + + +class GitlabTooManyTokensException(OceanAbortException): + def __init__(self): + super().__init__( + "There are too many tokens in tokenMapping. When useSystemHook = true," + " there should be only one token configured" + ) + + +class GitlabEventListenerConflict(OceanAbortException): + pass + + +event_handler = EventHandler() +system_event_handler = SystemEventHandler() + + +def validate_token_mapping(token_mapping: dict[str, list[str]]) -> None: + if len(token_mapping.keys()) == 0: + raise GitlabTokenNotFoundException( + "There must be at least one token in tokenMapping" + ) + + +def validate_use_system_hook(token_mapping: dict[str, list[str]]) -> None: + if len(token_mapping.keys()) > 1: + raise GitlabTooManyTokensException() + + +def validate_hooks_tokens_are_in_token_mapping( + token_mapping: dict[str, list[str]], + token_group_override_hooks_mapping: dict[str, list[str]], +) -> None: + for token in token_group_override_hooks_mapping: + if token not in token_mapping: + raise GitlabTokenNotFoundException( + "Tokens from tokenGroupHooksOverrideMapping should also be in tokenMapping" + ) + + +def isHeirarchal(group_path: str, second_group_path: str): + return ( + second_group_path.startswith(group_path) + and second_group_path[len(group_path)] == "/" + ) + + +def validate_unique_groups_paths(groups_paths: list[str]): + for group_path in groups_paths: + if groups_paths.count(group_path) > 1: + raise GitlabEventListenerConflict( + f"Cannot listen to the same group multiple times. group: {group_path}" + ) + for second_group_path in groups_paths: + if second_group_path != group_path and isHeirarchal( + group_path, second_group_path + ): + raise GitlabEventListenerConflict( + "Cannot listen to multiple groups with hierarchy to one another." + f" Group: {second_group_path} is inside group: {group_path}" + ) + + +def validate_hooks_override_config( + token_mapping: dict[str, list[str]], + token_group_override_hooks_mapping: dict[str, list[str]], +) -> None: + if not token_group_override_hooks_mapping: + return + + validate_hooks_tokens_are_in_token_mapping( + token_mapping, token_group_override_hooks_mapping + ) + groups_paths: list[str] = sum(token_group_override_hooks_mapping.values(), []) + validate_unique_groups_paths(groups_paths) + + +def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> None: + handlers = [ + PushHook(gitlab_service), + MergeRequest(gitlab_service), + Job(gitlab_service), + Issues(gitlab_service), + Pipelines(gitlab_service), + GroupHook(gitlab_service), + ] + for handler in handlers: + event_ids = [f"{event_name}:{webhook_id}" for event_name in handler.events] + event_handler.on(event_ids, handler.on_hook) + + +def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None: + handlers: List[Type[HookHandler]] = [ + PushHook, + MergeRequest, + Job, + Issues, + Pipelines, + GroupHook, + ] + for handler in handlers: + system_event_handler.on(handler) + + for gitlab_service in gitlab_clients: + system_event_handler.add_client(gitlab_service) + + +def create_webhooks_by_client( + gitlab_host: str, + app_host: str, + token: str, + groups_hooks_override_paths: list[str] | None, + group_mapping: list[str], +) -> tuple[GitlabService, list[int | str]]: + gitlab_client = Gitlab(gitlab_host, token) + gitlab_service = GitlabService(gitlab_client, app_host, group_mapping) + + groups_for_webhooks = gitlab_service.get_filtered_groups_for_webhooks( + groups_hooks_override_paths + ) + webhook_ids = gitlab_service.create_webhooks(groups_for_webhooks) + + return gitlab_service, webhook_ids + + +def setup_application( + token_mapping: dict[str, list[str]], + gitlab_host: str, + app_host: str, + use_system_hook: bool, + token_group_override_hooks_mapping: dict[str, list[str]], +) -> None: + validate_token_mapping(token_mapping) + + if use_system_hook: + validate_use_system_hook(token_mapping) + token, group_mapping = list(token_mapping.items())[0] + gitlab_client = Gitlab(gitlab_host, token) + gitlab_service = GitlabService(gitlab_client, app_host, group_mapping) + setup_system_listeners([gitlab_service]) + + else: + validate_hooks_override_config( + token_mapping, token_group_override_hooks_mapping + ) + + client_to_webhooks: list[tuple[GitlabService, list[int | str]]] = [] + for token, group_mapping in token_mapping.items(): + groups_override_paths_list: list[str] | None = ( + token_group_override_hooks_mapping.get(token, []) + if token_group_override_hooks_mapping + else None + ) + + client_to_webhooks.append( + create_webhooks_by_client( + gitlab_host, + app_host, + token, + groups_override_paths_list, + group_mapping, + ) + ) + + for client, webhook_ids in client_to_webhooks: + for webhook_id in webhook_ids: + setup_listeners(client, webhook_id) diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index 70f4d9ab65..5c665e34dd 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -170,23 +170,53 @@ def get_root_groups(self) -> List[Group]: List[Group], [group for group in groups if group.parent_id is None] ) - def create_webhooks(self) -> list[int | str]: - root_partial_groups = self.get_root_groups() - logger.info("Getting all the root groups to create webhooks for") - # Filter out root groups that are not in the group mapping and creating webhooks for the rest - filtered_partial_groups = [ - group - for group in root_partial_groups - if any( - does_pattern_apply(mapping.split("/")[0], group.attributes["full_path"]) - for mapping in self.group_mapping - ) - ] + def filter_groups_by_paths(self, groups_full_paths: list[str]) -> List[Group]: + groups = self.gitlab_client.groups.list(get_all=True) + return typing.cast( + List[Group], + [ + group + for group in groups + if group.attributes["full_path"] in groups_full_paths + ], + ) + + def get_filtered_groups_for_webhooks( + self, + groups_hooks_override_list: list[str] | None, + ) -> List[Group]: + groups_for_webhooks = [] + if groups_hooks_override_list is not None: + if groups_hooks_override_list: + logger.info( + "Getting all the specified groups in the mapping for a token to create their webhooks" + ) + groups_for_webhooks = self.filter_groups_by_paths( + groups_hooks_override_list + ) + else: + logger.info("Getting all the root groups to create their webhooks") + root_groups = self.get_root_groups() + groups_for_webhooks = [ + group + for group in root_groups + if any( + does_pattern_apply( + mapping.split("/")[0], group.attributes["full_path"] + ) + for mapping in self.group_mapping + ) + ] + + return groups_for_webhooks + + def create_webhooks(self, groups_for_webhooks) -> list[int | str]: + # Filter out groups that are not in the group mapping and creating webhooks for the rest logger.info( - f"Creating webhooks for the root groups. Groups: {[group.attributes['full_path'] for group in filtered_partial_groups]}" + f"Creating webhooks for the groups: {[group.attributes['full_path'] for group in groups_for_webhooks]}" ) webhook_ids = [] - for partial_group in filtered_partial_groups: + for partial_group in groups_for_webhooks: group_id = partial_group.get_id() if group_id is None: logger.info( diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index d3c567d7e9..25de6a1051 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -8,12 +8,13 @@ from starlette.requests import Request from port_ocean.context.event import event -from gitlab_integration.bootstrap import event_handler, system_event_handler -from gitlab_integration.bootstrap import setup_application +from gitlab_integration.events.setup import event_handler, system_event_handler +from gitlab_integration.events.setup import setup_application from gitlab_integration.git_integration import GitlabResourceConfig from gitlab_integration.utils import ObjectKind, get_cached_all_services from port_ocean.context.ocean import ocean from port_ocean.core.ocean_types import ASYNC_GENERATOR_RESYNC_TYPE +from port_ocean.log.sensetive import sensitive_log_filter NO_WEBHOOK_WARNING = "Without setting up the webhook, the integration will not export live changes from the gitlab" PROJECT_RESYNC_BATCH_SIZE = 10 @@ -41,12 +42,20 @@ async def handle_system_webhook(request: Request) -> dict[str, Any]: @ocean.on_start() async def on_start() -> None: + integration_config = ocean.integration_config + token_mapping: dict = integration_config["token_mapping"] + hook_override_mapping: dict = integration_config[ + "token_group_hooks_override_mapping" + ] + sensitive_log_filter.hide_sensitive_strings( + *token_mapping.keys(), *hook_override_mapping.keys() + ) + if ocean.event_listener_type == "ONCE": logger.info("Skipping webhook creation because the event listener is ONCE") return - logic_settings = ocean.integration_config - if not logic_settings.get("app_host"): + if not integration_config.get("app_host"): logger.warning( f"No app host provided, skipping webhook creation. {NO_WEBHOOK_WARNING}" ) @@ -54,10 +63,11 @@ async def on_start() -> None: try: setup_application( - logic_settings["token_mapping"], - logic_settings["gitlab_host"], - logic_settings["app_host"], - logic_settings["use_system_hook"], + integration_config["token_mapping"], + integration_config["gitlab_host"], + integration_config["app_host"], + integration_config["use_system_hook"], + integration_config["token_group_hooks_override_mapping"], ) except Exception as e: logger.warning( diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 376f0473d8..3ebcec8373 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.57" +version = "0.1.58" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "]