[Integration][Gitlab] - Allow to create webhook for specific groups (#…

…425)

# Description

What - Support webhooks creation on specified groups
Why - To support more granular way of setting webhooks in cases where we
only want to listen to specific groups which isn't root groups
How - accept a param containing a list of all the wanted gourps
full_paths

## Type of change

Please leave one option from the following and delete the rest:

- [x] New feature (non-breaking change which adds functionality)

---------

Co-authored-by: Tom Tankilevitch <[email protected]>
Co-authored-by: Yair Siman Tov <[email protected]>

integrations/gitlab/.port/spec.yaml

@@ -13,7 +13,7 @@ configurations:
   - name: tokenMapping
     required: true
     type: object
-    description: "Mapping of Gitlab tokens to Port Ocean tokens. example: {\"THE_GROUP_TOKEN\":[\"getport-labs/**\", \"GROUP/PROJECT PATTERN TO RUN FOR\"]}"
+    description: "Mapping of Gitlab tokens to the groups scopes to ingest data from into port. Example: {\"THE_GROUP_TOKEN\":[\"getport-labs/**\", \"GROUP/PROJECT PATTERN TO RUN FOR\"]}"
     sensitive: true
   - name: appHost
     required: false
@@ -29,3 +29,8 @@ configurations:
     type: boolean
     description: If set to true, will use system hook instead of project hooks.
     default: false
+  - name: tokenGroupHooksOverrideMapping
+    required: false
+    type: object
+    description: "Mapping of Gitlab tokens to groups in which to create webhooks, if not set, it will create webhooks only on root groups. Example: {\"THE_GROUP_ADMIN_TOKEN\":[\"GROUP1_FULL_PATH\", \"GROUP2_FULL_PATH\"]}"
+    sensitive: true

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.1.58 (2024-03-20)
+===================
+
+### Features
+
+- Added support for webhooks creation by specified groups through the config (PORT-7140)
+
+
 0.1.57 (2024-03-20)
 ===================
 

integrations/gitlab/gitlab_integration/events/setup.py

@@ -0,0 +1,186 @@
+from typing import Type, List
+
+from gitlab import Gitlab
+
+from gitlab_integration.events.event_handler import EventHandler, SystemEventHandler
+from gitlab_integration.events.hooks.base import HookHandler
+from gitlab_integration.events.hooks.issues import Issues
+from gitlab_integration.events.hooks.jobs import Job
+from gitlab_integration.events.hooks.merge_request import MergeRequest
+from gitlab_integration.events.hooks.pipelines import Pipelines
+from gitlab_integration.events.hooks.push import PushHook
+from gitlab_integration.events.hooks.group import GroupHook
+from gitlab_integration.gitlab_service import GitlabService
+from port_ocean.exceptions.core import OceanAbortException
+
+
+class GitlabTokenNotFoundException(OceanAbortException):
+    pass
+
+
+class GitlabTooManyTokensException(OceanAbortException):
+    def __init__(self):
+        super().__init__(
+            "There are too many tokens in tokenMapping. When useSystemHook = true,"
+            " there should be only one token configured"
+        )
+
+
+class GitlabEventListenerConflict(OceanAbortException):
+    pass
+
+
+event_handler = EventHandler()
+system_event_handler = SystemEventHandler()
+
+
+def validate_token_mapping(token_mapping: dict[str, list[str]]) -> None:
+    if len(token_mapping.keys()) == 0:
+        raise GitlabTokenNotFoundException(
+            "There must be at least one token in tokenMapping"
+        )
+
+
+def validate_use_system_hook(token_mapping: dict[str, list[str]]) -> None:
+    if len(token_mapping.keys()) > 1:
+        raise GitlabTooManyTokensException()
+
+
+def validate_hooks_tokens_are_in_token_mapping(
+    token_mapping: dict[str, list[str]],
+    token_group_override_hooks_mapping: dict[str, list[str]],
+) -> None:
+    for token in token_group_override_hooks_mapping:
+        if token not in token_mapping:
+            raise GitlabTokenNotFoundException(
+                "Tokens from tokenGroupHooksOverrideMapping should also be in tokenMapping"
+            )
+
+
+def isHeirarchal(group_path: str, second_group_path: str):
+    return (
+        second_group_path.startswith(group_path)
+        and second_group_path[len(group_path)] == "/"
+    )
+
+
+def validate_unique_groups_paths(groups_paths: list[str]):
+    for group_path in groups_paths:
+        if groups_paths.count(group_path) > 1:
+            raise GitlabEventListenerConflict(
+                f"Cannot listen to the same group multiple times. group: {group_path}"
+            )
+        for second_group_path in groups_paths:
+            if second_group_path != group_path and isHeirarchal(
+                group_path, second_group_path
+            ):
+                raise GitlabEventListenerConflict(
+                    "Cannot listen to multiple groups with hierarchy to one another."
+                    f" Group: {second_group_path} is inside group: {group_path}"
+                )
+
+
+def validate_hooks_override_config(
+    token_mapping: dict[str, list[str]],
+    token_group_override_hooks_mapping: dict[str, list[str]],
+) -> None:
+    if not token_group_override_hooks_mapping:
+        return
+
+    validate_hooks_tokens_are_in_token_mapping(
+        token_mapping, token_group_override_hooks_mapping
+    )
+    groups_paths: list[str] = sum(token_group_override_hooks_mapping.values(), [])
+    validate_unique_groups_paths(groups_paths)
+
+
+def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> None:
+    handlers = [
+        PushHook(gitlab_service),
+        MergeRequest(gitlab_service),
+        Job(gitlab_service),
+        Issues(gitlab_service),
+        Pipelines(gitlab_service),
+        GroupHook(gitlab_service),
+    ]
+    for handler in handlers:
+        event_ids = [f"{event_name}:{webhook_id}" for event_name in handler.events]
+        event_handler.on(event_ids, handler.on_hook)
+
+
+def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None:
+    handlers: List[Type[HookHandler]] = [
+        PushHook,
+        MergeRequest,
+        Job,
+        Issues,
+        Pipelines,
+        GroupHook,
+    ]
+    for handler in handlers:
+        system_event_handler.on(handler)
+
+    for gitlab_service in gitlab_clients:
+        system_event_handler.add_client(gitlab_service)
+
+
+def create_webhooks_by_client(
+    gitlab_host: str,
+    app_host: str,
+    token: str,
+    groups_hooks_override_paths: list[str] | None,
+    group_mapping: list[str],
+) -> tuple[GitlabService, list[int | str]]:
+    gitlab_client = Gitlab(gitlab_host, token)
+    gitlab_service = GitlabService(gitlab_client, app_host, group_mapping)
+
+    groups_for_webhooks = gitlab_service.get_filtered_groups_for_webhooks(
+        groups_hooks_override_paths
+    )
+    webhook_ids = gitlab_service.create_webhooks(groups_for_webhooks)
+
+    return gitlab_service, webhook_ids
+
+
+def setup_application(
+    token_mapping: dict[str, list[str]],
+    gitlab_host: str,
+    app_host: str,
+    use_system_hook: bool,
+    token_group_override_hooks_mapping: dict[str, list[str]],
+) -> None:
+    validate_token_mapping(token_mapping)
+
+    if use_system_hook:
+        validate_use_system_hook(token_mapping)
+        token, group_mapping = list(token_mapping.items())[0]
+        gitlab_client = Gitlab(gitlab_host, token)
+        gitlab_service = GitlabService(gitlab_client, app_host, group_mapping)
+        setup_system_listeners([gitlab_service])
+
+    else:
+        validate_hooks_override_config(
+            token_mapping, token_group_override_hooks_mapping
+        )
+
+        client_to_webhooks: list[tuple[GitlabService, list[int | str]]] = []
+        for token, group_mapping in token_mapping.items():
+            groups_override_paths_list: list[str] | None = (
+                token_group_override_hooks_mapping.get(token, [])
+                if token_group_override_hooks_mapping
+                else None
+            )
+
+            client_to_webhooks.append(
+                create_webhooks_by_client(
+                    gitlab_host,
+                    app_host,
+                    token,
+                    groups_override_paths_list,
+                    group_mapping,
+                )
+            )
+
+        for client, webhook_ids in client_to_webhooks:
+            for webhook_id in webhook_ids:
+                setup_listeners(client, webhook_id)

integrations/gitlab/gitlab_integration/gitlab_service.py

@@ -170,23 +170,53 @@ def get_root_groups(self) -> List[Group]:
             List[Group], [group for group in groups if group.parent_id is None]
         )
 
-    def create_webhooks(self) -> list[int | str]:
-        root_partial_groups = self.get_root_groups()
-        logger.info("Getting all the root groups to create webhooks for")
-        # Filter out root groups that are not in the group mapping and creating webhooks for the rest
-        filtered_partial_groups = [
-            group
-            for group in root_partial_groups
-            if any(
-                does_pattern_apply(mapping.split("/")[0], group.attributes["full_path"])
-                for mapping in self.group_mapping
-            )
-        ]
+    def filter_groups_by_paths(self, groups_full_paths: list[str]) -> List[Group]:
+        groups = self.gitlab_client.groups.list(get_all=True)
+        return typing.cast(
+            List[Group],
+            [
+                group
+                for group in groups
+                if group.attributes["full_path"] in groups_full_paths
+            ],
+        )
+
+    def get_filtered_groups_for_webhooks(
+        self,
+        groups_hooks_override_list: list[str] | None,
+    ) -> List[Group]:
+        groups_for_webhooks = []
+        if groups_hooks_override_list is not None:
+            if groups_hooks_override_list:
+                logger.info(
+                    "Getting all the specified groups in the mapping for a token to create their webhooks"
+                )
+                groups_for_webhooks = self.filter_groups_by_paths(
+                    groups_hooks_override_list
+                )
+        else:
+            logger.info("Getting all the root groups to create their webhooks")
+            root_groups = self.get_root_groups()
+            groups_for_webhooks = [
+                group
+                for group in root_groups
+                if any(
+                    does_pattern_apply(
+                        mapping.split("/")[0], group.attributes["full_path"]
+                    )
+                    for mapping in self.group_mapping
+                )
+            ]
+
+        return groups_for_webhooks
+
+    def create_webhooks(self, groups_for_webhooks) -> list[int | str]:
+        # Filter out groups that are not in the group mapping and creating webhooks for the rest
         logger.info(
-            f"Creating webhooks for the root groups. Groups: {[group.attributes['full_path'] for group in filtered_partial_groups]}"
+            f"Creating webhooks for the groups: {[group.attributes['full_path'] for group in groups_for_webhooks]}"
         )
         webhook_ids = []
-        for partial_group in filtered_partial_groups:
+        for partial_group in groups_for_webhooks:
             group_id = partial_group.get_id()
             if group_id is None:
                 logger.info(