[Integration][Gitlab] Add support for gitlab system hooks (#263)

integrations/gitlab/.port/spec.yaml

@@ -23,3 +23,8 @@ configurations:
     type: url
     default: https://gitlab.com
     description: The host of the Gitlab instance. If not specified, the default will be https://gitlab.com.
+  - name: useSystemHook
+    required: false
+    type: boolean
+    description: If set to true, will use system hook instead of project hooks.
+    default: false

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.1.34 (2023-12-12)
+
+### Improvements
+
+- Added support for system hooks, this capability can be enabled using the useSystemHook flag. Enabling this capability will create system hooks instead of group webhooks (PORT-5220)
+
+
 0.1.33 (2023-12-05)
 ===================
 

integrations/gitlab/gitlab_integration/bootstrap.py

@@ -1,6 +1,9 @@
+from typing import Type, List
+
 from gitlab import Gitlab
 
-from gitlab_integration.events.event_handler import EventHandler
+from gitlab_integration.events.event_handler import EventHandler, SystemEventHandler
+from gitlab_integration.events.hooks.base import HookHandler
 from gitlab_integration.events.hooks.issues import Issues
 from gitlab_integration.events.hooks.jobs import Job
 from gitlab_integration.events.hooks.merge_request import MergeRequest
@@ -9,6 +12,7 @@
 from gitlab_integration.gitlab_service import GitlabService
 
 event_handler = EventHandler()
+system_event_handler = SystemEventHandler()
 
 
 def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> None:
@@ -24,12 +28,37 @@ def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> Non
         event_handler.on(event_ids, handler.on_hook)
 
 
+def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None:
+    handlers: List[Type[HookHandler]] = [
+        PushHook,
+        MergeRequest,
+        Job,
+        Issues,
+        Pipelines,
+    ]
+    for handler in handlers:
+        system_event_handler.on(handler)
+
+    for gitlab_service in gitlab_clients:
+        system_event_handler.add_client(gitlab_service)
+
+
 def setup_application(
-    token_mapping: dict[str, list[str]], gitlab_host: str, app_host: str
+    token_mapping: dict[str, list[str]],
+    gitlab_host: str,
+    app_host: str,
+    use_system_hook: bool,
 ) -> None:
+    clients = []
     for token, group_mapping in token_mapping.items():
         gitlab_client = Gitlab(gitlab_host, token)
         gitlab_service = GitlabService(gitlab_client, app_host, group_mapping)
-        webhook_ids = gitlab_service.create_webhooks()
-        for webhook_id in webhook_ids:
-            setup_listeners(gitlab_service, webhook_id)
+        clients.append(gitlab_service)
+        if use_system_hook:
+            gitlab_service.create_system_hook()
+        else:
+            webhook_ids = gitlab_service.create_webhooks()
+            for webhook_id in webhook_ids:
+                setup_listeners(gitlab_service, webhook_id)
+    if use_system_hook:
+        setup_system_listeners(clients)

integrations/gitlab/gitlab_integration/events/event_handler.py

@@ -1,9 +1,11 @@
 import asyncio
 from collections import defaultdict
-from typing import Awaitable, Callable, Any
+from typing import Awaitable, Callable, Any, Type
 
+from gitlab_integration.events.hooks.base import HookHandler
+from gitlab_integration.gitlab_service import GitlabService
 
-Observer = Callable[[str, str, dict[str, Any]], Awaitable[Any]]
+Observer = Callable[[str, dict[str, Any]], Awaitable[Any]]
 
 
 class EventHandler:
@@ -14,9 +16,32 @@ def on(self, events: list[str], observer: Observer) -> None:
         for event in events:
             self._observers[event].append(observer)
 
-    async def notify(
-        self, event: str, group_id: str, body: dict[str, Any]
-    ) -> Awaitable[Any]:
+    async def notify(self, event: str, body: dict[str, Any]) -> Awaitable[Any]:
         return asyncio.gather(
-            *(observer(event, group_id, body) for observer in self._observers[event])
+            *(observer(event, body) for observer in self._observers.get(event, []))
+        )
+
+
+class SystemEventHandler:
+    def __init__(self) -> None:
+        self._hook_handlers: dict[str, list[Type[HookHandler]]] = defaultdict(list)
+        self._clients: list[GitlabService] = []
+
+    def on(self, hook_handler: Type[HookHandler]) -> None:
+        for system_event in hook_handler.system_events:
+            self._hook_handlers[system_event].append(hook_handler)
+
+    def add_client(self, client: GitlabService) -> None:
+        self._clients.append(client)
+
+    async def notify(self, event: str, body: dict[str, Any]) -> Awaitable[Any]:
+        # best effort to notify using all clients, as we don't know which one of the clients have the permission to
+        # access the project
+        return asyncio.gather(
+            *(
+                hook_handler(client).on_hook(event, body)
+                for client in self._clients
+                for hook_handler in self._hook_handlers.get(event, [])
+            ),
+            return_exceptions=True,
         )

integrations/gitlab/gitlab_integration/events/hooks/base.py

@@ -8,24 +8,20 @@
 
 
 class HookHandler(ABC):
+    events: List[str] = []
+    system_events: List[str] = []
+
     def __init__(
         self,
         gitlab_service: GitlabService,
     ):
         self.gitlab_service = gitlab_service
 
-    @property
-    @abstractmethod
-    def events(self) -> List[str]:
-        return []
-
     @abstractmethod
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         pass
 
-    async def on_hook(self, event: str, group_id: str, body: dict[str, Any]) -> None:
+    async def on_hook(self, event: str, body: dict[str, Any]) -> None:
         logger.info(f"Handling {event}")
 
         project_id = (
@@ -37,7 +33,7 @@ async def on_hook(self, event: str, group_id: str, body: dict[str, Any]) -> None
             logger.info(
                 f"Handling hook {event} for project {project.path_with_namespace}"
             )
-            await self._on_hook(group_id, body, project)
+            await self._on_hook(body, project)
             logger.info(f"Finished handling {event}")
         else:
             logger.info(

integrations/gitlab/gitlab_integration/events/hooks/issues.py

@@ -9,9 +9,8 @@
 
 class Issues(HookHandler):
     events = ["Issue Hook"]
+    system_events = ["issue"]
 
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         issue = gitlab_project.issues.get(body["object_attributes"]["id"])
         await ocean.register_raw(ObjectKind.ISSUE, [issue.asdict()])

integrations/gitlab/gitlab_integration/events/hooks/jobs.py

@@ -9,9 +9,8 @@
 
 class Job(HookHandler):
     events = ["Job Hook"]
+    system_events = ["job"]
 
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         job = gitlab_project.jobs.get(body["build_id"])
         await ocean.register_raw(ObjectKind.JOB, [job.asdict()])

integrations/gitlab/gitlab_integration/events/hooks/merge_request.py

@@ -9,10 +9,9 @@
 
 class MergeRequest(HookHandler):
     events = ["Merge Request Hook"]
+    system_events = ["merge_request"]
 
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         merge_requests = gitlab_project.mergerequests.get(
             body["object_attributes"]["iid"]
         )

integrations/gitlab/gitlab_integration/events/hooks/pipelines.py

@@ -9,9 +9,8 @@
 
 class Pipelines(HookHandler):
     events = ["Pipeline Hook"]
+    system_events = ["pipeline"]
 
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         pipeline = gitlab_project.pipelines.get(body["object_attributes"]["id"])
         await ocean.register_raw(ObjectKind.PIPELINE, [pipeline.asdict()])

integrations/gitlab/gitlab_integration/events/hooks/push.py

@@ -15,10 +15,9 @@
 
 class PushHook(HookHandler):
     events = ["Push Hook"]
+    system_events = ["push"]
 
-    async def _on_hook(
-        self, group_id: str, body: dict[str, Any], gitlab_project: Project
-    ) -> None:
+    async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
         before, after, ref = body.get("before"), body.get("after"), body.get("ref")
 
         if before is None or after is None or ref is None:

integrations/gitlab/gitlab_integration/gitlab_service.py

@@ -186,6 +186,33 @@ def create_webhooks(self) -> list[int | str]:
 
         return webhook_ids
 
+    def create_system_hook(self) -> None:
+        logger.debug("Checking if system hook already exists")
+        try:
+            for hook in self.gitlab_client.hooks.list(iterator=True):
+                if hook.url == f"{self.app_host}/integration/system/hook":
+                    logger.debug("System hook already exists, no need to create")
+                    return
+        except Exception:
+            logger.error(
+                "Failed to check if system hook exists, skipping trying to create, to avoid duplicates"
+            )
+            return
+
+        logger.debug("Creating system hook")
+        try:
+            resp = self.gitlab_client.hooks.create(
+                {
+                    "url": f"{self.app_host}/integration/system/hook",
+                    "push_events": True,
+                    "merge_requests_events": True,
+                    "repository_update_events": False,
+                }
+            )
+            logger.debug(f"Created system hook with id {resp.get_id()}")
+        except Exception:
+            logger.error("Failed to create system hook")
+
     def get_project(self, project_id: int) -> Project | None:
         """
         Returns project if it should be processed, None otherwise

integrations/gitlab/gitlab_integration/ocean.py

@@ -7,7 +7,7 @@
 from starlette.requests import Request
 from port_ocean.context.event import event
 
-from gitlab_integration.bootstrap import event_handler
+from gitlab_integration.bootstrap import event_handler, system_event_handler
 from gitlab_integration.bootstrap import setup_application
 from gitlab_integration.git_integration import GitlabResourceConfig
 from gitlab_integration.utils import ObjectKind, get_cached_all_services
@@ -20,9 +20,21 @@
 @ocean.router.post("/hook/{group_id}")
 async def handle_webhook(group_id: str, request: Request) -> dict[str, Any]:
     event_id = f'{request.headers.get("X-Gitlab-Event")}:{group_id}'
+    with logger.contextualize(event_id=event_id):
+        body = await request.json()
+        await event_handler.notify(event_id, body)
+        return {"ok": True}
+
+
+@ocean.router.post("/system/hook")
+async def handle_system_webhook(request: Request) -> dict[str, Any]:
     body = await request.json()
-    await event_handler.notify(event_id, group_id, body)
-    return {"ok": True}
+    # some system hooks have event_type instead of event_name in the body, such as merge_request events
+    event_name = body.get("event_name") or body.get("event_type")
+    with logger.contextualize(event_name=event_name):
+        logger.debug("Handling system hook")
+        await system_event_handler.notify(event_name, body)
+        return {"ok": True}
 
 
 @ocean.on_start()
@@ -43,6 +55,7 @@ async def on_start() -> None:
             logic_settings["token_mapping"],
             logic_settings["gitlab_host"],
             logic_settings["app_host"],
+            logic_settings["use_system_hook"],
         )
     except Exception as e:
         logger.warning(

integrations/gitlab/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gitlab"
-version = "0.1.33"
+version = "0.1.34"
 description = "Gitlab integration for Port using Port-Ocean Framework"
 authors = ["Yair Siman-Tov <[email protected]>"]