From fab5684bc58bde52f109cceee25cd4f2f8f1a7fc Mon Sep 17 00:00:00 2001 From: Tom Tankilevitch <59158507+Tankilevitch@users.noreply.github.com> Date: Wed, 13 Dec 2023 15:07:29 +0200 Subject: [PATCH] [Integration][Gitlab] Add support for gitlab system hooks (#263) --- integrations/gitlab/.port/spec.yaml | 5 +++ integrations/gitlab/CHANGELOG.md | 7 ++++ .../gitlab/gitlab_integration/bootstrap.py | 39 ++++++++++++++++--- .../events/event_handler.py | 37 +++++++++++++++--- .../gitlab_integration/events/hooks/base.py | 16 +++----- .../gitlab_integration/events/hooks/issues.py | 5 +-- .../gitlab_integration/events/hooks/jobs.py | 5 +-- .../events/hooks/merge_request.py | 5 +-- .../events/hooks/pipelines.py | 5 +-- .../gitlab_integration/events/hooks/push.py | 5 +-- .../gitlab_integration/gitlab_service.py | 27 +++++++++++++ .../gitlab/gitlab_integration/ocean.py | 19 +++++++-- integrations/gitlab/pyproject.toml | 2 +- 13 files changed, 137 insertions(+), 40 deletions(-) diff --git a/integrations/gitlab/.port/spec.yaml b/integrations/gitlab/.port/spec.yaml index 9ac2edfc02..280e98e83f 100644 --- a/integrations/gitlab/.port/spec.yaml +++ b/integrations/gitlab/.port/spec.yaml @@ -23,3 +23,8 @@ configurations: type: url default: https://gitlab.com description: The host of the Gitlab instance. If not specified, the default will be https://gitlab.com. + - name: useSystemHook + required: false + type: boolean + description: If set to true, will use system hook instead of project hooks. + default: false diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index a758e8e6b9..efab4e48b8 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,13 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.34 (2023-12-12) + +### Improvements + +- Added support for system hooks, this capability can be enabled using the useSystemHook flag. Enabling this capability will create system hooks instead of group webhooks (PORT-5220) + + 0.1.33 (2023-12-05) =================== diff --git a/integrations/gitlab/gitlab_integration/bootstrap.py b/integrations/gitlab/gitlab_integration/bootstrap.py index bd4c3e811a..dc95321ee2 100644 --- a/integrations/gitlab/gitlab_integration/bootstrap.py +++ b/integrations/gitlab/gitlab_integration/bootstrap.py @@ -1,6 +1,9 @@ +from typing import Type, List + from gitlab import Gitlab -from gitlab_integration.events.event_handler import EventHandler +from gitlab_integration.events.event_handler import EventHandler, SystemEventHandler +from gitlab_integration.events.hooks.base import HookHandler from gitlab_integration.events.hooks.issues import Issues from gitlab_integration.events.hooks.jobs import Job from gitlab_integration.events.hooks.merge_request import MergeRequest @@ -9,6 +12,7 @@ from gitlab_integration.gitlab_service import GitlabService event_handler = EventHandler() +system_event_handler = SystemEventHandler() def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> None: @@ -24,12 +28,37 @@ def setup_listeners(gitlab_service: GitlabService, webhook_id: str | int) -> Non event_handler.on(event_ids, handler.on_hook) +def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None: + handlers: List[Type[HookHandler]] = [ + PushHook, + MergeRequest, + Job, + Issues, + Pipelines, + ] + for handler in handlers: + system_event_handler.on(handler) + + for gitlab_service in gitlab_clients: + system_event_handler.add_client(gitlab_service) + + def setup_application( - token_mapping: dict[str, list[str]], gitlab_host: str, app_host: str + token_mapping: dict[str, list[str]], + gitlab_host: str, + app_host: str, + use_system_hook: bool, ) -> None: + clients = [] for token, group_mapping in token_mapping.items(): gitlab_client = Gitlab(gitlab_host, token) gitlab_service = GitlabService(gitlab_client, app_host, group_mapping) - webhook_ids = gitlab_service.create_webhooks() - for webhook_id in webhook_ids: - setup_listeners(gitlab_service, webhook_id) + clients.append(gitlab_service) + if use_system_hook: + gitlab_service.create_system_hook() + else: + webhook_ids = gitlab_service.create_webhooks() + for webhook_id in webhook_ids: + setup_listeners(gitlab_service, webhook_id) + if use_system_hook: + setup_system_listeners(clients) diff --git a/integrations/gitlab/gitlab_integration/events/event_handler.py b/integrations/gitlab/gitlab_integration/events/event_handler.py index 5a9eded346..40d10c6a55 100644 --- a/integrations/gitlab/gitlab_integration/events/event_handler.py +++ b/integrations/gitlab/gitlab_integration/events/event_handler.py @@ -1,9 +1,11 @@ import asyncio from collections import defaultdict -from typing import Awaitable, Callable, Any +from typing import Awaitable, Callable, Any, Type +from gitlab_integration.events.hooks.base import HookHandler +from gitlab_integration.gitlab_service import GitlabService -Observer = Callable[[str, str, dict[str, Any]], Awaitable[Any]] +Observer = Callable[[str, dict[str, Any]], Awaitable[Any]] class EventHandler: @@ -14,9 +16,32 @@ def on(self, events: list[str], observer: Observer) -> None: for event in events: self._observers[event].append(observer) - async def notify( - self, event: str, group_id: str, body: dict[str, Any] - ) -> Awaitable[Any]: + async def notify(self, event: str, body: dict[str, Any]) -> Awaitable[Any]: return asyncio.gather( - *(observer(event, group_id, body) for observer in self._observers[event]) + *(observer(event, body) for observer in self._observers.get(event, [])) + ) + + +class SystemEventHandler: + def __init__(self) -> None: + self._hook_handlers: dict[str, list[Type[HookHandler]]] = defaultdict(list) + self._clients: list[GitlabService] = [] + + def on(self, hook_handler: Type[HookHandler]) -> None: + for system_event in hook_handler.system_events: + self._hook_handlers[system_event].append(hook_handler) + + def add_client(self, client: GitlabService) -> None: + self._clients.append(client) + + async def notify(self, event: str, body: dict[str, Any]) -> Awaitable[Any]: + # best effort to notify using all clients, as we don't know which one of the clients have the permission to + # access the project + return asyncio.gather( + *( + hook_handler(client).on_hook(event, body) + for client in self._clients + for hook_handler in self._hook_handlers.get(event, []) + ), + return_exceptions=True, ) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/base.py b/integrations/gitlab/gitlab_integration/events/hooks/base.py index f4a3f5fbf6..4d88e92a35 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/base.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/base.py @@ -8,24 +8,20 @@ class HookHandler(ABC): + events: List[str] = [] + system_events: List[str] = [] + def __init__( self, gitlab_service: GitlabService, ): self.gitlab_service = gitlab_service - @property - @abstractmethod - def events(self) -> List[str]: - return [] - @abstractmethod - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: pass - async def on_hook(self, event: str, group_id: str, body: dict[str, Any]) -> None: + async def on_hook(self, event: str, body: dict[str, Any]) -> None: logger.info(f"Handling {event}") project_id = ( @@ -37,7 +33,7 @@ async def on_hook(self, event: str, group_id: str, body: dict[str, Any]) -> None logger.info( f"Handling hook {event} for project {project.path_with_namespace}" ) - await self._on_hook(group_id, body, project) + await self._on_hook(body, project) logger.info(f"Finished handling {event}") else: logger.info( diff --git a/integrations/gitlab/gitlab_integration/events/hooks/issues.py b/integrations/gitlab/gitlab_integration/events/hooks/issues.py index 82121edffa..c23c0deb8a 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/issues.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/issues.py @@ -9,9 +9,8 @@ class Issues(HookHandler): events = ["Issue Hook"] + system_events = ["issue"] - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: issue = gitlab_project.issues.get(body["object_attributes"]["id"]) await ocean.register_raw(ObjectKind.ISSUE, [issue.asdict()]) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/jobs.py b/integrations/gitlab/gitlab_integration/events/hooks/jobs.py index 105570222d..e55eee8af7 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/jobs.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/jobs.py @@ -9,9 +9,8 @@ class Job(HookHandler): events = ["Job Hook"] + system_events = ["job"] - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: job = gitlab_project.jobs.get(body["build_id"]) await ocean.register_raw(ObjectKind.JOB, [job.asdict()]) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/merge_request.py b/integrations/gitlab/gitlab_integration/events/hooks/merge_request.py index 41acc99e9f..2e48b766e6 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/merge_request.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/merge_request.py @@ -9,10 +9,9 @@ class MergeRequest(HookHandler): events = ["Merge Request Hook"] + system_events = ["merge_request"] - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: merge_requests = gitlab_project.mergerequests.get( body["object_attributes"]["iid"] ) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/pipelines.py b/integrations/gitlab/gitlab_integration/events/hooks/pipelines.py index e6b5fec863..76e2450ae9 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/pipelines.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/pipelines.py @@ -9,9 +9,8 @@ class Pipelines(HookHandler): events = ["Pipeline Hook"] + system_events = ["pipeline"] - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: pipeline = gitlab_project.pipelines.get(body["object_attributes"]["id"]) await ocean.register_raw(ObjectKind.PIPELINE, [pipeline.asdict()]) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index a9d421e355..234cc89924 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -15,10 +15,9 @@ class PushHook(HookHandler): events = ["Push Hook"] + system_events = ["push"] - async def _on_hook( - self, group_id: str, body: dict[str, Any], gitlab_project: Project - ) -> None: + async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: before, after, ref = body.get("before"), body.get("after"), body.get("ref") if before is None or after is None or ref is None: diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index 01903af285..97e018dc4d 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -186,6 +186,33 @@ def create_webhooks(self) -> list[int | str]: return webhook_ids + def create_system_hook(self) -> None: + logger.debug("Checking if system hook already exists") + try: + for hook in self.gitlab_client.hooks.list(iterator=True): + if hook.url == f"{self.app_host}/integration/system/hook": + logger.debug("System hook already exists, no need to create") + return + except Exception: + logger.error( + "Failed to check if system hook exists, skipping trying to create, to avoid duplicates" + ) + return + + logger.debug("Creating system hook") + try: + resp = self.gitlab_client.hooks.create( + { + "url": f"{self.app_host}/integration/system/hook", + "push_events": True, + "merge_requests_events": True, + "repository_update_events": False, + } + ) + logger.debug(f"Created system hook with id {resp.get_id()}") + except Exception: + logger.error("Failed to create system hook") + def get_project(self, project_id: int) -> Project | None: """ Returns project if it should be processed, None otherwise diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index 34e6492367..c89d68440d 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -7,7 +7,7 @@ from starlette.requests import Request from port_ocean.context.event import event -from gitlab_integration.bootstrap import event_handler +from gitlab_integration.bootstrap import event_handler, system_event_handler from gitlab_integration.bootstrap import setup_application from gitlab_integration.git_integration import GitlabResourceConfig from gitlab_integration.utils import ObjectKind, get_cached_all_services @@ -20,9 +20,21 @@ @ocean.router.post("/hook/{group_id}") async def handle_webhook(group_id: str, request: Request) -> dict[str, Any]: event_id = f'{request.headers.get("X-Gitlab-Event")}:{group_id}' + with logger.contextualize(event_id=event_id): + body = await request.json() + await event_handler.notify(event_id, body) + return {"ok": True} + + +@ocean.router.post("/system/hook") +async def handle_system_webhook(request: Request) -> dict[str, Any]: body = await request.json() - await event_handler.notify(event_id, group_id, body) - return {"ok": True} + # some system hooks have event_type instead of event_name in the body, such as merge_request events + event_name = body.get("event_name") or body.get("event_type") + with logger.contextualize(event_name=event_name): + logger.debug("Handling system hook") + await system_event_handler.notify(event_name, body) + return {"ok": True} @ocean.on_start() @@ -43,6 +55,7 @@ async def on_start() -> None: logic_settings["token_mapping"], logic_settings["gitlab_host"], logic_settings["app_host"], + logic_settings["use_system_hook"], ) except Exception as e: logger.warning( diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 05c6b21ddb..c264c2b821 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.33" +version = "0.1.34" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "]