diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index 2858990505..3cff5ced76 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.1.141 (2024-11-13) +=================== + +### Features + +- Added support for gitlab member ingestion (PORT-7708) + + 0.1.140 (2024-11-12) ==================== diff --git a/integrations/gitlab/gitlab_integration/core/async_fetcher.py b/integrations/gitlab/gitlab_integration/core/async_fetcher.py index 1024d37249..586f58058d 100644 --- a/integrations/gitlab/gitlab_integration/core/async_fetcher.py +++ b/integrations/gitlab/gitlab_integration/core/async_fetcher.py @@ -12,6 +12,8 @@ ProjectPipeline, Issue, Group, + User, + GroupMember, ProjectFile, ) from loguru import logger @@ -36,6 +38,8 @@ async def fetch_single( Issue, Project, Group, + User, + GroupMember, ], ], *args, diff --git a/integrations/gitlab/gitlab_integration/events/hooks/base.py b/integrations/gitlab/gitlab_integration/events/hooks/base.py index 8b2b8a3bcf..91a675fff0 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/base.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/base.py @@ -1,9 +1,16 @@ from abc import ABC, abstractmethod -from typing import List, Any +from typing import List, Any, Dict, Optional +import typing from loguru import logger -from gitlab.v4.objects import Project - +from gitlab.v4.objects import Project, Group +from gitlab.base import RESTObject from gitlab_integration.gitlab_service import GitlabService +from port_ocean.context.ocean import ocean +from port_ocean.context.event import event +from gitlab_integration.git_integration import ( + GitlabPortAppConfig, + GitlabMemberSelector, +) class HookHandler(ABC): @@ -20,6 +27,41 @@ def __init__( async def on_hook(self, event: str, body: dict[str, Any]) -> None: pass + async def _register_object_with_members(self, kind: str, gitlab_object: RESTObject): + resource_configs = typing.cast( + GitlabPortAppConfig, event.port_app_config + ).resources + + matching_resource_configs = [ + resource_config + for resource_config in resource_configs + if ( + resource_config.kind == kind + and isinstance(resource_config.selector, GitlabMemberSelector) + ) + ] + + if not matching_resource_configs: + logger.info( + "Resource not found in port app config, update port app config to include the resource type" + ) + return + + for resource_config in matching_resource_configs: + include_bot_members = resource_config.selector.include_bot_members + include_inherited_members = ( + resource_config.selector.include_inherited_members + ) + + object_result: RESTObject = ( + await self.gitlab_service.enrich_object_with_members( + gitlab_object, + include_bot_members, + include_inherited_members, + ) + ) + await ocean.register_raw(resource_config.kind, [object_result.asdict()]) + class ProjectHandler(HookHandler): async def on_hook(self, event: str, body: dict[str, Any]) -> None: @@ -49,3 +91,23 @@ async def on_hook(self, event: str, body: dict[str, Any]) -> None: @abstractmethod async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: pass + + +class GroupHandler(HookHandler): + async def on_hook(self, event: str, body: dict[str, Any]) -> None: + logger.info(f"Handling {event}") + + group_id = body.get("group_id", body.get("group", {}).get("id")) + group = await self.gitlab_service.get_group(group_id) + await self._on_hook(body, group) + group_path = body.get("full_path", body.get("group_path")) + logger.info(f"Finished handling {event} for group {group_path}") + + @abstractmethod + async def _on_hook( + self, body: dict[str, Any], gitlab_group: Optional[Group] + ) -> None: + pass + + async def _register_group(self, kind: str, gitlab_group: Dict[str, Any]) -> None: + await ocean.register_raw(kind, [gitlab_group]) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/group.py b/integrations/gitlab/gitlab_integration/events/hooks/group.py index d4713ac786..e2c6ffda12 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/group.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/group.py @@ -1,39 +1,48 @@ -from typing import Any +from typing import Any, Optional from loguru import logger -from gitlab_integration.events.hooks.base import HookHandler from gitlab_integration.utils import ObjectKind +from gitlab_integration.events.hooks.base import GroupHandler from port_ocean.context.ocean import ocean +from gitlab.v4.objects import Group -class GroupHook(HookHandler): +class Groups(GroupHandler): events = ["Subgroup Hook"] - system_events = [ - "group_destroy", - "group_create", - "group_rename", - ] + system_events = ["group_destroy", "group_create", "group_rename"] - async def on_hook(self, event: str, body: dict[str, Any]) -> None: - event_name = body["event_name"] - - logger.info(f"Handling {event_name} for {event}") - - group_id = body["group_id"] if "group_id" in body else body["group"]["id"] - - logger.info(f"Handling hook {event} for group {group_id}") - - group = await self.gitlab_service.get_group(group_id) + async def _on_hook( + self, body: dict[str, Any], gitlab_group: Optional[Group] + ) -> None: + group_id = body.get("group_id") group_full_path = body.get("full_path") - if group: - await ocean.register_raw(ObjectKind.GROUP, [group.asdict()]) + event_name = body["event_name"] + + logger.info( + f"Handling event '{event_name}' for group with ID '{group_id}' and full path '{group_full_path}'" + ) + if gitlab_group: + await self._register_group( + ObjectKind.GROUP, + gitlab_group.asdict(), + ) + await self._register_object_with_members( + ObjectKind.GROUPWITHMEMBERS, gitlab_group + ) + logger.info(f"Registered group {group_id}") elif ( group_full_path and self.gitlab_service.should_run_for_path(group_full_path) and event_name in ("subgroup_destroy", "group_destroy") ): await ocean.unregister_raw(ObjectKind.GROUP, [body]) + await ocean.unregister_raw(ObjectKind.GROUPWITHMEMBERS, [body]) + logger.info(f"Unregistered group {group_id}") + return + else: - logger.info(f"Group {group_id} was filtered for event {event}. Skipping...") + logger.info( + f"Group {group_id} was filtered for event {event_name}. Skipping..." + ) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/members.py b/integrations/gitlab/gitlab_integration/events/hooks/members.py new file mode 100644 index 0000000000..5f7050194b --- /dev/null +++ b/integrations/gitlab/gitlab_integration/events/hooks/members.py @@ -0,0 +1,29 @@ +from typing import Any, Optional +from loguru import logger + +from gitlab_integration.utils import ObjectKind +from gitlab_integration.events.hooks.base import GroupHandler +from gitlab.v4.objects import Group + + +class Members(GroupHandler): + events = ["Member Hook"] + system_events = [ + "user_remove_from_group", + "user_update_for_group", + "user_add_to_group", + ] + + async def _on_hook( + self, body: dict[str, Any], gitlab_group: Optional[Group] + ) -> None: + if gitlab_group: + event_name, user_username = (body["event_name"], body["user_username"]) + logger.info(f"Handling {event_name} for group member {user_username}") + await self._register_object_with_members( + ObjectKind.GROUPWITHMEMBERS, gitlab_group + ) + else: + logger.info( + f"Group member's group {body['group_id']} was filtered for event {body['event_name']}. Skipping..." + ) diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index 11291e277b..8cbc804415 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -103,7 +103,10 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: enriched_project = await self.gitlab_service.enrich_project_with_extras( gitlab_project ) - await ocean.register_raw(ObjectKind.PROJECT, [enriched_project]) + await ocean.register_raw(ObjectKind.PROJECT, [enriched_project.asdict()]) + await self._register_object_with_members( + ObjectKind.PROJECTWITHMEMBERS, enriched_project + ) else: logger.debug( diff --git a/integrations/gitlab/gitlab_integration/events/setup.py b/integrations/gitlab/gitlab_integration/events/setup.py index 53cae4e792..c528dcd97c 100644 --- a/integrations/gitlab/gitlab_integration/events/setup.py +++ b/integrations/gitlab/gitlab_integration/events/setup.py @@ -11,7 +11,8 @@ from gitlab_integration.events.hooks.merge_request import MergeRequest from gitlab_integration.events.hooks.pipelines import Pipelines from gitlab_integration.events.hooks.push import PushHook -from gitlab_integration.events.hooks.group import GroupHook +from gitlab_integration.events.hooks.members import Members +from gitlab_integration.events.hooks.group import Groups from gitlab_integration.events.hooks.project_files import ProjectFiles from gitlab_integration.gitlab_service import GitlabService from gitlab_integration.models.webhook_groups_override_config import ( @@ -122,7 +123,8 @@ def setup_listeners(gitlab_service: GitlabService, group_id: str) -> None: Job(gitlab_service), Issues(gitlab_service), Pipelines(gitlab_service), - GroupHook(gitlab_service), + Groups(gitlab_service), + Members(gitlab_service), ProjectFiles(gitlab_service), ] for handler in handlers: @@ -140,7 +142,8 @@ def setup_system_listeners(gitlab_clients: list[GitlabService]) -> None: Job, Issues, Pipelines, - GroupHook, + Groups, + Members, ProjectFiles, ] for handler in handlers: diff --git a/integrations/gitlab/gitlab_integration/git_integration.py b/integrations/gitlab/gitlab_integration/git_integration.py index 1a5f7ac456..7c8faca24f 100644 --- a/integrations/gitlab/gitlab_integration/git_integration.py +++ b/integrations/gitlab/gitlab_integration/git_integration.py @@ -121,6 +121,25 @@ class GitlabResourceConfig(ResourceConfig): selector: GitlabSelector +class GitlabMemberSelector(Selector): + + include_inherited_members: bool = Field( + alias="includeInheritedMembers", + default=False, + description="If set to true, the integration will include inherited members in the group members list. Default value is false", + ) + include_bot_members: bool = Field( + alias="includeBotMembers", + default=True, + description="If set to false, bots will be filtered out from the members list. Default value is true", + ) + + +class GitlabObjectWithMembersResourceConfig(ResourceConfig): + kind: Literal["project-with-members", "group-with-members"] + selector: GitlabMemberSelector + + class FilesSelector(BaseModel): path: str = Field(description="The path to get the files from") repos: List[str] = Field( @@ -146,7 +165,7 @@ class GitlabPortAppConfig(PortAppConfig): project_visibility_filter: str | None = Field( alias="projectVisibilityFilter", default=None ) - resources: list[GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list) # type: ignore + resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list) # type: ignore def _get_project_from_cache(project_id: int) -> Project | None: diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index b70311b009..0ecaec3671 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -8,8 +8,9 @@ import aiolimiter import anyio.to_thread import yaml +import gitlab.exceptions from gitlab import Gitlab, GitlabError, GitlabList -from gitlab.base import RESTObject +from gitlab.base import RESTObject, RESTObjectList from gitlab.v4.objects import ( Group, GroupMergeRequest, @@ -29,14 +30,21 @@ from port_ocean.context.event import event from port_ocean.core.models import Entity +from port_ocean.utils.cache import cache_iterator_result +import functools PROJECTS_CACHE_KEY = "__cache_all_projects" + + MAX_ALLOWED_FILE_SIZE_IN_BYTES = 1024 * 1024 # 1MB GITLAB_SEARCH_RATE_LIMIT = 100 if TYPE_CHECKING: from gitlab_integration.git_integration import GitlabPortAppConfig +MAXIMUM_CONCURRENT_TASK = 10 +semaphore = asyncio.BoundedSemaphore(MAXIMUM_CONCURRENT_TASK) + class GitlabService: all_events_in_webhook: list[str] = [ @@ -49,6 +57,7 @@ class GitlabService: "tag_push_events", "subgroup_events", "confidential_issues_events", + "member_events", ] def __init__( @@ -443,19 +452,35 @@ async def get_project(self, project_id: int) -> Project | None: else: return None - async def get_group(self, group_id: int) -> Group | None: - logger.info(f"fetching group {group_id}") - group = await AsyncFetcher.fetch_single(self.gitlab_client.groups.get, group_id) - if isinstance(group, Group) and self.should_run_for_group(group): - return group - else: - return None + async def get_group(self, group_id: int) -> Optional[Group]: + try: + logger.info(f"Fetching group with ID: {group_id}") + group = await AsyncFetcher.fetch_single( + self.gitlab_client.groups.get, group_id + ) + if isinstance(group, Group) and self.should_run_for_group(group): + return group + else: + return None + except gitlab.exceptions.GitlabGetError as err: + if err.response_code == 404: + logger.warning(f"Group with ID {group_id} not found (404).") + return None + else: + logger.error(f"Failed to fetch group with ID {group_id}: {err}") + raise - async def get_all_groups(self) -> typing.AsyncIterator[List[Group]]: + @cache_iterator_result() + async def get_all_groups( + self, skip_validation: bool = False + ) -> typing.AsyncIterator[List[Group]]: logger.info("fetching all groups for the token") + async for groups_batch in AsyncFetcher.fetch_batch( fetch_func=self.gitlab_client.groups.list, - validation_func=self.should_run_for_group, + validation_func=( + self.should_run_for_group if not (skip_validation) else None + ), pagination="offset", order_by="id", sort="asc", @@ -534,20 +559,15 @@ async def async_project_language_wrapper(cls, project: Project) -> dict[str, Any return {"__languages": {}} @classmethod - async def enrich_project_with_extras(cls, project: Project) -> dict[str, Any]: + async def enrich_project_with_extras(cls, project: Project) -> Project: tasks = [ cls.async_project_language_wrapper(project), ] tasks_extras = await asyncio.gather(*tasks) - project_with_extras = project.asdict() - project_with_extras.update( - **{ - key: value - for task_extras in tasks_extras - for key, value in task_extras.items() - } - ) - return project_with_extras + for task_extras in tasks_extras: + for key, value in task_extras.items(): + setattr(project, key, value) # Update the project object + return project @staticmethod def validate_file_is_directory( @@ -683,6 +703,70 @@ async def get_all_issues(self, group: Group) -> typing.AsyncIterator[List[Issue] issues: List[Issue] = typing.cast(List[Issue], issues_batch) yield issues + def should_run_for_members(self, include_bot_members: bool, member: RESTObject): + return include_bot_members or not member.username.__contains__("bot") + + async def enrich_object_with_members( + self, + obj: RESTObject, + include_inherited_members: bool = False, + include_bot_members: bool = True, + ) -> RESTObject: + """ + Enriches an object (e.g., Project or Group) with its members and optionally their public emails. + """ + members_list = [ + member + async for members in self.get_all_object_members( + obj, include_inherited_members, include_bot_members + ) + for member in members + ] + + setattr(obj, "__members", [member.asdict() for member in members_list]) + return obj + + async def get_all_object_members( + self, + obj: RESTObject, + include_inherited_members: bool = False, + include_bot_members: bool = True, + ) -> AsyncIterator[RESTObjectList]: + """ + Fetches all members of an object (e.g., Project or Group) generically. + """ + try: + obj_name = getattr(obj, "name", "unknown") + logger.info(f"Fetching all members of {obj_name}") + + members_attr = "members_all" if include_inherited_members else "members" + members_manager = getattr(obj, members_attr, None) + if not members_manager: + raise AttributeError(f"Object does not have attribute '{members_attr}'") + + fetch_func = members_manager.list + + validation_func = functools.partial( + self.should_run_for_members, include_bot_members + ) + + async for members_batch in AsyncFetcher.fetch_batch( + fetch_func=fetch_func, + validation_func=validation_func, + pagination="offset", + order_by="id", + sort="asc", + ): + members: RESTObjectList = typing.cast(RESTObjectList, members_batch) + + logger.info( + f"Queried {len(members)} members {[member.username for member in members]} from {obj_name}" + ) + yield members + except Exception as e: + logger.error(f"Failed to get members for object='{obj_name}'. Error: {e}") + return + async def get_entities_diff( self, project: Project, diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index 31f6c1be97..c6b8766920 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -15,6 +15,7 @@ from gitlab_integration.git_integration import ( GitlabResourceConfig, GitLabFilesResourceConfig, + GitlabObjectWithMembersResourceConfig, ) from gitlab_integration.utils import ObjectKind, get_cached_all_services from port_ocean.context.event import event @@ -24,7 +25,7 @@ from port_ocean.utils.async_iterators import stream_async_iterators_tasks NO_WEBHOOK_WARNING = "Without setting up the webhook, the integration will not export live changes from the gitlab" -PROJECT_RESYNC_BATCH_SIZE = 10 +RESYNC_BATCH_SIZE = 10 async def start_processors() -> None: @@ -129,8 +130,40 @@ async def resync_groups(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: yield [group.asdict() for group in groups_batch] +@ocean.on_resync(ObjectKind.GROUPWITHMEMBERS) +async def resync_groups_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + + for service in get_cached_all_services(): + group_with_members_resource_config: GitlabObjectWithMembersResourceConfig = ( + typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config) + ) + group_with_members_selector = group_with_members_resource_config.selector + include_inherited_members = ( + group_with_members_selector.include_inherited_members + ) + include_bot_members = group_with_members_selector.include_bot_members + + async for groups in service.get_all_groups(): + groups_batch_iter = iter(groups) + groups_processed_in_full_batch = 0 + + while groups_batch := tuple(islice(groups_batch_iter, RESYNC_BATCH_SIZE)): + groups_processed_in_full_batch += len(groups_batch) + logger.info( + f"Processing extras for {groups_processed_in_full_batch}/{len(groups)} groups in batch" + ) + tasks = [ + service.enrich_object_with_members( + group, include_inherited_members, include_bot_members + ) + for group in groups_batch + ] + enriched_groups = await asyncio.gather(*tasks) + yield [enriched_group.asdict() for enriched_group in enriched_groups] + + @ocean.on_resync(ObjectKind.PROJECT) -async def on_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: +async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: for service in get_cached_all_services(): masked_token = len(str(service.gitlab_client.private_token)[:-4]) * "*" logger.info(f"fetching projects for token {masked_token}") @@ -141,7 +174,7 @@ async def on_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: projects_batch_iter = iter(projects) projects_processed_in_full_batch = 0 while projects_batch := tuple( - islice(projects_batch_iter, PROJECT_RESYNC_BATCH_SIZE) + islice(projects_batch_iter, RESYNC_BATCH_SIZE) ): projects_processed_in_full_batch += len(projects_batch) logger.info( @@ -154,7 +187,61 @@ async def on_resync(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: logger.info( f"Finished Processing extras for {projects_processed_in_full_batch}/{len(projects)} projects in batch" ) - yield enriched_projects + yield [ + enriched_project.asdict() for enriched_project in enriched_projects + ] + + +@ocean.on_resync(ObjectKind.PROJECTWITHMEMBERS) +async def resync_project_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + + for service in get_cached_all_services(): + + project_with_members_resource_config: GitlabObjectWithMembersResourceConfig = ( + typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config) + ) + if not isinstance( + project_with_members_resource_config, GitlabObjectWithMembersResourceConfig + ): + return + + project_with_members_selector = project_with_members_resource_config.selector + include_inherited_members = ( + project_with_members_selector.include_inherited_members + ) + include_bot_members = project_with_members_selector.include_bot_members + + async for projects in service.get_all_projects(): + projects_batch_iter = iter(projects) + projects_processed_in_full_batch = 0 + while projects_batch := tuple( + islice(projects_batch_iter, RESYNC_BATCH_SIZE) + ): + projects_processed_in_full_batch += len(projects_batch) + logger.info( + f"Processing extras for {projects_processed_in_full_batch}/{len(projects)} projects in batch" + ) + tasks = [ + service.enrich_project_with_extras(project) + for project in projects_batch + ] + projects_enriched_with_extras = await asyncio.gather(*tasks) + logger.info( + f"Finished Processing extras for {projects_processed_in_full_batch}/{len(projects)} projects in batch" + ) + members_tasks = [ + service.enrich_object_with_members( + project, + include_inherited_members, + include_bot_members, + ) + for project in projects_enriched_with_extras + ] + projects_enriched_with_members = await asyncio.gather(*members_tasks) + yield [ + enriched_projects.asdict() + for enriched_projects in projects_enriched_with_members + ] @ocean.on_resync(ObjectKind.FOLDER) @@ -198,7 +285,7 @@ async def resync_files(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: projects_batch_iter = iter(projects) projects_processed_in_full_batch = 0 while projects_batch := tuple( - islice(projects_batch_iter, PROJECT_RESYNC_BATCH_SIZE) + islice(projects_batch_iter, RESYNC_BATCH_SIZE) ): projects_processed_in_full_batch += len(projects_batch) logger.info( diff --git a/integrations/gitlab/gitlab_integration/utils.py b/integrations/gitlab/gitlab_integration/utils.py index 5b6592d9f2..c39b1cc46e 100644 --- a/integrations/gitlab/gitlab_integration/utils.py +++ b/integrations/gitlab/gitlab_integration/utils.py @@ -7,6 +7,8 @@ from port_ocean.context.ocean import ocean from port_ocean.exceptions.context import EventContextNotFoundError +RETRY_TRANSIENT_ERRORS = True + def get_all_services() -> List[GitlabService]: logic_settings = ocean.integration_config @@ -16,7 +18,11 @@ def get_all_services() -> List[GitlabService]: f"Creating gitlab clients for {len(logic_settings['token_mapping'])} tokens" ) for token, group_mapping in logic_settings["token_mapping"].items(): - gitlab_client = Gitlab(logic_settings["gitlab_host"], token) + gitlab_client = Gitlab( + logic_settings["gitlab_host"], + token, + retry_transient_errors=RETRY_TRANSIENT_ERRORS, + ) gitlab_service = GitlabService( gitlab_client, logic_settings["app_host"], group_mapping ) @@ -46,3 +52,5 @@ class ObjectKind: PROJECT = "project" FOLDER = "folder" FILE = "file" + GROUPWITHMEMBERS = "group-with-members" + PROJECTWITHMEMBERS = "project-with-members" diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 20869c7094..73b6d0455c 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.1.140" +version = "0.1.141" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "] diff --git a/integrations/gitlab/tests/gitlab_integration/test_gitlab_service.py b/integrations/gitlab/tests/gitlab_integration/test_gitlab_service.py index bd5accf0e3..b7f5036ede 100644 --- a/integrations/gitlab/tests/gitlab_integration/test_gitlab_service.py +++ b/integrations/gitlab/tests/gitlab_integration/test_gitlab_service.py @@ -1,6 +1,8 @@ from typing import Any -from unittest.mock import MagicMock +from unittest.mock import MagicMock, Mock from gitlab_integration.gitlab_service import GitlabService +from gitlab.base import RESTObject +import pytest def mock_search(page: int, *args: Any, **kwargs: Any) -> Any: @@ -213,3 +215,151 @@ async def test_get_and_parse_single_file_json( # Assert assert expected_parsed_single_file == actual_parsed_single_file + + +class MockMember(RESTObject): + def __init__(self, id, username): + self.id = id + self.username = username + + def asdict(self): + return self.__dict__ + + def __setattr__(self, name, value): + self.__dict__[name] = value + + +class MockGroup(RESTObject): + def __init__(self, id, name): + self.id = id + self.name = name + self.members = self.MockMembers() + self.members_all = self.MockMembersAll() + + class MockMembers: + def list(self, page, *args: Any, **kwargs: Any): + if page == 1: + return [ + MockMember(1, "user1"), + MockMember(1, "bot_user1"), + ] + elif page == 2: + return [ + MockMember(2, "user2"), + MockMember(2, "bot_user2"), + ] + elif page == 3: + return [ + MockMember(3, "user3"), + MockMember(3, "bot_user3"), + ] + return + + class MockMembersAll: + def list(self, page, *args: Any, **kwargs: Any): + if page == 1: + return [ + MockMember(1, "user1"), + MockMember(1, "bot_user1"), + MockMember(1, "inherited_member_1"), + ] + elif page == 2: + return [ + MockMember(2, "user2"), + MockMember(2, "bot_user2"), + MockMember(2, "inherited_member_2"), + ] + elif page == 3: + return [ + MockMember(3, "user3"), + MockMember(3, "bot_user3"), + MockMember(3, "inherited_member_3"), + ] + return + + def asdict(self): + return { + "id": self.id, + "name": self.name, + "path": f"get{self.name}-path", + "full_name": self.name, + "full_path": f"get{self.name}-path", + } + + def __setattr__(self, name, value): + self.__dict__[name] = value + + +def test_should_run_for_members( + monkeypatch: Any, mocked_gitlab_service: GitlabService +) -> None: + + bot_member = Mock(spec=RESTObject) + bot_member.username = "bot_user" + + non_bot_member = Mock(spec=RESTObject) + non_bot_member.username = "regular_user" + + assert mocked_gitlab_service.should_run_for_members(True, bot_member) is True + assert mocked_gitlab_service.should_run_for_members(True, non_bot_member) is True + + assert mocked_gitlab_service.should_run_for_members(False, bot_member) is False + assert mocked_gitlab_service.should_run_for_members(False, non_bot_member) is True + + +@pytest.mark.asyncio +async def test_get_all_object_members( + monkeypatch: Any, mocked_gitlab_service: GitlabService +) -> None: + + # Arrange + obj = MockGroup(123, "test_project") + + # Act + from typing import List + + results_without_inherited_members: List[RESTObject] = [] + async for members in mocked_gitlab_service.get_all_object_members( + obj, include_inherited_members=False, include_bot_members=True + ): + results_without_inherited_members.extend(members) + + results_with_inherited_members: List[RESTObject] = [] + async for members in mocked_gitlab_service.get_all_object_members( + obj, include_inherited_members=True, include_bot_members=True + ): + results_with_inherited_members.extend(members) + + results_without_bot_members: List[RESTObject] = [] + async for members in mocked_gitlab_service.get_all_object_members( + obj, include_inherited_members=True, include_bot_members=False + ): + results_without_bot_members.extend(members) + + # Assert + assert len(results_without_inherited_members) == 6 + assert results_without_inherited_members[0].username == "user1" + assert results_without_inherited_members[1].username == "bot_user1" + assert len(results_with_inherited_members) == 9 + assert len(results_without_bot_members) == 6 + + +@pytest.mark.asyncio +async def test_enrich_object_with_members( + monkeypatch: Any, mocked_gitlab_service: GitlabService +) -> None: + + # Arrange + obj = MockGroup(123, "test_group") + + # Act + enriched_obj: RESTObject = await mocked_gitlab_service.enrich_object_with_members( + obj, + include_inherited_members=False, + include_bot_members=True, + ) + + # Assert + assert enriched_obj.name == "test_group" + assert len(enriched_obj.__members) == 6 + assert enriched_obj.__members[0] == {"id": 1, "username": "user1"} diff --git a/integrations/gitlab/tests/gitlab_integration/test_gitlab_service_webhook.py b/integrations/gitlab/tests/gitlab_integration/test_gitlab_service_webhook.py index f3656de006..2fe823cc5d 100644 --- a/integrations/gitlab/tests/gitlab_integration/test_gitlab_service_webhook.py +++ b/integrations/gitlab/tests/gitlab_integration/test_gitlab_service_webhook.py @@ -72,6 +72,7 @@ async def test_create_group_webhook_success( "tag_push_events": False, "subgroup_events": False, "confidential_issues_events": False, + "member_events": False, } ) @@ -103,5 +104,6 @@ async def test_create_group_webhook_failure( "tag_push_events": False, "subgroup_events": False, "confidential_issues_events": False, + "member_events": False, } )