From 5f6a8917a1439bce18af9aa41bf10960f31eb47f Mon Sep 17 00:00:00 2001 From: PagesCoffy Date: Fri, 20 Dec 2024 09:03:20 +0000 Subject: [PATCH] [Integration][GitLab] Make Project Labels Optional (#1257) # Description What - Added mechanism to make the enrichment of project labels optional Why - Not making it optional could leave to potential rate limit How - Added an `includeLabels` param to the project kind ## Type of change Please leave one option from the following and delete the rest: - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation)

All tests should be run against the port production environment(using a testing org).

### Core testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync finishes successfully - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Scheduled resync able to abort existing resync and start a new one - [ ] Tested with at least 2 integrations from scratch - [ ] Tested with Kafka and Polling event listeners - [ ] Tested deletion of entities that don't pass the selector ### Integration testing checklist - [ ] Integration able to create all default resources from scratch - [ ] Resync able to create entities - [ ] Resync able to update entities - [ ] Resync able to detect and delete entities - [ ] Resync finishes successfully - [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory. - [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved - [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected - [ ] Docs PR link [here](#) ### Preflight checklist - [ ] Handled rate limiting - [ ] Handled pagination - [ ] Implemented the code in async - [ ] Support Multi account ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --- .../gitlab/.port/resources/blueprints.json | 7 +++++++ .../.port/resources/port-app-config.yaml | 2 ++ integrations/gitlab/CHANGELOG.md | 8 ++++++++ .../gitlab_integration/events/hooks/push.py | 19 +++++++++++++++++-- .../gitlab_integration/git_integration.py | 16 ++++++++++++++-- .../gitlab_integration/gitlab_service.py | 15 ++++++++++----- .../gitlab/gitlab_integration/ocean.py | 13 +++++++++---- integrations/gitlab/pyproject.toml | 2 +- 8 files changed, 68 insertions(+), 14 deletions(-) diff --git a/integrations/gitlab/.port/resources/blueprints.json b/integrations/gitlab/.port/resources/blueprints.json index b51ae9d793..1db263a330 100644 --- a/integrations/gitlab/.port/resources/blueprints.json +++ b/integrations/gitlab/.port/resources/blueprints.json @@ -45,6 +45,13 @@ "Other": "yellow" }, "icon": "DefaultProperty" + }, + "labels": { + "items": { + "type": "object" + }, + "type": "array", + "title": "Labels" } }, "required": [] diff --git a/integrations/gitlab/.port/resources/port-app-config.yaml b/integrations/gitlab/.port/resources/port-app-config.yaml index d1c2882763..c2b5817b83 100644 --- a/integrations/gitlab/.port/resources/port-app-config.yaml +++ b/integrations/gitlab/.port/resources/port-app-config.yaml @@ -4,6 +4,7 @@ resources: - kind: project selector: query: "true" + includeLabels: "true" port: entity: mappings: @@ -15,3 +16,4 @@ resources: readme: file://README.md description: .description language: .__languages | to_entries | max_by(.value) | .key + labels: .__labels diff --git a/integrations/gitlab/CHANGELOG.md b/integrations/gitlab/CHANGELOG.md index ec61ef0747..e022334d3c 100644 --- a/integrations/gitlab/CHANGELOG.md +++ b/integrations/gitlab/CHANGELOG.md @@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm +0.2.6 (2024-12-19) +================== + +### Improvements + +- Added mechanism to make the enrichment of project labels optional + + 0.2.5 (2024-12-16) ================== diff --git a/integrations/gitlab/gitlab_integration/events/hooks/push.py b/integrations/gitlab/gitlab_integration/events/hooks/push.py index 8cbc804415..99e04db4c6 100644 --- a/integrations/gitlab/gitlab_integration/events/hooks/push.py +++ b/integrations/gitlab/gitlab_integration/events/hooks/push.py @@ -7,7 +7,10 @@ from gitlab_integration.core.utils import generate_ref, does_pattern_apply from gitlab_integration.events.hooks.base import ProjectHandler -from gitlab_integration.git_integration import GitlabPortAppConfig +from gitlab_integration.git_integration import ( + GitlabPortAppConfig, + GitLabProjectResourceConfig, +) from gitlab_integration.utils import ObjectKind from port_ocean.clients.port.types import UserAgentType @@ -100,8 +103,9 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None: logger.info( f"Updating project information after push hook for project {gitlab_project.path_with_namespace}" ) + include_labels = self.should_enrich_project_with_labels(config) enriched_project = await self.gitlab_service.enrich_project_with_extras( - gitlab_project + gitlab_project, include_labels ) await ocean.register_raw(ObjectKind.PROJECT, [enriched_project.asdict()]) await self._register_object_with_members( @@ -188,3 +192,14 @@ async def _process_files( logger.debug( f"Skipped {len(skipped_files)} files as they didn't match {spec_path} Skipped files: {skipped_files}" ) + + def should_enrich_project_with_labels(self, config: GitlabPortAppConfig) -> bool: + """Determine if a project should be enriched with labels based on config.""" + return any( + resource_config.selector.include_labels + for resource_config in config.resources + if ( + resource_config.kind == ObjectKind.PROJECT + and isinstance(resource_config, GitLabProjectResourceConfig) + ) + ) diff --git a/integrations/gitlab/gitlab_integration/git_integration.py b/integrations/gitlab/gitlab_integration/git_integration.py index 7c8faca24f..0195af7af4 100644 --- a/integrations/gitlab/gitlab_integration/git_integration.py +++ b/integrations/gitlab/gitlab_integration/git_integration.py @@ -122,7 +122,6 @@ class GitlabResourceConfig(ResourceConfig): class GitlabMemberSelector(Selector): - include_inherited_members: bool = Field( alias="includeInheritedMembers", default=False, @@ -147,6 +146,14 @@ class FilesSelector(BaseModel): ) +class GitLabProjectSelector(Selector): + include_labels: bool = Field( + alias="includeLabels", + description="Whether to enrich projects with labels", + default=False, + ) + + class GitLabFilesSelector(Selector): files: FilesSelector @@ -156,6 +163,11 @@ class GitLabFilesResourceConfig(ResourceConfig): kind: Literal["file"] +class GitLabProjectResourceConfig(ResourceConfig): + selector: GitLabProjectSelector + kind: Literal["project"] + + class GitlabPortAppConfig(PortAppConfig): spec_path: str | List[str] = Field(alias="specPath", default="**/port.yml") branch: str | None @@ -165,7 +177,7 @@ class GitlabPortAppConfig(PortAppConfig): project_visibility_filter: str | None = Field( alias="projectVisibilityFilter", default=None ) - resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list) # type: ignore + resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitLabProjectResourceConfig | GitlabResourceConfig] = Field(default_factory=list) # type: ignore def _get_project_from_cache(project_id: int) -> Project | None: diff --git a/integrations/gitlab/gitlab_integration/gitlab_service.py b/integrations/gitlab/gitlab_integration/gitlab_service.py index b2fa6519de..0e21033806 100644 --- a/integrations/gitlab/gitlab_integration/gitlab_service.py +++ b/integrations/gitlab/gitlab_integration/gitlab_service.py @@ -571,11 +571,16 @@ async def async_project_language_wrapper(cls, project: Project) -> dict[str, Any return {"__languages": {}} @classmethod - async def enrich_project_with_extras(cls, project: Project) -> Project: - tasks = [ - cls.async_project_language_wrapper(project), - cls.async_project_labels_wrapper(project), - ] + async def enrich_project_with_extras( + cls, project: Project, include_labels: bool = False + ) -> Project: + if include_labels: + tasks = [ + cls.async_project_language_wrapper(project), + cls.async_project_labels_wrapper(project), + ] + else: + tasks = [cls.async_project_language_wrapper(project)] tasks_extras = await asyncio.gather(*tasks) for task_extras in tasks_extras: for key, value in task_extras.items(): diff --git a/integrations/gitlab/gitlab_integration/ocean.py b/integrations/gitlab/gitlab_integration/ocean.py index c6b8766920..5340faf1ac 100644 --- a/integrations/gitlab/gitlab_integration/ocean.py +++ b/integrations/gitlab/gitlab_integration/ocean.py @@ -16,6 +16,7 @@ GitlabResourceConfig, GitLabFilesResourceConfig, GitlabObjectWithMembersResourceConfig, + GitLabProjectResourceConfig, ) from gitlab_integration.utils import ObjectKind, get_cached_all_services from port_ocean.context.event import event @@ -132,7 +133,6 @@ async def resync_groups(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: @ocean.on_resync(ObjectKind.GROUPWITHMEMBERS) async def resync_groups_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: - for service in get_cached_all_services(): group_with_members_resource_config: GitlabObjectWithMembersResourceConfig = ( typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config) @@ -164,6 +164,9 @@ async def resync_groups_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: @ocean.on_resync(ObjectKind.PROJECT) async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: + project_selector = typing.cast( + GitLabProjectResourceConfig, event.resource_config + ).selector for service in get_cached_all_services(): masked_token = len(str(service.gitlab_client.private_token)[:-4]) * "*" logger.info(f"fetching projects for token {masked_token}") @@ -182,7 +185,11 @@ async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: ) tasks = [] for project in projects_batch: - tasks.append(service.enrich_project_with_extras(project)) + tasks.append( + service.enrich_project_with_extras( + project, project_selector.include_labels + ) + ) enriched_projects = await asyncio.gather(*tasks) logger.info( f"Finished Processing extras for {projects_processed_in_full_batch}/{len(projects)} projects in batch" @@ -194,9 +201,7 @@ async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: @ocean.on_resync(ObjectKind.PROJECTWITHMEMBERS) async def resync_project_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE: - for service in get_cached_all_services(): - project_with_members_resource_config: GitlabObjectWithMembersResourceConfig = ( typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config) ) diff --git a/integrations/gitlab/pyproject.toml b/integrations/gitlab/pyproject.toml index 83dc642747..f24540325e 100644 --- a/integrations/gitlab/pyproject.toml +++ b/integrations/gitlab/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "gitlab" -version = "0.2.5" +version = "0.2.6" description = "Gitlab integration for Port using Port-Ocean Framework" authors = ["Yair Siman-Tov "]