[Integration][GitLab] Make Project Labels Optional (#1257)

# Description

What - Added mechanism to make the enrichment of project labels optional

Why - Not making it optional could leave to potential rate limit

How - Added an `includeLabels` param to the project kind

## Type of change

Please leave one option from the following and delete the rest:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [ ] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

<h4> All tests should be run against the port production
environment(using a testing org). </h4>

### Core testing checklist

- [ ] Integration able to create all default resources from scratch
- [ ] Resync finishes successfully
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Scheduled resync able to abort existing resync and start a new one
- [ ] Tested with at least 2 integrations from scratch
- [ ] Tested with Kafka and Polling event listeners
- [ ] Tested deletion of entities that don't pass the selector


### Integration testing checklist

- [ ] Integration able to create all default resources from scratch
- [ ] Resync able to create entities
- [ ] Resync able to update entities
- [ ] Resync able to detect and delete entities
- [ ] Resync finishes successfully
- [ ] If new resource kind is added or updated in the integration, add
example raw data, mapping and expected result to the `examples` folder
in the integration directory.
- [ ] If resource kind is updated, run the integration with the example
data and check if the expected result is achieved
- [ ] If new resource kind is added or updated, validate that
live-events for that resource are working as expected
- [ ] Docs PR link [here](#)

### Preflight checklist

- [ ] Handled rate limiting
- [ ] Handled pagination
- [ ] Implemented the code in async
- [ ] Support Multi account

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

Provide links to the API documentation used for this integration.

integrations/gitlab/.port/resources/blueprints.json

@@ -45,6 +45,13 @@
             "Other": "yellow"
           },
           "icon": "DefaultProperty"
+        },
+        "labels": {
+          "items": {
+            "type": "object"
+          },
+          "type": "array",
+          "title": "Labels"
         }
       },
       "required": []

integrations/gitlab/.port/resources/port-app-config.yaml

@@ -4,6 +4,7 @@ resources:
   - kind: project
     selector:
       query: "true"
+      includeLabels: "true"
     port:
       entity:
         mappings:
@@ -15,3 +16,4 @@ resources:
             readme: file://README.md
             description: .description
             language: .__languages | to_entries | max_by(.value) | .key
+            labels: .__labels

integrations/gitlab/CHANGELOG.md

@@ -7,6 +7,14 @@ this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.htm
 
 <!-- towncrier release notes start -->
 
+0.2.6 (2024-12-19)
+==================
+
+### Improvements
+
+- Added mechanism to make the enrichment of project labels optional
+
+
 0.2.5 (2024-12-16)
 ==================
 

integrations/gitlab/gitlab_integration/events/hooks/push.py

@@ -7,7 +7,10 @@
 
 from gitlab_integration.core.utils import generate_ref, does_pattern_apply
 from gitlab_integration.events.hooks.base import ProjectHandler
-from gitlab_integration.git_integration import GitlabPortAppConfig
+from gitlab_integration.git_integration import (
+    GitlabPortAppConfig,
+    GitLabProjectResourceConfig,
+)
 from gitlab_integration.utils import ObjectKind
 
 from port_ocean.clients.port.types import UserAgentType
@@ -100,8 +103,9 @@ async def _on_hook(self, body: dict[str, Any], gitlab_project: Project) -> None:
             logger.info(
                 f"Updating project information after push hook for project {gitlab_project.path_with_namespace}"
             )
+            include_labels = self.should_enrich_project_with_labels(config)
             enriched_project = await self.gitlab_service.enrich_project_with_extras(
-                gitlab_project
+                gitlab_project, include_labels
             )
             await ocean.register_raw(ObjectKind.PROJECT, [enriched_project.asdict()])
             await self._register_object_with_members(
@@ -188,3 +192,14 @@ async def _process_files(
             logger.debug(
                 f"Skipped {len(skipped_files)} files as they didn't match {spec_path} Skipped files: {skipped_files}"
             )
+
+    def should_enrich_project_with_labels(self, config: GitlabPortAppConfig) -> bool:
+        """Determine if a project should be enriched with labels based on config."""
+        return any(
+            resource_config.selector.include_labels
+            for resource_config in config.resources
+            if (
+                resource_config.kind == ObjectKind.PROJECT
+                and isinstance(resource_config, GitLabProjectResourceConfig)
+            )
+        )

integrations/gitlab/gitlab_integration/git_integration.py

@@ -122,7 +122,6 @@ class GitlabResourceConfig(ResourceConfig):
 
 
 class GitlabMemberSelector(Selector):
-
     include_inherited_members: bool = Field(
         alias="includeInheritedMembers",
         default=False,
@@ -147,6 +146,14 @@ class FilesSelector(BaseModel):
     )
 
 
+class GitLabProjectSelector(Selector):
+    include_labels: bool = Field(
+        alias="includeLabels",
+        description="Whether to enrich projects with labels",
+        default=False,
+    )
+
+
 class GitLabFilesSelector(Selector):
     files: FilesSelector
 
@@ -156,6 +163,11 @@ class GitLabFilesResourceConfig(ResourceConfig):
     kind: Literal["file"]
 
 
+class GitLabProjectResourceConfig(ResourceConfig):
+    selector: GitLabProjectSelector
+    kind: Literal["project"]
+
+
 class GitlabPortAppConfig(PortAppConfig):
     spec_path: str | List[str] = Field(alias="specPath", default="**/port.yml")
     branch: str | None
@@ -165,7 +177,7 @@ class GitlabPortAppConfig(PortAppConfig):
     project_visibility_filter: str | None = Field(
         alias="projectVisibilityFilter", default=None
     )
-    resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitlabResourceConfig] = Field(default_factory=list)  # type: ignore
+    resources: list[GitlabObjectWithMembersResourceConfig | GitLabFilesResourceConfig | GitLabProjectResourceConfig | GitlabResourceConfig] = Field(default_factory=list)  # type: ignore
 
 
 def _get_project_from_cache(project_id: int) -> Project | None:

integrations/gitlab/gitlab_integration/gitlab_service.py

@@ -571,11 +571,16 @@ async def async_project_language_wrapper(cls, project: Project) -> dict[str, Any
             return {"__languages": {}}
 
     @classmethod
-    async def enrich_project_with_extras(cls, project: Project) -> Project:
-        tasks = [
-            cls.async_project_language_wrapper(project),
-            cls.async_project_labels_wrapper(project),
-        ]
+    async def enrich_project_with_extras(
+        cls, project: Project, include_labels: bool = False
+    ) -> Project:
+        if include_labels:
+            tasks = [
+                cls.async_project_language_wrapper(project),
+                cls.async_project_labels_wrapper(project),
+            ]
+        else:
+            tasks = [cls.async_project_language_wrapper(project)]
         tasks_extras = await asyncio.gather(*tasks)
         for task_extras in tasks_extras:
             for key, value in task_extras.items():

integrations/gitlab/gitlab_integration/ocean.py

@@ -16,6 +16,7 @@
     GitlabResourceConfig,
     GitLabFilesResourceConfig,
     GitlabObjectWithMembersResourceConfig,
+    GitLabProjectResourceConfig,
 )
 from gitlab_integration.utils import ObjectKind, get_cached_all_services
 from port_ocean.context.event import event
@@ -132,7 +133,6 @@ async def resync_groups(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
 
 @ocean.on_resync(ObjectKind.GROUPWITHMEMBERS)
 async def resync_groups_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
-
     for service in get_cached_all_services():
         group_with_members_resource_config: GitlabObjectWithMembersResourceConfig = (
             typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config)
@@ -164,6 +164,9 @@ async def resync_groups_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
 
 @ocean.on_resync(ObjectKind.PROJECT)
 async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
+    project_selector = typing.cast(
+        GitLabProjectResourceConfig, event.resource_config
+    ).selector
     for service in get_cached_all_services():
         masked_token = len(str(service.gitlab_client.private_token)[:-4]) * "*"
         logger.info(f"fetching projects for token {masked_token}")
@@ -182,7 +185,11 @@ async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
                 )
                 tasks = []
                 for project in projects_batch:
-                    tasks.append(service.enrich_project_with_extras(project))
+                    tasks.append(
+                        service.enrich_project_with_extras(
+                            project, project_selector.include_labels
+                        )
+                    )
                 enriched_projects = await asyncio.gather(*tasks)
                 logger.info(
                     f"Finished Processing extras for {projects_processed_in_full_batch}/{len(projects)} projects in batch"
@@ -194,9 +201,7 @@ async def resync_projects(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
 
 @ocean.on_resync(ObjectKind.PROJECTWITHMEMBERS)
 async def resync_project_with_members(kind: str) -> ASYNC_GENERATOR_RESYNC_TYPE:
-
     for service in get_cached_all_services():
-
         project_with_members_resource_config: GitlabObjectWithMembersResourceConfig = (
             typing.cast(GitlabObjectWithMembersResourceConfig, event.resource_config)
         )

integrations/gitlab/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "gitlab"
-version = "0.2.5"
+version = "0.2.6"
 description = "Gitlab integration for Port using Port-Ocean Framework"
 authors = ["Yair Siman-Tov <[email protected]>"]