✨ Source Gilab: Deployments Streams (#31183)

Co-authored-by: samwinder <[email protected]> Co-authored-by: Ben Shaw <[email protected]> Co-authored-by: Ben Shaw <[email protected]> Co-authored-by: darynaishchenko <[email protected]>
airbytehq · Oct 12, 2023 · 9034271 · 9034271
1 parent 33c683b
commit 9034271
Show file tree

Hide file tree

Showing 11 changed files with 198 additions and 70 deletions.
diff --git a/airbyte-integrations/connectors/source-gitlab/Dockerfile b/airbyte-integrations/connectors/source-gitlab/Dockerfile
@@ -13,5 +13,5 @@ COPY main.py ./
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=1.6.0
+LABEL io.airbyte.version=1.7.0
 LABEL io.airbyte.name=airbyte/source-gitlab
diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-gitlab/integration_tests/configured_catalog.json
@@ -185,6 +185,19 @@
       "destination_sync_mode": "overwrite",
       "primary_key": [["name"]]
     },
+    {
+      "stream": {
+        "name": "deployments",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["updated_at"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "full_refresh",
+      "destination_sync_mode": "overwrite",
+      "primary_key": [["id"]]
+    },
     {
       "stream": {
         "name": "tags",

diff --git a/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl b/airbyte-integrations/connectors/source-gitlab/integration_tests/expected_records.jsonl
diff --git a/...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl b/...e-integrations/connectors/source-gitlab/integration_tests/expected_records_with_ids.jsonl
diff --git a/airbyte-integrations/connectors/source-gitlab/metadata.yaml b/airbyte-integrations/connectors/source-gitlab/metadata.yaml
@@ -5,7 +5,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 5e6175e5-68e1-4c17-bff9-56103bbb0d80
-  dockerImageTag: 1.6.0
+  dockerImageTag: 1.7.0
   dockerRepository: airbyte/source-gitlab
   githubIssueLabel: source-gitlab
   icon: gitlab.svg

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/deployments.json b/airbyte-integrations/connectors/source-gitlab/source_gitlab/schemas/deployments.json
@@ -0,0 +1,52 @@
+{
+  "$schema": "https://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "id": {
+      "type": ["null", "integer"]
+    },
+    "status": {
+      "type": ["null", "string"]
+    },
+    "ref": {
+      "type": ["null", "string"]
+    },
+    "sha": {
+      "type": ["null", "string"]
+    },
+    "environment_name": {
+      "type": ["null", "string"]
+    },
+    "created_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "updated_at": {
+      "type": ["null", "string"],
+      "format": "date-time"
+    },
+    "user": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
+    "user_full_name": {
+      "type": ["null", "string"]
+    },
+    "user_username": {
+      "type": ["null", "string"]
+    },
+    "user_id": {
+      "type": ["null", "integer"]
+    },
+    "environment": {
+      "type": ["null", "object"],
+      "additionalProperties": true
+    },
+    "environment_id": {
+      "type": ["null", "integer"]
+    },
+    "project_id": {
+      "type": ["null", "integer"]
+    }
+  }
+}
diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/source.py
@@ -18,6 +18,7 @@
 from .streams import (
     Branches,
     Commits,
+    Deployments,
     EpicIssues,
     Epics,
     GitlabStream,
@@ -171,6 +172,7 @@ def streams(self, config: MutableMapping[str, Any]) -> List[Stream]:
             Branches(parent_stream=projects, repository_part=True, **auth_params),
             Commits(parent_stream=projects, repository_part=True, start_date=config["start_date"], **auth_params),
             epics,
+            Deployments(parent_stream=projects, **auth_params),
             EpicIssues(parent_stream=epics, **auth_params),
             GroupIssueBoards(parent_stream=groups, **auth_params),
             Issues(parent_stream=projects, start_date=config["start_date"], **auth_params),

diff --git a/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py b/airbyte-integrations/connectors/source-gitlab/source_gitlab/streams.py
@@ -404,3 +404,17 @@ class EpicIssues(GitlabChildStream):
     flatten_id_keys = ["milestone", "assignee", "author"]
     flatten_list_keys = ["assignees"]
     path_template = "groups/{group_id}/epics/{iid}/issues"
+
+
+class Deployments(GitlabChildStream):
+    primary_key = "id"
+    flatten_id_keys = ["user", "environment"]
+    path_template = "projects/{id}/deployments"
+
+    def transform(self, record, stream_slice: Mapping[str, Any] = None, **kwargs):
+        super().transform(record, stream_slice, **kwargs)
+        record["user_username"] = record["user"]["username"]
+        record["user_full_name"] = record["user"]["name"]
+        record["environment_name"] = record["environment"]["name"]
+        record["project_id"] = stream_slice["id"]
+        return record
diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_source.py
@@ -13,7 +13,7 @@ def test_streams(config, requests_mock):
     requests_mock.get("/api/v4/groups", json=[{"id": "g1"}, {"id": "g256"}])
     source = SourceGitlab()
     streams = source.streams(config)
-    assert len(streams) == 22
+    assert len(streams) == 23
     assert all([isinstance(stream, GitlabStream) for stream in streams])
     groups, projects, *_ = streams
     assert groups.group_ids == ["g1", "g256"]

diff --git a/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-gitlab/unit_tests/test_streams.py
@@ -6,7 +6,18 @@
 
 import pytest
 from airbyte_cdk.sources.streams.http.auth import NoAuth
-from source_gitlab.streams import Branches, Commits, Jobs, MergeRequestCommits, MergeRequests, Pipelines, Projects, Releases, Tags
+from source_gitlab.streams import (
+    Branches,
+    Commits,
+    Deployments,
+    Jobs,
+    MergeRequestCommits,
+    MergeRequests,
+    Pipelines,
+    Projects,
+    Releases,
+    Tags,
+)
 
 auth_params = {"authenticator": NoAuth(), "api_url": "gitlab.com"}
 start_date = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=14)
@@ -42,6 +53,11 @@ def jobs(pipelines):
     return Jobs(parent_stream=pipelines, **auth_params)
 
 
+@pytest.fixture()
+def deployments(projects):
+    return Deployments(parent_stream=projects, **auth_params)
+
+
 @pytest.fixture()
 def merge_request_commits(merge_requests):
     return MergeRequestCommits(parent_stream=merge_requests, **auth_params)
@@ -151,6 +167,36 @@ def test_should_retry(mocker, requests_mock, stream, extra_mocks, expected_call_
             }
         ],
     ),
+    (
+        "deployments",
+        (
+            (
+                "/api/v4/projects/p_1/deployments",
+                [
+                    {
+                        "id": "r_1",
+                        "user": {"name": "John", "id": "666", "username": "john"},
+                        "environment": {"name": "dev"},
+                        "commit": {"id": "abcd689"},
+                    }
+                ],
+            ),
+        ),
+        [
+            {
+                "id": "r_1",
+                "user": {"name": "John", "id": "666", "username": "john"},
+                "environment": {"name": "dev"},
+                "commit": {"id": "abcd689"},
+                "user_id": "666",
+                "environment_id": None,
+                "user_username": "john",
+                "user_full_name": "John",
+                "environment_name": "dev",
+                "project_id": "p_1",
+            }
+        ],
+    ),
     (
         "merge_request_commits",
         (

diff --git a/docs/integrations/sources/gitlab.md b/docs/integrations/sources/gitlab.md
@@ -94,6 +94,7 @@ This connector outputs the following streams:
 - [Group and Project members](https://docs.gitlab.com/ee/api/members.html)
 - [Tags](https://docs.gitlab.com/ee/api/tags.html)
 - [Releases](https://docs.gitlab.com/ee/api/releases/index.html)
+- [Deployments](https://docs.gitlab.com/ee/api/deployments/index.html)
 - [Group Labels](https://docs.gitlab.com/ee/api/group_labels.html)
 - [Project Labels](https://docs.gitlab.com/ee/api/labels.html)
 - [Epics](https://docs.gitlab.com/ee/api/epics.html) \(only available for GitLab Ultimate and GitLab.com Gold accounts\)
@@ -111,6 +112,7 @@ Gitlab has the [rate limits](https://docs.gitlab.com/ee/user/gitlab_com/index.ht
 
 | Version | Date       | Pull Request                                             | Subject                                                                                    |
 | :------ | :--------- | :------------------------------------------------------- | :----------------------------------------------------------------------------------------- |
+| 1.7.0   | 2023-08-08 | [27869](https://github.com/airbytehq/airbyte/pull/29203) | Add Deployments stream                                                                     |
 | 1.6.0   | 2023-06-30 | [27869](https://github.com/airbytehq/airbyte/pull/27869) | Add `shared_runners_setting` field to groups                                               |
 | 1.5.1   | 2023-06-24 | [27679](https://github.com/airbytehq/airbyte/pull/27679) | Fix formatting                                                                             |
 | 1.5.0   | 2023-06-15 | [27392](https://github.com/airbytehq/airbyte/pull/27392) | Make API URL an optional parameter in spec.                                                |