✨Source Looker - Migrate Python CDK to Low-code CDK (#37464)

Co-authored-by: Alexandre Girard <[email protected]> Co-authored-by: Natik Gadzhi <[email protected]>
airbytehq · Jul 31, 2024 · 54c1315 · 54c1315
1 parent c8b95c7
commit 54c1315
Show file tree

Hide file tree

Showing 17 changed files with 2,578 additions and 1,199 deletions.
diff --git a/airbyte-integrations/connectors/source-looker/README.md b/airbyte-integrations/connectors/source-looker/README.md
@@ -1,26 +1,30 @@
 # Looker source connector
 
 
-This is the repository for the Looker source connector, written in Python.
+This is the repository for the Looker configuration based source connector.
 For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/looker).
 
 ## Local development
 
 ### Prerequisites
-* Python (~=3.9)
-* Poetry (~=1.7) - installation instructions [here](https://python-poetry.org/docs/#installation)
+
+* Python (`^3.9`)
+* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation)
+
 
 
 ### Installing the connector
+
 From this connector directory, run:
 ```bash
 poetry install --with dev
 ```
 
 
 ### Create credentials
+
 **If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/looker)
-to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_looker/spec.yaml` file.
+to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_looker/spec.yaml` file.
 Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information.
 See `sample_files/sample_config.json` for a sample config file.
 
@@ -31,25 +35,34 @@ poetry run source-looker spec
 poetry run source-looker check --config secrets/config.json
 poetry run source-looker discover --config secrets/config.json
 poetry run source-looker read --config secrets/config.json --catalog sample_files/configured_catalog.json
+poetry run source-looker spec
+poetry run source-looker check --config secrets/config.json
+poetry run source-looker discover --config secrets/config.json
+poetry run source-looker read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
-### Running unit tests
-To run unit tests locally, from the connector directory run:
+### Running tests
+
+To run tests locally, from the connector directory run:
+
 ```
-poetry run pytest unit_tests
+poetry run pytest tests
 ```
 
 ### Building the docker image
+
 1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md)
 2. Run the following command to build the docker image:
 ```bash
 airbyte-ci connectors --name=source-looker build
 ```
 
 An image will be available on your host with the tag `airbyte/source-looker:dev`.
+An image will be available on your host with the tag `airbyte/source-looker:dev`.
 
 
 ### Running as a docker container
+
 Then run any of the connector commands as follows:
 ```
 docker run --rm airbyte/source-looker:dev spec
@@ -59,16 +72,19 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integrat
 ```
 
 ### Running our CI test suite
+
 You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md):
 ```bash
 airbyte-ci connectors --name=source-looker test
 ```
 
 ### Customizing acceptance Tests
+
 Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information.
 If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
 
 ### Dependency Management
+
 All of your dependencies should be managed via Poetry. 
 To add a new dependency, run:
 ```bash
@@ -78,6 +94,7 @@ poetry add <package-name>
 Please commit the changes to `pyproject.toml` and `poetry.lock` files.
 
 ## Publishing a new version of the connector
+
 You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
 1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-looker test`
 2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): 

diff --git a/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-looker/acceptance-test-config.yml
@@ -3,7 +3,7 @@
 connector_image: airbyte/source-looker:dev
 tests:
   spec:
-    - spec_path: "source_looker/spec.json"
+    - spec_path: "source_looker/spec.yaml"
   connection:
     - config_path: "secrets/config.json"
       status: "succeed"

diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-looker/integration_tests/abnormal_state.json
@@ -1,5 +1,5 @@
 {
   "query_history": {
-    "history_created_time": "2050-01-01T00:00:00Z"
+    "history.created_time": "2050-01-01T00:00:00Z"
   }
 }
diff --git a/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-looker/integration_tests/configured_catalog.json
@@ -140,7 +140,7 @@
     },
     {
       "stream": {
-        "name": "homepage_items",
+        "name": "board_items",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
@@ -150,7 +150,7 @@
     },
     {
       "stream": {
-        "name": "homepage_sections",
+        "name": "board_sections",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
@@ -160,7 +160,7 @@
     },
     {
       "stream": {
-        "name": "homepages",
+        "name": "boards",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh"],
         "source_defined_primary_key": [["id"]]
@@ -198,16 +198,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "lookml_dashboards",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "lookml_models",
@@ -292,8 +282,8 @@
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
         "source_defined_cursor": true,
-        "default_cursor_field": ["history_created_time"],
-        "source_defined_primary_key": [["query_id"], ["history_created_time"]]
+        "default_cursor_field": ["history.created_time"],
+        "source_defined_primary_key": [["query.id"], ["history.created_time"]]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append"
@@ -328,26 +318,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "spaces",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
-    {
-      "stream": {
-        "name": "space_ancestors",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
-        "source_defined_primary_key": [["id"]]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "user_attributes",
@@ -368,15 +338,6 @@
       "sync_mode": "full_refresh",
       "destination_sync_mode": "append"
     },
-    {
-      "stream": {
-        "name": "user_attribute_values",
-        "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
-      },
-      "sync_mode": "full_refresh",
-      "destination_sync_mode": "append"
-    },
     {
       "stream": {
         "name": "user_login_lockouts",

diff --git a/airbyte-integrations/connectors/source-looker/metadata.yaml b/airbyte-integrations/connectors/source-looker/metadata.yaml
@@ -2,7 +2,7 @@ data:
   connectorSubtype: api
   connectorType: source
   definitionId: 00405b19-9768-4e0c-b1ae-9fc2ee2b2a8c
-  dockerImageTag: 0.2.12
+  dockerImageTag: 1.0.0
   dockerRepository: airbyte/source-looker
   githubIssueLabel: source-looker
   icon: looker.svg
@@ -18,10 +18,15 @@ data:
     oss:
       enabled: true
   releaseStage: alpha
+  releases:
+    breakingChanges:
+      1.0.0:
+        message: Version 1.0.0 introduces changes to the connection configuration which has been upgraded to the API v4.0 from v3.1. Due to this upgrade, certain streams were affected and schemas changed generally.
+        upgradeDeadline: "2024-08-15"
   documentationUrl: https://docs.airbyte.com/integrations/sources/looker
   tags:
     - language:python
-    - cdk:python
+    - cdk:low-code
   ab_internal:
     sl: 100
     ql: 100
@@ -40,5 +45,5 @@ data:
   #           type: GSM
   #           alias: airbyte-connector-testing-secret-store
   connectorBuildOptions:
-    baseImage: docker.io/airbyte/python-connector-base:1.2.2@sha256:57703de3b4c4204bd68a7b13c9300f8e03c0189bffddaffc796f1da25d2dbea0
+    baseImage: docker.io/airbyte/python-connector-base:2.0.0@sha256:c44839ba84406116e8ba68722a0f30e8f6e7056c726f447681bb9e9ece8bd916
 metadataSpecVersion: "1.0"