From d4cd83c79ba5d8cb1b43879764f1dbb54219145e Mon Sep 17 00:00:00 2001 From: btkcodedev Date: Thu, 5 Oct 2023 20:29:45 +0530 Subject: [PATCH 1/4] chore: Migrate Harness to low code --- .../connectors/source-harness/.dockerignore | 6 ++ .../connectors/source-harness/Dockerfile | 38 +++++++++ .../connectors/source-harness/README.md | 82 +++++++++++++++++++ .../connectors/source-harness/__init__.py | 3 + .../source-harness/acceptance-test-config.yml | 39 +++++++++ .../source-harness/acceptance-test-docker.sh | 3 + .../connectors/source-harness/build.gradle | 8 ++ .../connectors/source-harness/icon.svg | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 ++ .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/configured_catalog.json | 13 +++ .../integration_tests/invalid_config.json | 8 ++ .../integration_tests/sample_config.json | 8 ++ .../integration_tests/sample_state.json | 5 ++ .../connectors/source-harness/main.py | 13 +++ .../connectors/source-harness/metadata.yaml | 25 ++++++ .../source-harness/requirements.txt | 2 + .../connectors/source-harness/setup.py | 30 +++++++ .../source-harness/source_harness/__init__.py | 8 ++ .../source_harness/manifest.yaml | 47 +++++++++++ .../source_harness/schemas/organizations.json | 39 +++++++++ .../source-harness/source_harness/source.py | 18 ++++ .../source-harness/source_harness/spec.yaml | 42 ++++++++++ docs/integrations/sources/harness.md | 23 +++--- 25 files changed, 482 insertions(+), 11 deletions(-) create mode 100644 airbyte-integrations/connectors/source-harness/.dockerignore create mode 100644 airbyte-integrations/connectors/source-harness/Dockerfile create mode 100644 airbyte-integrations/connectors/source-harness/README.md create mode 100644 airbyte-integrations/connectors/source-harness/__init__.py create mode 100644 airbyte-integrations/connectors/source-harness/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-harness/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-harness/build.gradle create mode 100644 airbyte-integrations/connectors/source-harness/icon.svg create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-harness/main.py create mode 100644 airbyte-integrations/connectors/source-harness/metadata.yaml create mode 100644 airbyte-integrations/connectors/source-harness/requirements.txt create mode 100644 airbyte-integrations/connectors/source-harness/setup.py create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/__init__.py create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/source.py create mode 100644 airbyte-integrations/connectors/source-harness/source_harness/spec.yaml diff --git a/airbyte-integrations/connectors/source-harness/.dockerignore b/airbyte-integrations/connectors/source-harness/.dockerignore new file mode 100644 index 000000000000..40467f139afd --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_harness +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-harness/Dockerfile b/airbyte-integrations/connectors/source-harness/Dockerfile new file mode 100644 index 000000000000..8542d6eca698 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_harness ./source_harness + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-harness diff --git a/airbyte-integrations/connectors/source-harness/README.md b/airbyte-integrations/connectors/source-harness/README.md new file mode 100644 index 000000000000..31e5b516fdee --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/README.md @@ -0,0 +1,82 @@ +# Harness Source + +This is the repository for the Harness configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/harness). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-harness:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/harness) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_harness/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source harness test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-harness:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-harness:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-harness:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harness:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-harness:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-harness:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with Docker, run: +``` +./acceptance-test-docker.sh +``` + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-harness:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-harness:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-harness/__init__.py b/airbyte-integrations/connectors/source-harness/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml new file mode 100644 index 000000000000..b417da61bfb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml @@ -0,0 +1,39 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-harness:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_harness/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-harness/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-harness/acceptance-test-docker.sh new file mode 100755 index 000000000000..b6d65deeccb4 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/acceptance-test-docker.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env sh + +source "$(git rev-parse --show-toplevel)/airbyte-integrations/bases/connector-acceptance-test/acceptance-test-docker.sh" diff --git a/airbyte-integrations/connectors/source-harness/build.gradle b/airbyte-integrations/connectors/source-harness/build.gradle new file mode 100644 index 000000000000..1f9d2819b281 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/build.gradle @@ -0,0 +1,8 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' +} + +airbytePython { + moduleDirectory 'source_harness' +} diff --git a/airbyte-integrations/connectors/source-harness/icon.svg b/airbyte-integrations/connectors/source-harness/icon.svg new file mode 100644 index 000000000000..e1770dde603c --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/icon.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py b/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..4cf0f64f27b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/configured_catalog.json @@ -0,0 +1,13 @@ +{ + "streams": [ + { + "stream": { + "name": "organizations", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json new file mode 100644 index 000000000000..2bf7d5ea14f9 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "", + "account_id": "xxxxxxxxxxxxxxxxx", + "api_url": "https://app.harness.io", + "cutoff_days": 90, + "page_size": 100, + "deployment_timeout": 100 +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json new file mode 100644 index 000000000000..a853a1b18dcc --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json @@ -0,0 +1,8 @@ +{ + "api_key": "xxxxxxxxxxxxxxxxxxxxxxxxxxx", + "account_id": "xxxxxxxxxxxxxxxxx", + "api_url": "https://app.harness.io", + "cutoff_days": 90, + "page_size": 100, + "deployment_timeout": 100 +} diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-harness/main.py b/airbyte-integrations/connectors/source-harness/main.py new file mode 100644 index 000000000000..b323465b96c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_harness import SourceHarness + +if __name__ == "__main__": + source = SourceHarness() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-harness/metadata.yaml b/airbyte-integrations/connectors/source-harness/metadata.yaml new file mode 100644 index 000000000000..857504fc2505 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/metadata.yaml @@ -0,0 +1,25 @@ +data: + allowedHosts: + hosts: + - api.harness.io + registries: + oss: + enabled: false + cloud: + enabled: false + connectorSubtype: api + connectorType: source + definitionId: b0e46f61-e143-47cc-a595-4bb73bfa8a15 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-harness + githubIssueLabel: source-harness + icon: harness.svg + license: MIT + name: Harness + releaseDate: 2023-10-10 + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/harness + tags: + - language:lowcode +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-harness/requirements.txt b/airbyte-integrations/connectors/source-harness/requirements.txt new file mode 100644 index 000000000000..cc57334ef619 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/connector-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-harness/setup.py b/airbyte-integrations/connectors/source-harness/setup.py new file mode 100644 index 000000000000..00b40a22b378 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_harness", + description="Source implementation for Harness.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-harness/source_harness/__init__.py b/airbyte-integrations/connectors/source-harness/source_harness/__init__.py new file mode 100644 index 000000000000..1af39ecab1b1 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceHarness + +__all__ = ["SourceHarness"] diff --git a/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml new file mode 100644 index 000000000000..f4cf0f5df5ba --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml @@ -0,0 +1,47 @@ +version: "0.29.0" + +definitions: + selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: ["data", "content"] + requester: + type: HttpRequester + url_base: "{{ config['api_url'] }}" + http_method: "GET" + authenticator: + type: "ApiKeyAuthenticator" + header: "x-api-key" + api_token: "{{ config['api_key'] }}" + request_parameters: + accountIdentifier: "{{ config['account_id'] }}" + + retriever: + type: SimpleRetriever + record_selector: + $ref: "#/definitions/selector" + paginator: + type: NoPagination + requester: + $ref: "#/definitions/requester" + + base_stream: + type: DeclarativeStream + retriever: + $ref: "#/definitions/retriever" + + organizations_stream: + $ref: "#/definitions/base_stream" + name: "organizations" + $parameters: + path: "/ng/api/organizations" + +streams: + - "#/definitions/organizations_stream" + +check: + type: CheckStream + stream_names: + - "organizations" + diff --git a/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json new file mode 100644 index 000000000000..d8902af093a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json @@ -0,0 +1,39 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Organizations schema", + "additionalProperties": true, + "type": ["object", "null"], + "properties": { + "organization": { + "type": ["object", "null"], + "properties": { + "identifier": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "tags": { + "type": ["object", "null"], + "properties": { + "identifier": { + "type": ["string", "null"] + } + } + } + } + }, + "createdAt": { + "type": ["number", "null"] + }, + "lastModifiedAt": { + "type": ["number", "null"] + }, + "harnessManaged": { + "type": ["boolean", "null"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-harness/source_harness/source.py b/airbyte-integrations/connectors/source-harness/source_harness/source.py new file mode 100644 index 000000000000..a52f4c06db86 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceHarness(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"}) diff --git a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml new file mode 100644 index 000000000000..9eea7d6ed241 --- /dev/null +++ b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml @@ -0,0 +1,42 @@ +documentationUrl: https://docs.airbyte.com/integrations/sources/harness +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Harness Spec + type: object + required: + - api_key + - account_id + - cutoff_days + additionalProperties: true + properties: + api_key: + type: string + title: API key + airbyte_secret: true + account_id: + type: string + title: Account ID + description: Harness Account ID + api_url: + type: string + title: API URL + description: The API URL for fetching data from Harness + default: https://app.harness.io + examples: + - https://my-harness-server.example.com + cutoff_days: + type: integer + title: Cutoff Days + default: 90 + description: Only fetch deployments updated after cutoff + page_size: + type: integer + title: Page Size + default: 100 + description: number of pipelines (builds) to fetch per call + deployment_timeout: + type: integer + title: Deployment Timeout + default: 24 + description: Max number of hours to consider for a deployment to be running/queued + before giving up on syncing it diff --git a/docs/integrations/sources/harness.md b/docs/integrations/sources/harness.md index f858f3617dbf..94ddd0d1ebf3 100644 --- a/docs/integrations/sources/harness.md +++ b/docs/integrations/sources/harness.md @@ -2,7 +2,7 @@ ## Overview -The Harness source is maintained by [Faros +The Harness source is migrated from [Faros AI](https://github.com/faros-ai/airbyte-connectors/tree/main/sources/harness-source). Please file any support requests on that repo to minimize response time from the maintainers. The source supports both Full Refresh and Incremental syncs. You @@ -13,19 +13,19 @@ the tables and columns you set up for replication, every time a sync is run. Only one stream is currently available from this source: -* [Executions](https://docs.harness.io/article/ba4vs50071-use-workflows-api) \(Incremental\) +* [Organization](https://apidocs.harness.io/tag/Organization#operation/getOrganizationList) If there are more endpoints you'd like Faros AI to support, please [create an issue.](https://github.com/faros-ai/airbyte-connectors/issues/new) ### Features -| Feature | Supported? | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental Sync | Yes | -| SSL connection | Yes | -| Namespaces | No | +| Feature | Supported? | +| :----------------- | :--------- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| SSL connection | No | +| Namespaces | No | ### Performance considerations @@ -47,6 +47,7 @@ Key](https://ngdocs.harness.io/article/tdoad7xrh9-add-and-manage-api-keys#harnes ## Changelog -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| 0.1.23 | 2021-11-16 | [153](https://github.com/faros-ai/airbyte-connectors/pull/153) | Add Harness source and Faros destination's converter | +| Version | Date | Pull Request | Subject | +| :--------- | :--------- | :------------------------------------------------------------------ | :---------------------------------------------------- | +| 0.2.0 | 2023-10-10 | [000000](https://github.com/airbyte/airbyte-connectors/pull/000000) | Migrate to low code | +| 0.1.23 | 2021-11-16 | [153](https://github.com/faros-ai/airbyte-connectors/pull/153) | Add Harness source and Faros destination's converter | From 5d765edbacac0c82a8c441c2638962c27a11c331 Mon Sep 17 00:00:00 2001 From: btkcodedev Date: Thu, 5 Oct 2023 20:38:24 +0530 Subject: [PATCH 2/4] chore: remove unwanted fields, update docs --- .../integration_tests/invalid_config.json | 5 +---- .../integration_tests/sample_config.json | 5 +---- .../source-harness/source_harness/spec.yaml | 17 ----------------- docs/integrations/sources/harness.md | 2 +- 4 files changed, 3 insertions(+), 26 deletions(-) diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json index 2bf7d5ea14f9..f0d7d0c01c14 100644 --- a/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-harness/integration_tests/invalid_config.json @@ -1,8 +1,5 @@ { "api_key": "", "account_id": "xxxxxxxxxxxxxxxxx", - "api_url": "https://app.harness.io", - "cutoff_days": 90, - "page_size": 100, - "deployment_timeout": 100 + "api_url": "https://app.harness.io" } diff --git a/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json index a853a1b18dcc..b792a18806ff 100644 --- a/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-harness/integration_tests/sample_config.json @@ -1,8 +1,5 @@ { "api_key": "xxxxxxxxxxxxxxxxxxxxxxxxxxx", "account_id": "xxxxxxxxxxxxxxxxx", - "api_url": "https://app.harness.io", - "cutoff_days": 90, - "page_size": 100, - "deployment_timeout": 100 + "api_url": "https://app.harness.io" } diff --git a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml index 9eea7d6ed241..0a7ec16e751d 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml +++ b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml @@ -6,7 +6,6 @@ connectionSpecification: required: - api_key - account_id - - cutoff_days additionalProperties: true properties: api_key: @@ -24,19 +23,3 @@ connectionSpecification: default: https://app.harness.io examples: - https://my-harness-server.example.com - cutoff_days: - type: integer - title: Cutoff Days - default: 90 - description: Only fetch deployments updated after cutoff - page_size: - type: integer - title: Page Size - default: 100 - description: number of pipelines (builds) to fetch per call - deployment_timeout: - type: integer - title: Deployment Timeout - default: 24 - description: Max number of hours to consider for a deployment to be running/queued - before giving up on syncing it diff --git a/docs/integrations/sources/harness.md b/docs/integrations/sources/harness.md index 94ddd0d1ebf3..b6433e30483a 100644 --- a/docs/integrations/sources/harness.md +++ b/docs/integrations/sources/harness.md @@ -49,5 +49,5 @@ Key](https://ngdocs.harness.io/article/tdoad7xrh9-add-and-manage-api-keys#harnes | Version | Date | Pull Request | Subject | | :--------- | :--------- | :------------------------------------------------------------------ | :---------------------------------------------------- | -| 0.2.0 | 2023-10-10 | [000000](https://github.com/airbyte/airbyte-connectors/pull/000000) | Migrate to low code | +| 0.1.0 | 2023-10-10 | [31103](https://github.com/airbytehq/airbyte/pull/31103) | Migrate to low code | | 0.1.23 | 2021-11-16 | [153](https://github.com/faros-ai/airbyte-connectors/pull/153) | Add Harness source and Faros destination's converter | From 73af381af34f3bf8973984d9377a1edb910535cc Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 23 Oct 2023 15:26:32 -0300 Subject: [PATCH 3/4] remove files and imports --- .../connectors/source-harness/build.gradle | 8 -------- airbyte-integrations/connectors/source-harness/setup.py | 5 +---- 2 files changed, 1 insertion(+), 12 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-harness/build.gradle diff --git a/airbyte-integrations/connectors/source-harness/build.gradle b/airbyte-integrations/connectors/source-harness/build.gradle deleted file mode 100644 index 1f9d2819b281..000000000000 --- a/airbyte-integrations/connectors/source-harness/build.gradle +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - id 'airbyte-python' - id 'airbyte-docker' -} - -airbytePython { - moduleDirectory 'source_harness' -} diff --git a/airbyte-integrations/connectors/source-harness/setup.py b/airbyte-integrations/connectors/source-harness/setup.py index 00b40a22b378..6bef3ce1447c 100644 --- a/airbyte-integrations/connectors/source-harness/setup.py +++ b/airbyte-integrations/connectors/source-harness/setup.py @@ -5,15 +5,12 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = [ - "airbyte-cdk~=0.1", -] +MAIN_REQUIREMENTS = ["airbyte-cdk"] TEST_REQUIREMENTS = [ "requests-mock~=1.9.3", "pytest~=6.2", "pytest-mock~=3.6.1", - "connector-acceptance-test", ] setup( From 8db461fb7fe4a10fe07fd7cc0bccf5361fef5d35 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 23 Oct 2023 15:32:52 -0300 Subject: [PATCH 4/4] run format --- .../source-harness/acceptance-test-config.yml | 26 +++++++++---------- .../source_harness/manifest.yaml | 1 - .../source_harness/schemas/organizations.json | 2 +- .../source-harness/source_harness/spec.yaml | 6 ++--- 4 files changed, 17 insertions(+), 18 deletions(-) diff --git a/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml index b417da61bfb3..28456c8a61fb 100644 --- a/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-harness/acceptance-test-config.yml @@ -19,20 +19,20 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.jsonl" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: + # TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file + # expect_records: + # path: "integration_tests/expected_records.jsonl" + # extra_fields: no + # exact_order: no + # extra_records: yes + incremental: bypass_reason: "This connector does not implement incremental sync" -# TODO uncomment this block this block if your connector implements incremental sync: -# tests: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state: -# future_state_path: "integration_tests/abnormal_state.json" + # TODO uncomment this block this block if your connector implements incremental sync: + # tests: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state: + # future_state_path: "integration_tests/abnormal_state.json" full_refresh: tests: - config_path: "secrets/config.json" diff --git a/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml index f4cf0f5df5ba..e01c0d3bd201 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml +++ b/airbyte-integrations/connectors/source-harness/source_harness/manifest.yaml @@ -44,4 +44,3 @@ check: type: CheckStream stream_names: - "organizations" - diff --git a/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json index d8902af093a6..88372907ea2b 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json +++ b/airbyte-integrations/connectors/source-harness/source_harness/schemas/organizations.json @@ -36,4 +36,4 @@ "type": ["boolean", "null"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml index 0a7ec16e751d..d0bcc389d5f7 100644 --- a/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml +++ b/airbyte-integrations/connectors/source-harness/source_harness/spec.yaml @@ -4,8 +4,8 @@ connectionSpecification: title: Harness Spec type: object required: - - api_key - - account_id + - api_key + - account_id additionalProperties: true properties: api_key: @@ -22,4 +22,4 @@ connectionSpecification: description: The API URL for fetching data from Harness default: https://app.harness.io examples: - - https://my-harness-server.example.com + - https://my-harness-server.example.com